Context Navigation

source: XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/check.py @ 1771

Last change on this file since 1771 was 1771, checked in by yushan, 5 years ago
dev_trunk_omp : add xios_report.html
File size: 17.4 KB

Line
1	import os
2	import sys
3	import subprocess
4	import socket
5	import os.path
6	from os import path
7
8
9	from default_param import *
10	from user_param import *
11
12	# load default_param first, variables defines in user_param will overwrite the default value
13
14
15
16	def OSinfo(runthis):
17	red = lambda text: '\033[0;31m' + text + '\033[0m'
18	osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
19	theInfo = osstdout.communicate()[0].strip()
20	if osstdout.returncode!=0:
21	print(red(runthis+" FAILED"))
22	print >> sys.stderr, osstdout.returncode
23	sys.exit()
24	# else:
25	# print(runthis+" OK")
26
27
28	def write_jz_job(configName, nbproc=8):
29	red = lambda text: '\033[0;31m' + text + '\033[0m'
30	f=open("jz_job_"+configName+".sh", "w")
31	f.write("#!/bin/bash\n")
32	f.write("#SBATCH --ntasks="+repr(nbproc)+" # Nombre total de processus MPI\n")
33	f.write("#SBATCH --ntasks-per-node=16\n")
34	f.write("#SBATCH --hint=nomultithread\n")
35	f.write("#SBATCH -t 00:10:00 # Temps elapsed maximum d'execution\n")
36	f.write("#SBATCH -o single_config.out # Nom du fichier de sortie\n")
37	f.write("#SBATCH -e single_config.err # Nom du fichier d'erreur\n")
38	f.write("#SBATCH --account=psl@cpu\n")
39	f.write("cd ${SLURM_SUBMIT_DIR}\n")
40	f.write("module purge\n")
41	f.write("module load intel-all/19.0.4\n")
42	f.write("module load netcdf/4.7.0/intel-19.0.4-mpi\n")
43	f.write("module load netcdf-cxx/4.2/intel-19.0.4-mpi\n")
44	f.write("module load netcdf-fortran/4.4.5/intel-19.0.4-mpi\n")
45	f.write("module load gcc/6.5.0/gcc-4.8.5\n")
46	f.write("module load cmake\n")
47	f.write("module load cdo\n")
48	f.write("ulimit -c 0\n")
49	xios_root=os.getenv('xios_dir')
50	build_folder=os.getenv('build_dir')
51	f.write("export xios_dir="+xios_root+"\n")
52	f.write("export build_dir="+build_folder+"\n")
53	f.write("export machine_name=jeanzay\n")
54	f.write("cp ../setup.py ./\n")
55	f.write("cp ../run_sub_test.sh ./\n")
56	f.write("mv user_params.def user_params.def_bkp\n")
57	f.write("cp "+configName+"/setup/user_param.py user_params.def\n")
58	f.write("python setup.py\n")
59	f.write("cmake .\n")
60	f.write("ctest -V #enable all output\n")
61	f.write("rm -f setup.py\n")
62	f.write("rm -f run_sub_test.sh\n")
63	f.write("rm -f run_test_*.py\n")
64	f.write("rm -f CMakeLists.txt\n")
65	f.write("rm -f context_grid_dynamico.xml\n")
66	f.write("rm -f dynamico_grid.nc\n")
67	f.write("rm -f default_param.*\n")
68	f.write("rm -f user_param.*\n")
69	f.write("mv user_params.def_bkp user_params.def\n")
70	f.write("\n")
71	f.close
72
73	def write_irene_job(configName, nbproc=8):
74	red = lambda text: '\033[0;31m' + text + '\033[0m'
75	f=open("irene_job_"+configName+".sh", "w")
76	f.write("#!/bin/bash\n")
77	f.write("#MSUB -r XIOS\n")
78	f.write("#MSUB -o client_output.out # standard output\n")
79	f.write("#MSUB -e client_error.err #error output\n")
80	f.write("#MSUB -eo\n")
81	f.write("#MSUB -c 1\n")
82	f.write("#MSUB -n "+ repr(nbproc) + " # Number of MPI tasks (SPMD case) or cores (MPMD case)\n")
83	f.write("#MSUB -X \n")
84	f.write("#MSUB -x \n")
85	f.write("#MSUB -T 600 # Wall clock limit (seconds)\n")
86	f.write("#MSUB -q skylake # thin nodes\n")
87	f.write("#MSUB -A devcmip6\n")
88	f.write("#MSUB -Q test\n")
89	f.write("#MSUB -m work\n")
90	f.write("cd $BRIDGE_MSUB_PWD\n")
91	f.write("module unload netcdf-c netcdf-fortran hdf5 flavor perl hdf5 boost blitz mpi gnu\n")
92	f.write("module load gnu\n")
93	f.write("module load mpi/intelmpi/2017.0.6.256\n")
94	f.write("module load flavor/buildcompiler/intel/17\n")
95	f.write("module load flavor/hdf5/parallel\n")
96	f.write("module load netcdf-fortran/4.4.4\n")
97	f.write("module load hdf5/1.8.20\n")
98	f.write("module load boost\n")
99	f.write("module load blitz\n")
100	f.write("module load feature/bridge/heterogenous_mpmd\n")
101	f.write("module load nco\n")
102	f.write("module load cdo\n")
103	f.write("export KMP_STACKSIZE=3g\n")
104	f.write("export KMP_LIBRARY=turnaround\n")
105	f.write("export MKL_SERIAL=YES\n")
106	f.write("export OMP_NUM_THREADS=${BRIDGE_MSUB_NCORE}\n")
107	f.write("set -x \n")
108	xios_root=os.getenv('xios_dir')
109	build_folder=os.getenv('build_dir')
110	f.write("export xios_dir="+xios_root+"\n")
111	f.write("export build_dir="+build_folder+"\n")
112	f.write("export machine_name=irene\n")
113	f.write("cp ../setup.py ./\n")
114	f.write("cp ../run_sub_test.sh ./\n")
115	f.write("mv user_params.def user_params.def_bkp\n")
116	f.write("cp "+configName+"/setup/user_param.py user_params.def\n")
117	f.write("python setup.py\n")
118	f.write("cmake .\n")
119	f.write("ctest -V #enable all output\n")
120	f.write("rm -f setup.py\n")
121	f.write("rm -f run_sub_test.sh\n")
122	f.write("rm -f run_test_*.py\n")
123	f.write("rm -f CMakeLists.txt\n")
124	f.write("rm -f context_grid_dynamico.xml\n")
125	f.write("rm -f dynamico_grid.nc\n")
126	f.write("rm -f default_param.*\n")
127	f.write("rm -f user_param.*\n")
128	f.write("mv user_params.def_bkp user_params.def\n")
129	f.write("\n")
130	f.close
131
132
133
134	def main():
135
136	black = lambda text: '\033[0;30m' + text + '\033[0m'
137	red = lambda text: '\033[0;31m' + text + '\033[0m'
138	green = lambda text: '\033[0;32m' + text + '\033[0m'
139	yellow = lambda text: '\033[0;33m' + text + '\033[0m'
140	blue = lambda text: '\033[0;34m' + text + '\033[0m'
141	magenta = lambda text: '\033[0;35m' + text + '\033[0m'
142	cyan = lambda text: '\033[0;36m' + text + '\033[0m'
143	lgray = lambda text: '\033[0;37m' + text + '\033[0m'
144	dgray = lambda text: '\033[1;30m' + text + '\033[0m'
145	lred = lambda text: '\033[1;32m' + text + '\033[0m'
146	lgreen = lambda text: '\033[1;32m' + text + '\033[0m'
147	lyellow = lambda text: '\033[1;33m' + text + '\033[0m'
148	lblue = lambda text: '\033[1;34m' + text + '\033[0m'
149	lpurple = lambda text: '\033[1;35m' + text + '\033[0m'
150	lcyan = lambda text: '\033[1;36m' + text + '\033[0m'
151	white = lambda text: '\033[1;37m' + text + '\033[0m'
152
153	configNamePattern=[0]*7
154
155	f=open("user_param.py", "r")
156	for line in f:
157	line = line.strip()
158	if line.startswith("UsingServer2"):
159	configNamePattern[0] = '_UsingSrv2=' + UsingServer2
160	elif line.startswith("RatioServer2"):
161	configNamePattern[1] = '_RatioSrv2=' + repr(RatioServer2)
162	elif line.startswith("NumberPoolsServer2"):
163	configNamePattern[2] = '_NbPoolsSrv2=' + repr(NumberPoolsServer2)
164	elif line.startswith("NumberClients"):
165	configNamePattern[3] = '_NbClients=' + repr(NumberClients)
166	elif line.startswith("NumberServers"):
167	configNamePattern[4] = '_NbServers=' + repr(NumberServers)
168	elif line.startswith("Duration"):
169	configNamePattern[5] = '_Duration=' + Duration
170	elif line.startswith("ATMdomain"):
171	configNamePattern[5] = '_ATMdomain=' + ATMdomain
172	f.close()
173
174
175
176	configName = "config"
177	defaultConfig = True
178
179	for i in range(len(configNamePattern)):
180	if configNamePattern[i] != 0:
181	configName = configName + configNamePattern[i]
182	defaultConfig = False
183
184	if defaultConfig:
185	configName = "config_default"
186
187
188	if not os.path.exists(configName):
189
190
191
192
193	os.mkdir(configName)
194	print blue("\n ****************************************************************")
195	print blue(" * Directory " + repr(configName) + " Created ")
196	print blue(" * Copy results to \'not_validated_reference\' folder")
197	print blue(" ****************************************************************\n")
198
199	OSinfo("mkdir "+configName+'/setup')
200	OSinfo('mkdir '+configName+'/tmp_reference')
201	OSinfo('mkdir '+configName+'/xios_output')
202
203	OSinfo('mv param.def '+configName+'/setup/')
204	OSinfo('mv all_param.def '+configName+'/setup/')
205	OSinfo('mv user_param.py '+configName+'/setup/')
206	OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ')
207	OSinfo('mv iodef.xml '+configName+'/setup/')
208	OSinfo('cp context_atm.xml '+configName+'/setup/')
209	OSinfo('cp context_grid_dynamico.xml '+configName+'/setup/')
210	OSinfo('cp dynamico_grid.nc '+configName+'/setup/')
211	OSinfo('mv xios_*.out '+configName+'/xios_output/')
212	OSinfo('mv xios_*.err '+configName+'/xios_output/')
213
214
215	f=open("checkfile.def", "r")
216	h=open("report.txt", "a+")
217	l=open("report.html", "a+")
218	preport=open("../plein_report.txt", "a+")
219	h.write("Config : "+configName+"\n")
220	l.write("<p>Config : "+configName+"</p>")
221	endloop=False
222	for line in f:
223	line=line.strip()
224	line=line.rstrip()
225
226	if line.startswith("all"):
227	allfiles = os.listdir(os.getcwd())
228	for thisfile in allfiles:
229	if thisfile.endswith(".nc") and thisfile!="dynamico_grid.nc":
230	cmd = 'mv '+thisfile + ' '+configName+'/tmp_reference/'
231	OSinfo(cmd)
232
233	h.write("test for "+lpurple("all NC files")+" \t"+yellow('INITIALIZED')+"\n")
234	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"_all_nc_files"+" "+repr(0)+" \n")
235	l.write("<p>test for <font color=\"purple\">"+"all NC files"+"</font>        <font color=\"darkorange\">INITIALIZED</font></p>")
236	print yellow("\n ****************************************")
237	print yellow(" "+"all NC files"+" is stored as temporal reference !!! ")
238	print yellow("\n ****************************************")
239
240	break
241
242	if not line.startswith("#") and len(line) != 0:
243	cmd = 'mv '+line + ' '+configName+'/tmp_reference/'
244	OSinfo(cmd)
245
246	h.write("test for "+lpurple(line)+" \t"+yellow('INITIALIZED')+"\n")
247	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n")
248	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"darkorange\">INITIALIZED</font></p>")
249	print yellow("\n ****************************************")
250	print yellow(" "+line+" is stored as temporal reference !!! ")
251	print yellow("\n ****************************************")
252
253	f.close()
254	h.write("\n")
255	l.write("<br>")
256	h.close()
257	l.close()
258	preport.close()
259	#generate job scripts
260	hostname=os.getenv('machine_name')
261	if hostname.startswith("jeanzay"):
262	write_jz_job(configName, NumberClients+NumberServers)
263
264	elif hostname.startswith("irene"):
265	write_irene_job(configName, NumberClients+NumberServers)
266
267	else :
268	print("machine unknown")
269	print >> sys.stderr, 1
270	sys.exit()
271
272
273	else:
274
275
276
277	print blue("\n ********************************************************")
278	print blue(" * Directory " + repr(configName) + " already exists")
279	print blue(" * Start comparing results with references")
280	print blue(" ********************************************************\n")
281
282	tmp_ref = os.path.isdir(configName+'/tmp_reference')
283	ref = os.path.isdir(configName+'/reference') and not tmp_ref
284
285	if ref:
286	print blue("\n =====================================")
287	print blue(" \| Comparing results with references \|")
288	print blue(" =====================================\n")
289	path = configName+'/reference/'
290	elif tmp_ref:
291	print blue("\n ===================================================")
292	print blue(" \| Comparing results with NON-Validated references \|")
293	print blue(" ===================================================\n")
294	path = configName+'/tmp_reference/'
295
296	f=open("checkfile.def", "r")
297	h=open("report.txt", "a+")
298	l=open("report.html", "a+")
299	preport=open("../plein_report.txt", "a+")
300
301
302	h.write("Config : "+configName+"\n")
303	l.write("<p>Config : "+configName+"</p>")
304	for line in f:
305
306	line=line.strip()
307	line=line.rstrip()
308
309	if line.startswith("all") :
310	for thisfile in os.listdir(path):
311	if thisfile.endswith(".nc"):
312	cmd = 'cdo -W diffn '+ thisfile +' '+ path + thisfile + ' \| tail -1 > diff.txt'
313
314	OSinfo(cmd)
315
316	if os.stat("diff.txt").st_size==0:
317	h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n")
318	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n")
319	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"green\">PASSED</font></p>")
320	print green("\n ****************************************")
321	print green(" "+thisfile+" is valid !!! ")
322	print green("\n ****************************************")
323
324	else:
325	g=open("diff.txt", "r")
326	for gline in g:
327	if gline.strip().startswith("0") or ":" in gline :
328	h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n")
329	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n")
330	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"green\">PASSED</font></p>")
331	print green("\n ****************************************")
332	print green(" "+thisfile+" is valid !!! ")
333	print green("\n ****************************************")
334	else:
335	h.write("test for "+lpurple(thisfile)+" \t"+red('FAILED')+"\n")
336	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(-1)+" \n")
337	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"red\">FAILED</font></p>")
338	print red("\n **************************************************")
339	print red(" "+thisfile+" is NOT valid. Please debugging.. ")
340	print red("\n **************************************************")
341	g.close()
342
343	# generate job submission script
344	hostname=os.getenv('machine_name')
345
346	if hostname.startswith("jeanzay"):
347	write_jz_job(configName, NumberClients+NumberServers)
348
349	elif hostname.startswith("irene"):
350	write_irene_job(configName, NumberClients+NumberServers)
351
352	else :
353	print("machine unknown")
354	print >> sys.stderr, 1
355	sys.exit()
356	break
357
358
359	elif not line.startswith("#") and len(line) != 0:
360	cmd = 'cdo -W diffn '+ line +' '+ path + line + ' \| tail -1 > diff.txt'
361
362	OSinfo(cmd)
363
364	if os.stat("diff.txt").st_size==0:
365	h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n")
366	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")
367	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"green\">PASSED</font></p>")
368	print green("\n ****************************************")
369	print green(" "+line+" is valid !!! ")
370	print green("\n ****************************************")
371
372	else:
373	g=open("diff.txt", "r")
374	for gline in g:
375	if gline.strip().startswith("0") or ":" in gline :
376	h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n")
377	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")
378	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"green\">PASSED</font></p>")
379	print green("\n ****************************************")
380	print green(" "+line+" is valid !!! ")
381	print green("\n ****************************************")
382	else:
383	h.write("test for "+lpurple(line)+" \t"+red('FAILED')+"\n")
384	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n")
385	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"red\">FAILED</font></p>")
386	print red("\n **************************************************")
387	print red(" "+line+" is NOT valid. Please debugging.. ")
388	print red("\n **************************************************")
389	g.close()
390
391	# generate job submission script
392	hostname=os.getenv('machine_name')
393
394	if hostname.startswith("jeanzay"):
395	write_jz_job(configName, NumberClients+NumberServers)
396
397	elif hostname.startswith("irene"):
398	write_irene_job(configName, NumberClients+NumberServers)
399
400	else :
401	print("machine unknown")
402	print >> sys.stderr, 1
403	sys.exit()
404
405
406	f.close()
407	h.write("\n")
408	l.write("<br>")
409	h.close()
410	l.close()
411	preport.close()
412
413	if not os.path.exists('current_run'):
414	OSinfo('mkdir current_run')
415
416	if not os.path.exists('current_run/'+configName):
417	OSinfo('mkdir current_run/'+configName)
418
419	OSinfo('mv -f param.def current_run/'+configName)
420	OSinfo('mv -f all_param.def current_run/'+configName)
421	OSinfo('mv -f user_param.py current_run/'+configName)
422	OSinfo('mv -f iodef.xml current_run/'+configName)
423	OSinfo('rm -f current_run/'+configName+'/context_atm.xml')
424	OSinfo('cp -n context_atm.xml current_run/'+configName)
425	OSinfo('cp -f context_grid_dynamico.xml current_run/'+configName)
426	OSinfo('cp -f dynamico_grid.nc current_run/'+configName)
427	OSinfo('mv -f xios_*.out current_run/'+configName)
428	OSinfo('mv -f xios_*.err current_run/'+configName)
429
430
431
432
433	if __name__== "__main__":
434	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: