1 | import glob |
---|
2 | import sys |
---|
3 | import subprocess |
---|
4 | import os |
---|
5 | import json |
---|
6 | import itertools |
---|
7 | import copy |
---|
8 | |
---|
9 | param_list = ["ATMdomain", "UsingServer2", "NumberClients", "NumberServers", "RatioServer2", "NumberPoolsServer2", "Duration"] |
---|
10 | param_short_list = ["ATMdom", "Srv2", "NbClnt", "NbSrv", "RatioSrv2", "NbPlSrv2", "Duration"] |
---|
11 | |
---|
12 | mode=os.getenv("mode") |
---|
13 | arch=os.getenv("arch") |
---|
14 | |
---|
15 | def OSinfo(runthis): |
---|
16 | red = lambda text: '\033[0;31m' + text + '\033[0m' |
---|
17 | osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) |
---|
18 | theInfo = osstdout.communicate()[0].strip() |
---|
19 | if osstdout.returncode!=0: |
---|
20 | print(red(runthis+" FAILED")) |
---|
21 | print >> sys.stderr, osstdout.returncode |
---|
22 | sys.exit() |
---|
23 | |
---|
24 | def product_dict(**kwargs): |
---|
25 | keys = kwargs.keys() |
---|
26 | vals = kwargs.values() |
---|
27 | for instance in itertools.product(*vals): |
---|
28 | yield dict(zip(keys, instance)) |
---|
29 | |
---|
30 | def get_default_param(): |
---|
31 | f=open("default_param.json", 'r') |
---|
32 | default_param = json.load(f) |
---|
33 | f.close() |
---|
34 | return default_param[0] |
---|
35 | |
---|
36 | def generate_job_irene(fn, n): |
---|
37 | with open(fn, "w") as fh: |
---|
38 | fh.write("#!/bin/bash\n") |
---|
39 | fh.write("#MSUB -r XIOS\n") |
---|
40 | fh.write("#MSUB -eo\n") |
---|
41 | fh.write("#MSUB -o client_output.out\n") |
---|
42 | fh.write("#MSUB -e client_error.out\n") |
---|
43 | fh.write("#MSUB -c 1\n") |
---|
44 | fh.write("#MSUB -n 24\n") |
---|
45 | fh.write("#MSUB -X\n") |
---|
46 | fh.write("#MSUB -x\n") |
---|
47 | fh.write("#MSUB -T 1800\n") |
---|
48 | fh.write("#MSUB -q skylake\n") |
---|
49 | fh.write("#MSUB -A gen0826\n") |
---|
50 | fh.write("#MSUB -Q test\n") |
---|
51 | fh.write("#MSUB -m work,scratch\n") |
---|
52 | fh.write("source ../../../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
53 | fh.write("ccc_mprun -n "+str(n)+" generic_testcase.exe\n") |
---|
54 | |
---|
55 | |
---|
56 | def update_full_job(location, n): |
---|
57 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
58 | fh.write("\ncd ${location}/"+location+"; ccc_mprun -E \'--exclusive\' -n "+str(n)+" generic_testcase.exe > output.out 2> error.out &\n") |
---|
59 | fh.write("PIDS+=($!)\n") |
---|
60 | fh.write("CONFIGS+=("+location+")\n") |
---|
61 | |
---|
62 | |
---|
63 | def main(): |
---|
64 | with open("full_job_"+arch+"_"+mode+".sh", "w") as fh: |
---|
65 | fh.write("#!/bin/bash\n") |
---|
66 | fh.write("#MSUB -r XIOS\n") |
---|
67 | fh.write("#MSUB -eo\n") |
---|
68 | fh.write("#MSUB -o client_output.out\n") |
---|
69 | fh.write("#MSUB -e client_error.err\n") |
---|
70 | fh.write("#MSUB -c 1\n") |
---|
71 | fh.write("#MSUB -n 24\n") |
---|
72 | fh.write("#MSUB -X\n") |
---|
73 | fh.write("#MSUB -x\n") |
---|
74 | fh.write("#MSUB -T 1800\n") |
---|
75 | fh.write("#MSUB -q skylake\n") |
---|
76 | fh.write("#MSUB -A gen0826\n") |
---|
77 | fh.write("#MSUB -Q test\n") |
---|
78 | fh.write("#MSUB -m work,scratch\n") |
---|
79 | fh.write("export location=/ccc/cont003/home/gencmip6/wangyush/XIOS/trunk/xios_test_suite/RUN_TEST_SUITE\n") |
---|
80 | fh.write("export log_location=/ccc/cont003/home/gencmip6/wangyush/XIOS/trunk/xios_test_suite/RUN_TEST_SUITE\n") |
---|
81 | fh.write("source ../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
82 | fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n") |
---|
83 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
84 | |
---|
85 | test_folder_list = glob.glob('test_*') |
---|
86 | all_config=dict() |
---|
87 | |
---|
88 | default_param = get_default_param() |
---|
89 | # print(default_param) |
---|
90 | |
---|
91 | for test_folder in test_folder_list: |
---|
92 | config_list=[] |
---|
93 | config_name=[] |
---|
94 | with open(test_folder+"/user_param.json", "r") as f: |
---|
95 | config_dict = json.load(f) |
---|
96 | |
---|
97 | for i in range(len(config_dict)): |
---|
98 | config_list.extend(list(product_dict(**config_dict[i]))) |
---|
99 | #print(config_list) |
---|
100 | for i in range(len(config_list)): |
---|
101 | # print(config_list[i]) |
---|
102 | keylist = list(config_list[i].keys()) |
---|
103 | # print(keylist) |
---|
104 | full_config = copy.deepcopy(default_param) |
---|
105 | for j in range(len(keylist)): |
---|
106 | full_config[ keylist[j] ] = config_list[i][keylist[j]] |
---|
107 | |
---|
108 | #print(full_config) |
---|
109 | mystr = str(full_config) |
---|
110 | mystr = mystr.replace("{", "") |
---|
111 | mystr = mystr.replace("}", "") |
---|
112 | mystr = mystr.replace("[", "") |
---|
113 | mystr = mystr.replace("]", "") |
---|
114 | mystr = mystr.replace(",", "") |
---|
115 | mystr = mystr.replace(":", "") |
---|
116 | mystr = mystr.replace("'", "") |
---|
117 | mystr = mystr.replace("b'", "") |
---|
118 | mystr = mystr.replace(" ", "_") |
---|
119 | for j in range(len(param_list)): |
---|
120 | mystr = mystr.replace(param_list[j], param_short_list[j]) |
---|
121 | if not mystr in config_name: |
---|
122 | config_name.append(mystr) |
---|
123 | #print(mystr) |
---|
124 | OSinfo("mkdir -p "+test_folder+"/CONFIG_"+mystr) |
---|
125 | OSinfo("cp build_"+arch+"_"+mode+"/bin/generic_testcase.exe "+test_folder+"/CONFIG_"+mystr) |
---|
126 | OSinfo("cp iodef.xml "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
127 | |
---|
128 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp", "r") as f: |
---|
129 | lines = f.readlines() |
---|
130 | for i in range(len(lines)): |
---|
131 | if "XIOS::" in lines[i]: |
---|
132 | config_keys = list(full_config.keys()) |
---|
133 | for idx in range(len(config_keys)): |
---|
134 | lines[i] = lines[i].replace("XIOS::"+config_keys[idx], str(full_config[config_keys[idx]])) |
---|
135 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml", "w") as g: |
---|
136 | for line in lines: |
---|
137 | g.write(line) |
---|
138 | |
---|
139 | OSinfo("rm -f "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
140 | OSinfo("cp "+test_folder+"/context_atm.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
141 | OSinfo("cp context_grid_dynamico.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
142 | OSinfo("cp dynamico_grid.nc "+test_folder+"/CONFIG_"+mystr+"/") |
---|
143 | OSinfo("cp "+test_folder+"/checkfile.def "+test_folder+"/CONFIG_"+mystr+"/") |
---|
144 | with open(test_folder+"/CONFIG_"+mystr+"/param.def", "w") as fh: |
---|
145 | fh.write("¶ms_run\n") |
---|
146 | fh.write("duration=\'"+full_config["Duration"]+"\'\n") |
---|
147 | fh.write("nb_proc_atm="+str(full_config["NumberClients"])+"\n") |
---|
148 | fh.write("/\n") |
---|
149 | with open(test_folder+"/CONFIG_"+mystr+"/all_param.def", "w") as fh: |
---|
150 | fh.write("¶ms_run\n") |
---|
151 | for param in param_list: |
---|
152 | fh.write(param+"="+str(full_config[param])+"\n") |
---|
153 | fh.write("/\n") |
---|
154 | |
---|
155 | generate_job_irene(test_folder+"/CONFIG_"+mystr+"/job.sh", full_config['NumberClients']+full_config['NumberServers']) |
---|
156 | update_full_job(test_folder+"/CONFIG_"+mystr, full_config['NumberClients']+full_config['NumberServers']) |
---|
157 | |
---|
158 | #print(config_name) |
---|
159 | all_config[test_folder] = config_name |
---|
160 | |
---|
161 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
162 | fh.write("\nfor pid in ${PIDS[@]}; do\n") |
---|
163 | fh.write("wait ${pid}\n") |
---|
164 | fh.write("STATUS+=($?)\ndone\n") |
---|
165 | fh.write("\ni=0\n") |
---|
166 | fh.write("#for st in ${STATUS[@]}; do\n") |
---|
167 | fh.write("#if [[ ${st} -ne 0 ]]; then\n") |
---|
168 | fh.write("#echo \"${CONFIGS[${i}]} -1\" >> ${location}/plain_report.txt\n") |
---|
169 | fh.write("#else\n") |
---|
170 | fh.write("#echo \"${CONFIGS[${i}]} 1\" >> ${location}/plain_report.txt\n") |
---|
171 | fh.write("#fi\n") |
---|
172 | fh.write("#((i+=1))\n") |
---|
173 | fh.write("#done\n\n") |
---|
174 | fh.write("#wait\n") |
---|
175 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
176 | |
---|
177 | #print(all_config) |
---|
178 | |
---|
179 | if __name__== "__main__": |
---|
180 | main() |
---|