[1888] | 1 | import glob |
---|
| 2 | import sys |
---|
| 3 | import subprocess |
---|
| 4 | import os |
---|
| 5 | import json |
---|
| 6 | import itertools |
---|
| 7 | import copy |
---|
| 8 | |
---|
| 9 | param_list = ["ATMdomain", "UsingServer2", "NumberClients", "NumberServers", "RatioServer2", "NumberPoolsServer2", "Duration"] |
---|
| 10 | param_short_list = ["ATMdom", "Srv2", "NbClnt", "NbSrv", "RatioSrv2", "NbPlSrv2", "Duration"] |
---|
| 11 | |
---|
| 12 | mode=os.getenv("mode") |
---|
| 13 | arch=os.getenv("arch") |
---|
| 14 | |
---|
| 15 | def OSinfo(runthis): |
---|
| 16 | red = lambda text: '\033[0;31m' + text + '\033[0m' |
---|
| 17 | osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) |
---|
| 18 | theInfo = osstdout.communicate()[0].strip() |
---|
| 19 | if osstdout.returncode!=0: |
---|
| 20 | print(red(runthis+" FAILED")) |
---|
| 21 | print >> sys.stderr, osstdout.returncode |
---|
| 22 | sys.exit() |
---|
| 23 | |
---|
| 24 | def product_dict(**kwargs): |
---|
| 25 | keys = kwargs.keys() |
---|
| 26 | vals = kwargs.values() |
---|
| 27 | for instance in itertools.product(*vals): |
---|
| 28 | yield dict(zip(keys, instance)) |
---|
| 29 | |
---|
| 30 | def get_default_param(): |
---|
| 31 | f=open("default_param.json", 'r') |
---|
| 32 | default_param = json.load(f) |
---|
| 33 | f.close() |
---|
| 34 | return default_param[0] |
---|
| 35 | |
---|
| 36 | def generate_job_irene(fn, n): |
---|
| 37 | with open(fn, "w") as fh: |
---|
| 38 | fh.write("#!/bin/bash\n") |
---|
| 39 | fh.write("#MSUB -r XIOS\n") |
---|
| 40 | fh.write("#MSUB -eo\n") |
---|
| 41 | fh.write("#MSUB -o client_output.out\n") |
---|
| 42 | fh.write("#MSUB -e client_error.out\n") |
---|
| 43 | fh.write("#MSUB -c 1\n") |
---|
| 44 | fh.write("#MSUB -n 24\n") |
---|
| 45 | fh.write("#MSUB -X\n") |
---|
| 46 | fh.write("#MSUB -x\n") |
---|
| 47 | fh.write("#MSUB -T 1800\n") |
---|
| 48 | fh.write("#MSUB -q skylake\n") |
---|
| 49 | fh.write("#MSUB -A gen0826\n") |
---|
| 50 | fh.write("#MSUB -Q test\n") |
---|
| 51 | fh.write("#MSUB -m work,scratch\n") |
---|
| 52 | fh.write("source ../../../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
| 53 | fh.write("ccc_mprun -n "+str(n)+" generic_testcase.exe\n") |
---|
| 54 | |
---|
| 55 | |
---|
| 56 | def update_full_job(location, n): |
---|
| 57 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 58 | fh.write("\ncd ${location}/"+location+"; ccc_mprun -E \'--exclusive\' -n "+str(n)+" generic_testcase.exe > output.out 2> error.out &\n") |
---|
| 59 | fh.write("PIDS+=($!)\n") |
---|
| 60 | fh.write("CONFIGS+=("+location+")\n") |
---|
| 61 | |
---|
| 62 | |
---|
| 63 | def main(): |
---|
| 64 | with open("full_job_"+arch+"_"+mode+".sh", "w") as fh: |
---|
| 65 | fh.write("#!/bin/bash\n") |
---|
| 66 | fh.write("#MSUB -r XIOS\n") |
---|
| 67 | fh.write("#MSUB -eo\n") |
---|
| 68 | fh.write("#MSUB -o client_output.out\n") |
---|
| 69 | fh.write("#MSUB -e client_error.err\n") |
---|
| 70 | fh.write("#MSUB -c 1\n") |
---|
| 71 | fh.write("#MSUB -n 24\n") |
---|
| 72 | fh.write("#MSUB -X\n") |
---|
| 73 | fh.write("#MSUB -x\n") |
---|
| 74 | fh.write("#MSUB -T 1800\n") |
---|
| 75 | fh.write("#MSUB -q skylake\n") |
---|
| 76 | fh.write("#MSUB -A gen0826\n") |
---|
| 77 | fh.write("#MSUB -Q test\n") |
---|
| 78 | fh.write("#MSUB -m work,scratch\n") |
---|
| 79 | fh.write("export location=/ccc/cont003/home/gencmip6/wangyush/XIOS/trunk/xios_test_suite/RUN_TEST_SUITE\n") |
---|
| 80 | fh.write("export log_location=/ccc/cont003/home/gencmip6/wangyush/XIOS/trunk/xios_test_suite/RUN_TEST_SUITE\n") |
---|
| 81 | fh.write("source ../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
| 82 | fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n") |
---|
| 83 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 84 | |
---|
| 85 | test_folder_list = glob.glob('test_*') |
---|
| 86 | all_config=dict() |
---|
| 87 | |
---|
| 88 | default_param = get_default_param() |
---|
| 89 | # print(default_param) |
---|
| 90 | |
---|
| 91 | for test_folder in test_folder_list: |
---|
| 92 | config_list=[] |
---|
| 93 | config_name=[] |
---|
| 94 | with open(test_folder+"/user_param.json", "r") as f: |
---|
| 95 | config_dict = json.load(f) |
---|
| 96 | |
---|
| 97 | for i in range(len(config_dict)): |
---|
| 98 | config_list.extend(list(product_dict(**config_dict[i]))) |
---|
| 99 | #print(config_list) |
---|
| 100 | for i in range(len(config_list)): |
---|
| 101 | # print(config_list[i]) |
---|
| 102 | keylist = list(config_list[i].keys()) |
---|
| 103 | # print(keylist) |
---|
| 104 | full_config = copy.deepcopy(default_param) |
---|
| 105 | for j in range(len(keylist)): |
---|
| 106 | full_config[ keylist[j] ] = config_list[i][keylist[j]] |
---|
| 107 | |
---|
| 108 | #print(full_config) |
---|
| 109 | mystr = str(full_config) |
---|
| 110 | mystr = mystr.replace("{", "") |
---|
| 111 | mystr = mystr.replace("}", "") |
---|
| 112 | mystr = mystr.replace("[", "") |
---|
| 113 | mystr = mystr.replace("]", "") |
---|
| 114 | mystr = mystr.replace(",", "") |
---|
| 115 | mystr = mystr.replace(":", "") |
---|
| 116 | mystr = mystr.replace("'", "") |
---|
| 117 | mystr = mystr.replace("b'", "") |
---|
| 118 | mystr = mystr.replace(" ", "_") |
---|
| 119 | for j in range(len(param_list)): |
---|
| 120 | mystr = mystr.replace(param_list[j], param_short_list[j]) |
---|
| 121 | if not mystr in config_name: |
---|
| 122 | config_name.append(mystr) |
---|
| 123 | #print(mystr) |
---|
| 124 | OSinfo("mkdir -p "+test_folder+"/CONFIG_"+mystr) |
---|
| 125 | OSinfo("cp build_"+arch+"_"+mode+"/bin/generic_testcase.exe "+test_folder+"/CONFIG_"+mystr) |
---|
| 126 | OSinfo("cp iodef.xml "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
| 127 | |
---|
| 128 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp", "r") as f: |
---|
| 129 | lines = f.readlines() |
---|
| 130 | for i in range(len(lines)): |
---|
| 131 | if "XIOS::" in lines[i]: |
---|
| 132 | config_keys = list(full_config.keys()) |
---|
| 133 | for idx in range(len(config_keys)): |
---|
| 134 | lines[i] = lines[i].replace("XIOS::"+config_keys[idx], str(full_config[config_keys[idx]])) |
---|
| 135 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml", "w") as g: |
---|
| 136 | for line in lines: |
---|
| 137 | g.write(line) |
---|
| 138 | |
---|
| 139 | OSinfo("rm -f "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
| 140 | OSinfo("cp "+test_folder+"/context_atm.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 141 | OSinfo("cp context_grid_dynamico.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 142 | OSinfo("cp dynamico_grid.nc "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 143 | OSinfo("cp "+test_folder+"/checkfile.def "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 144 | with open(test_folder+"/CONFIG_"+mystr+"/param.def", "w") as fh: |
---|
| 145 | fh.write("¶ms_run\n") |
---|
| 146 | fh.write("duration=\'"+full_config["Duration"]+"\'\n") |
---|
| 147 | fh.write("nb_proc_atm="+str(full_config["NumberClients"])+"\n") |
---|
| 148 | fh.write("/\n") |
---|
| 149 | with open(test_folder+"/CONFIG_"+mystr+"/all_param.def", "w") as fh: |
---|
| 150 | fh.write("¶ms_run\n") |
---|
| 151 | for param in param_list: |
---|
| 152 | fh.write(param+"="+str(full_config[param])+"\n") |
---|
| 153 | fh.write("/\n") |
---|
| 154 | |
---|
| 155 | generate_job_irene(test_folder+"/CONFIG_"+mystr+"/job.sh", full_config['NumberClients']+full_config['NumberServers']) |
---|
| 156 | update_full_job(test_folder+"/CONFIG_"+mystr, full_config['NumberClients']+full_config['NumberServers']) |
---|
| 157 | |
---|
| 158 | #print(config_name) |
---|
| 159 | all_config[test_folder] = config_name |
---|
| 160 | |
---|
| 161 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 162 | fh.write("\nfor pid in ${PIDS[@]}; do\n") |
---|
| 163 | fh.write("wait ${pid}\n") |
---|
| 164 | fh.write("STATUS+=($?)\ndone\n") |
---|
| 165 | fh.write("\ni=0\n") |
---|
| 166 | fh.write("#for st in ${STATUS[@]}; do\n") |
---|
| 167 | fh.write("#if [[ ${st} -ne 0 ]]; then\n") |
---|
| 168 | fh.write("#echo \"${CONFIGS[${i}]} -1\" >> ${location}/plain_report.txt\n") |
---|
| 169 | fh.write("#else\n") |
---|
| 170 | fh.write("#echo \"${CONFIGS[${i}]} 1\" >> ${location}/plain_report.txt\n") |
---|
| 171 | fh.write("#fi\n") |
---|
| 172 | fh.write("#((i+=1))\n") |
---|
| 173 | fh.write("#done\n\n") |
---|
| 174 | fh.write("#wait\n") |
---|
| 175 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 176 | |
---|
| 177 | #print(all_config) |
---|
| 178 | |
---|
| 179 | if __name__== "__main__": |
---|
| 180 | main() |
---|