[1888] | 1 | import glob |
---|
| 2 | import sys |
---|
| 3 | import subprocess |
---|
| 4 | import os |
---|
| 5 | import json |
---|
| 6 | import itertools |
---|
| 7 | import copy |
---|
| 8 | |
---|
[1924] | 9 | param_list = ["DomMask", "AxisMask", "Domain", "UseServer2", "NumberClients", "NumberServers", "PctServer2", "Duration", "OneSided", "NonDistTran"] |
---|
| 10 | param_short_list = ["DomMask", "AxisMask", "Dom", "Srv2", "NbClnt", "NbSrv", "PctSrv2", "Duration", "OneSided", "NDtTr"] |
---|
[1888] | 11 | |
---|
| 12 | mode=os.getenv("mode") |
---|
| 13 | arch=os.getenv("arch") |
---|
[1893] | 14 | machine=os.getenv("xios_machine_name") |
---|
[1916] | 15 | svnR=os.getenv("svnR") |
---|
[1944] | 16 | user_acct=os.getenv("user_account") |
---|
[1888] | 17 | |
---|
[1906] | 18 | nb_proc_irene=40 # this must be >= NumberClients+NumberServers for all configs for all test folders |
---|
| 19 | nb_proc_jz=40 # to run completly in parallel, this must be set to 40 for the moment |
---|
| 20 | # otherwise, jobs should be regrouped by test folder. jobs in folder are parallel, folders are sequential |
---|
| 21 | # in this case,nb_proc_jz >= sum(NumberClients+NumberServers) for each test folder |
---|
[1893] | 22 | |
---|
[1888] | 23 | def OSinfo(runthis): |
---|
| 24 | red = lambda text: '\033[0;31m' + text + '\033[0m' |
---|
| 25 | osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) |
---|
| 26 | theInfo = osstdout.communicate()[0].strip() |
---|
| 27 | if osstdout.returncode!=0: |
---|
| 28 | print(red(runthis+" FAILED")) |
---|
[1891] | 29 | print(theInfo) |
---|
[1888] | 30 | sys.exit() |
---|
| 31 | |
---|
| 32 | def product_dict(**kwargs): |
---|
| 33 | keys = kwargs.keys() |
---|
| 34 | vals = kwargs.values() |
---|
| 35 | for instance in itertools.product(*vals): |
---|
| 36 | yield dict(zip(keys, instance)) |
---|
| 37 | |
---|
| 38 | def get_default_param(): |
---|
| 39 | f=open("default_param.json", 'r') |
---|
| 40 | default_param = json.load(f) |
---|
| 41 | f.close() |
---|
[1909] | 42 | l = list(default_param[0].items()) |
---|
| 43 | l.sort() |
---|
| 44 | def_param=dict(l) |
---|
| 45 | return def_param |
---|
[1888] | 46 | |
---|
[1893] | 47 | def generate_job(fn, n): |
---|
| 48 | if machine=="irene": |
---|
| 49 | with open(fn, "w") as fh: |
---|
| 50 | fh.write("#!/bin/bash\n") |
---|
[1899] | 51 | fh.write("#====================================================\n") |
---|
| 52 | fh.write("# Generated by step1.py\n") |
---|
| 53 | fh.write("# Called by my_run.sh -> run_test -> run_test_irene\n") |
---|
[1906] | 54 | fh.write("# For Irene\n") |
---|
[1899] | 55 | fh.write("#====================================================\n") |
---|
[1893] | 56 | fh.write("#MSUB -r XIOS\n") |
---|
| 57 | fh.write("#MSUB -eo\n") |
---|
| 58 | fh.write("#MSUB -o client_output.out\n") |
---|
| 59 | fh.write("#MSUB -e client_error.out\n") |
---|
| 60 | fh.write("#MSUB -c 1\n") |
---|
| 61 | fh.write("#MSUB -n "+str(n)+"\n") |
---|
| 62 | fh.write("#MSUB -X\n") |
---|
| 63 | fh.write("#MSUB -x\n") |
---|
| 64 | fh.write("#MSUB -T 1800\n") |
---|
| 65 | fh.write("#MSUB -q skylake\n") |
---|
[1944] | 66 | fh.write("#MSUB -A "+user_acct+"\n") |
---|
[1893] | 67 | fh.write("#MSUB -Q test\n") |
---|
| 68 | fh.write("#MSUB -m work,scratch\n") |
---|
[1909] | 69 | fh.write("cp ../../build_"+arch+"_"+mode+"/bin/generic_testcase.exe ./\n") |
---|
| 70 | fh.write("source ../../build_"+arch+"_"+mode+"/arch.env\n") |
---|
[1893] | 71 | fh.write("ccc_mprun -n "+str(n)+" generic_testcase.exe\n") |
---|
[1888] | 72 | |
---|
[1893] | 73 | if machine=="jeanzay": |
---|
| 74 | with open(fn, "w") as fh: |
---|
| 75 | fh.write("#!/bin/bash\n") |
---|
[1899] | 76 | fh.write("#====================================================\n") |
---|
| 77 | fh.write("# Generated by step1.py\n") |
---|
| 78 | fh.write("# Called by my_run.sh -> run_test -> run_test_jeanzay\n") |
---|
[1906] | 79 | fh.write("# For Jean-Zay\n") |
---|
[1899] | 80 | fh.write("#====================================================\n") |
---|
[1893] | 81 | fh.write("#SBATCH --ntasks="+str(n)+"\n") |
---|
| 82 | fh.write("#SBATCH --hint=nomultithread\n") |
---|
| 83 | fh.write("#SBATCH -o output.out\n") |
---|
| 84 | fh.write("#SBATCH -e error.out\n") |
---|
| 85 | fh.write("#SBATCH -t 00:10:00\n") |
---|
[1944] | 86 | fh.write("#SBATCH --account="+user_acct+"\n") |
---|
[2431] | 87 | fh.write("#SBATCH --qos=qos_cpu-dev\n") |
---|
[2219] | 88 | fh.write("#SBATCH --exclusive\n") |
---|
[1893] | 89 | fh.write("ulimit -c 0\n") |
---|
| 90 | fh.write("cd ${SLURM_SUBMIT_DIR}\n") |
---|
[1909] | 91 | fh.write("cp ../../build_"+arch+"_"+mode+"/bin/generic_testcase.exe ./\n") |
---|
| 92 | fh.write("source ../../build_"+arch+"_"+mode+"/arch.env\n") |
---|
[1906] | 93 | fh.write("source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt\n") |
---|
[1893] | 94 | fh.write("srun generic_testcase.exe") |
---|
[1888] | 95 | |
---|
[1893] | 96 | |
---|
[1888] | 97 | def update_full_job(location, n): |
---|
[1893] | 98 | global my_counter |
---|
| 99 | if machine=="irene": |
---|
| 100 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
[1909] | 101 | fh.write("\ncd ${location}/"+location+"; ccc_mprun -E \'--exclusive\' -n "+str(n)+" generic_testcase.exe > output_"+arch+"_"+mode+".out 2> error_"+arch+"_"+mode+".out &\n") |
---|
[1888] | 102 | fh.write("PIDS+=($!)\n") |
---|
| 103 | fh.write("CONFIGS+=("+location+")\n") |
---|
[1893] | 104 | if machine=="jeanzay": |
---|
| 105 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
[1909] | 106 | fh.write("\ncd ${location}/"+location+"; srun --exclusive -n "+str(n)+" generic_testcase.exe > output_"+arch+"_"+mode+".out 2> error_"+arch+"_"+mode+".out &\n") |
---|
[1888] | 107 | |
---|
[1893] | 108 | def main(): |
---|
[1888] | 109 | |
---|
[1893] | 110 | if machine=="irene": |
---|
| 111 | with open("full_job_"+arch+"_"+mode+".sh", "w") as fh: |
---|
| 112 | fh.write("#!/bin/bash\n") |
---|
[1899] | 113 | fh.write("#====================================================\n") |
---|
| 114 | fh.write("# Generated by step1.py\n") |
---|
| 115 | fh.write("# Called by my_run.sh -> run_test -> run_test_irene\n") |
---|
| 116 | fh.write("#====================================================\n") |
---|
[1916] | 117 | fh.write("#MSUB -r XIOS_rev"+svnR+"\n") |
---|
[1893] | 118 | fh.write("#MSUB -eo\n") |
---|
[1909] | 119 | fh.write("#MSUB -o output_"+arch+"_"+mode+".out\n") |
---|
| 120 | fh.write("#MSUB -e error_"+arch+"_"+mode+".err\n") |
---|
[1893] | 121 | fh.write("#MSUB -c 1\n") |
---|
| 122 | fh.write("#MSUB -n "+str(nb_proc_irene)+"\n") |
---|
| 123 | fh.write("#MSUB -X\n") |
---|
| 124 | fh.write("#MSUB -x\n") |
---|
| 125 | fh.write("#MSUB -T 1800\n") |
---|
| 126 | fh.write("#MSUB -q skylake\n") |
---|
[1944] | 127 | fh.write("#MSUB -A "+user_acct+"\n") |
---|
[1893] | 128 | fh.write("#MSUB -Q test\n") |
---|
| 129 | fh.write("#MSUB -m work,scratch\n") |
---|
| 130 | fh.write("export location="+os.getcwd()+"\n") |
---|
| 131 | fh.write("export log_location="+os.getcwd()+"\n") |
---|
[1909] | 132 | fh.write("source build_"+arch+"_"+mode+"/arch.env\n") |
---|
[1893] | 133 | fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n") |
---|
| 134 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 135 | |
---|
| 136 | if machine=="jeanzay": |
---|
| 137 | with open("full_job_"+arch+"_"+mode+".sh", "w") as fh: |
---|
| 138 | fh.write("#!/bin/bash\n") |
---|
[1899] | 139 | fh.write("#====================================================\n") |
---|
| 140 | fh.write("# Generated by step1.py\n") |
---|
| 141 | fh.write("# Called by my_run.sh -> run_test -> run_test_jeanzay\n") |
---|
| 142 | fh.write("#====================================================\n") |
---|
[1893] | 143 | fh.write("#SBATCH --ntasks="+str(nb_proc_jz)+"\n") |
---|
[1909] | 144 | fh.write("#SBATCH -o output_"+arch+"_"+mode+".out\n") |
---|
| 145 | fh.write("#SBATCH -e error_"+arch+"_"+mode+".out\n") |
---|
[1893] | 146 | fh.write("#SBATCH --hint=nomultithread\n") |
---|
| 147 | fh.write("#SBATCH -t 00:10:00\n") |
---|
[1944] | 148 | fh.write("#SBATCH --account="+user_acct+"\n") |
---|
[1916] | 149 | fh.write("#SBATCH --job-name=XIOS_rev"+svnR+"\n") |
---|
[2431] | 150 | fh.write("#SBATCH --qos=qos_cpu-dev\n") |
---|
[2219] | 151 | fh.write("#SBATCH --exclusive\n") |
---|
[1893] | 152 | fh.write("ulimit -c 0\n") |
---|
| 153 | fh.write("cd ${SLURM_SUBMIT_DIR}\n") |
---|
[1909] | 154 | fh.write("source build_"+arch+"_"+mode+"/arch.env\n") |
---|
[1893] | 155 | fh.write("source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt\n") |
---|
| 156 | fh.write("export location="+os.getcwd()+"\n") |
---|
| 157 | fh.write("export log_location="+os.getcwd()+"\n") |
---|
| 158 | fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n") |
---|
| 159 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 160 | |
---|
| 161 | |
---|
| 162 | |
---|
[1888] | 163 | test_folder_list = glob.glob('test_*') |
---|
[1906] | 164 | #all_config=dict() |
---|
[1888] | 165 | |
---|
| 166 | default_param = get_default_param() |
---|
[1907] | 167 | #print(default_param) |
---|
[1888] | 168 | |
---|
| 169 | for test_folder in test_folder_list: |
---|
| 170 | config_list=[] |
---|
| 171 | config_name=[] |
---|
| 172 | with open(test_folder+"/user_param.json", "r") as f: |
---|
| 173 | config_dict = json.load(f) |
---|
| 174 | |
---|
| 175 | for i in range(len(config_dict)): |
---|
| 176 | config_list.extend(list(product_dict(**config_dict[i]))) |
---|
| 177 | #print(config_list) |
---|
| 178 | for i in range(len(config_list)): |
---|
| 179 | # print(config_list[i]) |
---|
| 180 | keylist = list(config_list[i].keys()) |
---|
| 181 | # print(keylist) |
---|
| 182 | full_config = copy.deepcopy(default_param) |
---|
| 183 | for j in range(len(keylist)): |
---|
| 184 | full_config[ keylist[j] ] = config_list[i][keylist[j]] |
---|
| 185 | |
---|
| 186 | #print(full_config) |
---|
| 187 | mystr = str(full_config) |
---|
| 188 | mystr = mystr.replace("{", "") |
---|
| 189 | mystr = mystr.replace("}", "") |
---|
| 190 | mystr = mystr.replace("[", "") |
---|
| 191 | mystr = mystr.replace("]", "") |
---|
| 192 | mystr = mystr.replace(",", "") |
---|
| 193 | mystr = mystr.replace(":", "") |
---|
[1907] | 194 | mystr = mystr.replace("u'", "") |
---|
[1909] | 195 | mystr = mystr.replace("b'", "") |
---|
[1888] | 196 | mystr = mystr.replace("'", "") |
---|
| 197 | mystr = mystr.replace(" ", "_") |
---|
| 198 | for j in range(len(param_list)): |
---|
| 199 | mystr = mystr.replace(param_list[j], param_short_list[j]) |
---|
| 200 | if not mystr in config_name: |
---|
| 201 | config_name.append(mystr) |
---|
| 202 | #print(mystr) |
---|
[2218] | 203 | if ( os.path.isdir( test_folder+"/CONFIG_"+mystr ) ) : |
---|
| 204 | OSinfo("rm -rf "+test_folder+"/CONFIG_"+mystr) |
---|
[1888] | 205 | OSinfo("mkdir -p "+test_folder+"/CONFIG_"+mystr) |
---|
| 206 | OSinfo("cp build_"+arch+"_"+mode+"/bin/generic_testcase.exe "+test_folder+"/CONFIG_"+mystr) |
---|
| 207 | OSinfo("cp iodef.xml "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
[1915] | 208 | OSinfo("cp "+test_folder+"/context_atm.xml "+test_folder+"/CONFIG_"+mystr+"/context_atm.xml.tmp") |
---|
[1888] | 209 | |
---|
| 210 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp", "r") as f: |
---|
| 211 | lines = f.readlines() |
---|
| 212 | for i in range(len(lines)): |
---|
| 213 | if "XIOS::" in lines[i]: |
---|
| 214 | config_keys = list(full_config.keys()) |
---|
| 215 | for idx in range(len(config_keys)): |
---|
| 216 | lines[i] = lines[i].replace("XIOS::"+config_keys[idx], str(full_config[config_keys[idx]])) |
---|
| 217 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml", "w") as g: |
---|
| 218 | for line in lines: |
---|
| 219 | g.write(line) |
---|
| 220 | |
---|
[1915] | 221 | with open(test_folder+"/CONFIG_"+mystr+"/context_atm.xml.tmp", "r") as f: |
---|
| 222 | lines = f.readlines() |
---|
| 223 | for i in range(len(lines)): |
---|
| 224 | if "XIOS::" in lines[i]: |
---|
| 225 | config_keys = list(full_config.keys()) |
---|
| 226 | for idx in range(len(config_keys)): |
---|
| 227 | lines[i] = lines[i].replace("XIOS::"+config_keys[idx], str(full_config[config_keys[idx]])) |
---|
| 228 | with open(test_folder+"/CONFIG_"+mystr+"/context_atm.xml", "w") as g: |
---|
| 229 | for line in lines: |
---|
| 230 | g.write(line) |
---|
| 231 | |
---|
| 232 | |
---|
[1888] | 233 | OSinfo("rm -f "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
[1915] | 234 | OSinfo("rm -f "+test_folder+"/CONFIG_"+mystr+"/context_atm.xml.tmp") |
---|
[1888] | 235 | OSinfo("cp context_grid_dynamico.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 236 | OSinfo("cp dynamico_grid.nc "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 237 | OSinfo("cp "+test_folder+"/checkfile.def "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 238 | with open(test_folder+"/CONFIG_"+mystr+"/param.def", "w") as fh: |
---|
| 239 | fh.write("¶ms_run\n") |
---|
| 240 | fh.write("duration=\'"+full_config["Duration"]+"\'\n") |
---|
| 241 | fh.write("nb_proc_atm="+str(full_config["NumberClients"])+"\n") |
---|
| 242 | fh.write("/\n") |
---|
| 243 | with open(test_folder+"/CONFIG_"+mystr+"/all_param.def", "w") as fh: |
---|
| 244 | fh.write("¶ms_run\n") |
---|
| 245 | for param in param_list: |
---|
| 246 | fh.write(param+"="+str(full_config[param])+"\n") |
---|
| 247 | fh.write("/\n") |
---|
| 248 | |
---|
[1906] | 249 | generate_job(test_folder+"/CONFIG_"+mystr+"/job_"+arch+"_"+mode+".sh", full_config['NumberClients']+full_config['NumberServers']) |
---|
[1888] | 250 | update_full_job(test_folder+"/CONFIG_"+mystr, full_config['NumberClients']+full_config['NumberServers']) |
---|
| 251 | |
---|
[1906] | 252 | #if machine=="jeanzay": |
---|
| 253 | # with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 254 | # fh.write("wait\nwait\n") |
---|
| 255 | # fh.write("echo \"tests in "+test_folder+" finished\"\n") |
---|
| 256 | #all_config[test_folder] = config_name |
---|
[1893] | 257 | |
---|
[1888] | 258 | |
---|
[1893] | 259 | if machine=="irene": |
---|
| 260 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
[1906] | 261 | #fh.write("\nfor pid in ${PIDS[@]}; do\n") |
---|
| 262 | #fh.write("wait ${pid}\n") |
---|
| 263 | #fh.write("STATUS+=($?)\ndone\n") |
---|
| 264 | #fh.write("\ni=0\n") |
---|
| 265 | #fh.write("#for st in ${STATUS[@]}; do\n") |
---|
| 266 | #fh.write("#if [[ ${st} -ne 0 ]]; then\n") |
---|
| 267 | #fh.write("#echo \"${CONFIGS[${i}]} -1\" >> ${location}/plain_report.txt\n") |
---|
| 268 | #fh.write("#else\n") |
---|
| 269 | #fh.write("#echo \"${CONFIGS[${i}]} 1\" >> ${location}/plain_report.txt\n") |
---|
| 270 | #fh.write("#fi\n") |
---|
| 271 | #fh.write("#((i+=1))\n") |
---|
| 272 | #fh.write("#done\n\n") |
---|
| 273 | fh.write("wait\nwait\n") |
---|
[1893] | 274 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 275 | if machine=="jeanzay": |
---|
| 276 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
[1906] | 277 | fh.write("wait\nwait\n") |
---|
[1893] | 278 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
[1888] | 279 | |
---|
| 280 | |
---|
| 281 | if __name__== "__main__": |
---|
| 282 | main() |
---|