Changeset 1909

Timestamp:

06/19/20 18:57:59 (5 years ago)

Author:

yushan

Message:

trunk : add reference generator and archievs

Location:

XIOS/trunk/xios_test_suite

Files:

• ENV/irene.env (modified) (1 diff)
• TEST_SUITE/default_param.json (modified) (1 diff)
• TEST_SUITE/iodef.xml (modified) (1 diff)
• TEST_SUITE/run_test (modified) (1 diff)
• TEST_SUITE/step1.py (modified) (10 diffs)
• TEST_SUITE/step2.py (modified) (4 diffs)
• generate_ref.sh (added)

XIOS/trunk/xios_test_suite/ENV/irene.env

@@ -10 +10 @@
 export xios_thredds_dir=/ccc/work/cont003/thredds/p86yann/XIOS_TEST_SUITE
 export xios_reference_dir=/ccc/work/cont003/gen0826/gen0826/XIOS_TEST_SUITE_REPOSITORY
-export ref_location=/ccc/cont003/home/gencmip6/wangyush/cron_xios
+export ref_location=/ccc/cont003/home/gencmip6/wangyush/reference_dir
 export ref_file=reference.tar.gz
 

XIOS/trunk/xios_test_suite/TEST_SUITE/default_param.json

@@ -1 +1 @@
 [
-        {
-            "ATMdomain": "lmdz",
-            "UsingServer2": "true",
-            "RatioServer2": 50,
-            "NumberPoolsServer2": 1,
-            "Duration": "1d",
-            "NumberClients": 4,
-            "NumberServers": 4
-        }
+  {
+    "ATMdomain": "lmdz",
+    "UsingServer2": "true",
+    "RatioServer2": 50,
+    "NumberPoolsServer2": 1,
+    "Duration": "1d",
+    "NumberClients": 4,
+    "NumberServers": 4,
+    "InfoLevel": 100
+  }
 ]

XIOS/trunk/xios_test_suite/TEST_SUITE/iodef.xml

@@ -53 +53 @@
       <variable_group id="parameters" >
         <variable id="using_server" type="bool">true</variable>
-        <variable id="info_level" type="int">100</variable>
+        <variable id="info_level" type="int">XIOS::InfoLevel</variable>
         <variable id="print_file" type="bool">true</variable>
         <variable id="using_server2" type="bool">XIOS::UsingServer2</variable>

XIOS/trunk/xios_test_suite/TEST_SUITE/run_test

@@ -25 +25 @@
   for i in $(ls -d test_*/)
   do
-    mkdir -p ${rundir}/def_files/${svnR}/${i%%}
+    mkdir -p ${rundir}/def_files/${svnR}/${i%%} ; CHMOD ${rundir}/def_files/${svnR}/${i%%}
     cp ${i%%}/user_param.json ${rundir}/def_files/${svnR}/${i%%}
     for j in $(ls -d ${i%%/}/CONFIG_*)
     do
-      mkdir -p ${rundir}/def_files/${svnR}/${j%%}
+      mkdir -p ${rundir}/def_files/${svnR}/${j%%} ; CHMOD ${rundir}/def_files/${svnR}/${j%%}
       cp ${j%%}/all_param.def ${rundir}/def_files/${svnR}/${j%%}
     done

XIOS/trunk/xios_test_suite/TEST_SUITE/step1.py

@@ -7 +7 @@
 import copy
 
-param_list = ["ATMdomain", "UsingServer2", "NumberClients", "NumberServers", "RatioServer2", "NumberPoolsServer2", "Duration"]
-param_short_list = ["ATMdom", "Srv2", "NbClnt", "NbSrv", "RatioSrv2", "NbPlSrv2", "Duration"]
+param_list = ["ATMdomain", "UsingServer2", "NumberClients", "NumberServers", "RatioServer2", "NumberPoolsServer2", "Duration", "InfoLevel"]
+param_short_list = ["ATMdom", "Srv2", "NbClnt", "NbSrv", "RatioSrv2", "NbPlSrv2", "Duration", "Info"]
 
 mode=os.getenv("mode")
@@ -38 +38 @@
     default_param = json.load(f)
     f.close()
-    return default_param[0]
+    l = list(default_param[0].items())
+    l.sort()
+    def_param=dict(l)
+    return def_param
 
 def generate_job(fn, n):
@@ -62 +65 @@
             fh.write("#MSUB -Q test\n")
             fh.write("#MSUB -m work,scratch\n")
-            fh.write("source ../../../BUILD/build_"+arch+"_"+mode+"/arch.env\n")
+            fh.write("cp ../../build_"+arch+"_"+mode+"/bin/generic_testcase.exe ./\n")
+            fh.write("source ../../build_"+arch+"_"+mode+"/arch.env\n")
             fh.write("ccc_mprun -n "+str(n)+" generic_testcase.exe\n")
 
@@ -81 +85 @@
             fh.write("ulimit -c 0\n")
             fh.write("cd ${SLURM_SUBMIT_DIR}\n")
-            fh.write("source ../../../BUILD/build_"+arch+"_"+mode+"/arch.env\n")
+            fh.write("cp ../../build_"+arch+"_"+mode+"/bin/generic_testcase.exe ./\n")
+            fh.write("source ../../build_"+arch+"_"+mode+"/arch.env\n")
             fh.write("source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt\n")
             fh.write("srun generic_testcase.exe")
@@ -90 +95 @@
     if machine=="irene":
         with open("full_job_"+arch+"_"+mode+".sh", "a") as fh:
-            fh.write("\ncd ${location}/"+location+"; ccc_mprun -E \'--exclusive\' -n "+str(n)+" generic_testcase.exe > output.out 2> error.out &\n")
+            fh.write("\ncd ${location}/"+location+"; ccc_mprun -E \'--exclusive\' -n "+str(n)+" generic_testcase.exe > output_"+arch+"_"+mode+".out 2> error_"+arch+"_"+mode+".out &\n")
             fh.write("PIDS+=($!)\n")
             fh.write("CONFIGS+=("+location+")\n")
     if machine=="jeanzay":
         with open("full_job_"+arch+"_"+mode+".sh", "a") as fh:
-            fh.write("\ncd ${location}/"+location+"; srun --exclusive -n "+str(n)+" generic_testcase.exe > output.out 2> error.out &\n")
+            fh.write("\ncd ${location}/"+location+"; srun --exclusive -n "+str(n)+" generic_testcase.exe > output_"+arch+"_"+mode+".out 2> error_"+arch+"_"+mode+".out &\n")
 
 def main():
@@ -108 +113 @@
             fh.write("#MSUB -r XIOS\n")
             fh.write("#MSUB -eo\n")
-            fh.write("#MSUB -o client_output.out\n")
-            fh.write("#MSUB -e client_error.err\n")
+            fh.write("#MSUB -o output_"+arch+"_"+mode+".out\n")
+            fh.write("#MSUB -e error_"+arch+"_"+mode+".err\n")
             fh.write("#MSUB -c 1\n")
             fh.write("#MSUB -n "+str(nb_proc_irene)+"\n")
@@ -121 +126 @@
             fh.write("export location="+os.getcwd()+"\n")
             fh.write("export log_location="+os.getcwd()+"\n")
-            fh.write("source ../BUILD/build_"+arch+"_"+mode+"/arch.env\n")
+            fh.write("source build_"+arch+"_"+mode+"/arch.env\n")
             fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n")
             fh.write("date >> ${log_location}/Log.txt\n")
@@ -133 +138 @@
             fh.write("#====================================================\n")
             fh.write("#SBATCH --ntasks="+str(nb_proc_jz)+"\n")
+            fh.write("#SBATCH -o output_"+arch+"_"+mode+".out\n")
+            fh.write("#SBATCH -e error_"+arch+"_"+mode+".out\n")
             fh.write("#SBATCH --hint=nomultithread\n")
             fh.write("#SBATCH -t 00:10:00\n")
@@ -138 +145 @@
             fh.write("ulimit -c 0\n")
             fh.write("cd ${SLURM_SUBMIT_DIR}\n")
-            fh.write("source ../BUILD/build_"+arch+"_"+mode+"/arch.env\n")
+            fh.write("source build_"+arch+"_"+mode+"/arch.env\n")
             fh.write("source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt\n")
             fh.write("export location="+os.getcwd()+"\n")
@@ -179 +186 @@
             mystr = mystr.replace(":", "")
             mystr = mystr.replace("u'", "")
+            mystr = mystr.replace("b'", "")
             mystr = mystr.replace("'", "")
             mystr = mystr.replace(" ", "_")

XIOS/trunk/xios_test_suite/TEST_SUITE/step2.py

@@ -7 +7 @@
 import copy
 
-param_list = ["ATMdomain", "UsingServer2", "NumberClients", "NumberServers", "RatioServer2", "NumberPoolsServer2", "Duration"]
-param_short_list = ["ATMdom", "Srv2", "NbClnt", "NbSrv", "RatioSrv2", "NbPlSrv2", "Duration"]
 
 mode=os.getenv("mode")
@@ -15 +13 @@
 ref_location=os.getenv("ref_location")
 ref_file=os.getenv("ref_file")
+
+
 
 def OSinfo(runthis):
@@ -26 +26 @@
 
 
-def get_default_param():
-    f=open("default_param.json", 'r')
-    default_param = json.load(f)
-    f.close()
-    return default_param[0]
 
 def nonblank_lines(f):
@@ -39 +34 @@
 
 def main():
-    OSinfo("cp "+ref_location+"/"+ref_file+" ./")
-    OSinfo("tar -zxvf "+ref_file)
+    ref_list = glob.glob(ref_location+"/*")
+    for i in range(len(ref_list)):
+        tmp = ref_list[i].split("/")
+        rev = tmp[len(tmp)-1]
+        ref_list[i] = int(rev)
+    ref_list.sort(reverse=True) #ref_list in descending order
+    
+    print(ref_list)
+
+    ref_rev = "0"
+    for ref in ref_list:
+        if int(svnr) >= ref :
+            ref_rev = str(rev)
+            break
+        
+    if not int(ref_rev):
+        print("no available reference found ... exit")
+        return
+    
+    OSinfo("cp "+ref_location+"/"+ref_rev+"/"+ref_file+" ./")
+    OSinfo("tar -zxvf "+ref_location+"/"+ref_rev+"/"+ref_file)
     OSinfo("rm -f "+ref_file)
+    
+    
     test_folder_list = glob.glob('test_*')
 
     for test_folder in test_folder_list:
         config_list = glob.glob(test_folder+"/CONFIG_*")
-        #print(*config_list, sep = "\n")