1 | import glob |
---|
2 | import sys |
---|
3 | import subprocess |
---|
4 | import os |
---|
5 | import json |
---|
6 | import itertools |
---|
7 | import copy |
---|
8 | |
---|
9 | import netCDF4 |
---|
10 | from netCDF4 import Dataset |
---|
11 | import numpy as np |
---|
12 | |
---|
13 | mode=os.getenv("mode") |
---|
14 | arch=os.getenv("arch") |
---|
15 | svnr=os.getenv("svnR") |
---|
16 | ref_location=os.getenv("ref_location") |
---|
17 | ref_file=os.getenv("ref_file") |
---|
18 | |
---|
19 | |
---|
20 | |
---|
21 | def OSinfo(runthis): |
---|
22 | red = lambda text: '\033[0;31m' + text + '\033[0m' |
---|
23 | osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) |
---|
24 | theInfo, theErr = osstdout.communicate() |
---|
25 | if theErr: |
---|
26 | print(red(runthis+" FAILED")) |
---|
27 | print(theErr) |
---|
28 | sys.exit() |
---|
29 | |
---|
30 | |
---|
31 | |
---|
32 | def nonblank_lines(f): |
---|
33 | for l in f: |
---|
34 | line = l.rstrip() |
---|
35 | if line and not line.startswith("#"): |
---|
36 | yield line |
---|
37 | |
---|
38 | def main(): |
---|
39 | ref_list = glob.glob(ref_location+"/*") |
---|
40 | for i in range(len(ref_list)): |
---|
41 | tmp = ref_list[i].split("/") |
---|
42 | rev = tmp[len(tmp)-1] |
---|
43 | ref_list[i] = int(rev) |
---|
44 | ref_list.sort(reverse=True) #ref_list in descending order |
---|
45 | |
---|
46 | ref_rev = "" |
---|
47 | for ref in ref_list: |
---|
48 | if int(svnr) >= ref : |
---|
49 | ref_rev = str(ref) |
---|
50 | print("corresponding reference = ", ref) |
---|
51 | break |
---|
52 | |
---|
53 | if not ref_rev: |
---|
54 | print("no available reference found ... exit") |
---|
55 | return |
---|
56 | |
---|
57 | OSinfo("cp "+ref_location+"/"+ref_rev+"/"+ref_file+" ./") |
---|
58 | OSinfo("tar -zxvf "+ref_location+"/"+ref_rev+"/"+ref_file) |
---|
59 | OSinfo("rm -f "+ref_file) |
---|
60 | |
---|
61 | |
---|
62 | test_folder_list = glob.glob('test_*') |
---|
63 | |
---|
64 | for test_folder in test_folder_list: |
---|
65 | config_list = glob.glob(test_folder+"/CONFIG_*") |
---|
66 | |
---|
67 | |
---|
68 | with open(test_folder+"/checkfile.def", "r") as fh: |
---|
69 | checkfiles = list(nonblank_lines(fh)) |
---|
70 | |
---|
71 | with open("report_"+svnr+"_"+arch+"_"+mode+".txt", "a") as report: |
---|
72 | for config in config_list: |
---|
73 | folder_name = list(config.split("/"))[0] |
---|
74 | config_name = list(config.split("/"))[1] |
---|
75 | for checkfile in checkfiles: |
---|
76 | if os.path.exists(config+"/"+checkfile) and os.path.exists("reference/ref_"+config+"/"+checkfile): |
---|
77 | #OSinfo("cdo -W diffn "+config+"/"+checkfile+" "+"reference/ref_"+config+"/"+checkfile+" 2>&1 |grep -v 'Found more than one time variable'|grep -v 'cdo diffn: Processed'|grep -v 'cdo diffn: Processed'|grep -v 'Time variable >time_counter< not found!' > diff_"+checkfile+".txt") |
---|
78 | #if os.stat("diff_"+checkfile+".txt").st_size==0: # if no diff -> set 0 |
---|
79 | # report.write(folder_name+" "+folder_name+"@"+config_name+" "+folder_name+"@"+config_name+"@"+checkfile+" "+str(1)+"\n") |
---|
80 | #else: # if cdo diffn returns diff -> set -1 |
---|
81 | # report.write(folder_name+" "+folder_name+"@"+config_name+" "+folder_name+"@"+config_name+"@"+checkfile+" "+str(-1)+"\n") |
---|
82 | ref = Dataset( "reference/ref_"+config+"/"+checkfile ) |
---|
83 | res = Dataset( config+"/"+checkfile ) |
---|
84 | validated = 1 |
---|
85 | for var in res.variables: |
---|
86 | if (not (var.startswith('lon_'))) and (not (var.startswith('lat_'))) and (not (var.startswith('time_'))) and (not (var.startswith('atm__'))): |
---|
87 | ref_interp = ref.variables[var] |
---|
88 | ref_array = ref_interp[:] |
---|
89 | res_interp = res.variables[var] |
---|
90 | res_array = res_interp[:] |
---|
91 | if (res_array.shape == ref_array.shape): |
---|
92 | diff = np.zeros_like( ref_array ) |
---|
93 | np.divide(ref_array-res_array,ref_array,diff,where=(ref_array[:]>10**-15)) |
---|
94 | if ( np.max(np.abs(diff)) > 1*10**-9 ): |
---|
95 | validated = -1 |
---|
96 | diff = np.zeros_like( ref_array ) |
---|
97 | np.divide(ref_array-res_array,res_array,diff,where=(ref_array[:]>10**-15)) |
---|
98 | if ( np.max(np.abs(diff)) > 1*10**-9 ): |
---|
99 | validated = -1 |
---|
100 | else: |
---|
101 | validated = -1 |
---|
102 | report.write(folder_name+" "+folder_name+"@"+config_name+" "+folder_name+"@"+config_name+"@"+checkfile+" "+str(validated)+"\n") |
---|
103 | |
---|
104 | elif os.path.exists(config+"/"+checkfile): # if no ref file -> set 0 |
---|
105 | report.write(folder_name+" "+folder_name+"@"+config_name+" "+folder_name+"@"+config_name+"@"+checkfile+" "+str(0)+"\n") |
---|
106 | elif os.path.exists("reference/ref_"+config+"/"+checkfile): # if no output file -> set -2 |
---|
107 | report.write(folder_name+" "+folder_name+"@"+config_name+" "+folder_name+"@"+config_name+"@"+checkfile+" "+str(-2)+"\n") |
---|
108 | |
---|
109 | |
---|
110 | |
---|
111 | if __name__== "__main__": |
---|
112 | main() |
---|