Changeset 5695
- Timestamp:
- 2015-08-17T20:12:12+02:00 (9 years ago)
- Location:
- branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
- Files:
-
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_ADA.cfg
r5693 r5695 1 # Ada configuration 1 ## Ada configuration 2 ##------------------ 2 3 3 # Available softwares:4 ## Available softwares: 4 5 CDO='cdo' 5 6 COMPILER='ifort' … … 7 8 NETCDF='netcdf/mpi' 8 9 9 # Job commands: LoadLeveler10 ## Job commands: LoadLeveler 10 11 JOB_SUBM="llsubmit ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 11 12 JOB_LIST=llq … … 14 15 JOB_DELE=llcancel 15 16 16 # Compiling & computing settings17 ## Compiling & computing settings 17 18 [ -z "$NPROC" ] && NPROC=16 18 19 [ -z "${TIME_LIMI}" ] && TIME_LIMI=10800 # 3h 19 20 20 # Architecture environment (be careful with compliance with XIOS)21 ## Architecture environment (be careful with compliance with XIOS) 21 22 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env 23 24 ## $CDO command to compare 2 datasets 25 CDOD=$CDO' diffn' -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_ADA_at.cfg
r5694 r5695 1 # Ada configuration 1 ## Ada configuration 2 ##------------------ 2 3 3 # Available softwares:4 ## Available softwares: 4 5 CDO='cdo/' 5 6 COMPILER='ifort' … … 7 8 NETCDF='netcdf/' 8 9 9 # Job commands: LoadLeveler10 ## Job commands: LoadLeveler 10 11 JOB_SUBM="llsubmit ${NEMO_TRUS}/batch/${NEMO_HPCC}.sh | grep -oe '\<[0-9]*\>'" 11 12 JOB_LIST=llq … … 14 15 JOB_DELE=llcancel 15 16 16 # Compiling & computing settings17 ## Compiling & computing settings 17 18 [ -z "$NPROC" ] && NPROC=16 18 19 [ -z "${TIME_LIMI}" ] && TIME_LIMI=10800 # 3h 19 20 20 # Architecture environment (be careful with compliance with XIOS)21 ## Architecture environment (be careful with compliance with XIOS) 21 22 ARCH_ENV=${DIR_XIOS}/arch/arch-${NEMO_HPCC}.env 23 24 ## $CDO command to compare 2 datasets 25 CDOD=$CDO' diffn' -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/romr005.cfg
r5685 r5695 34 34 REFE_DIR=${DIR_STOR}/${REFE_CONF}/${NEMO_BRAN} 35 35 TEST_DIR=${DIR_SCRA}/${REFE_CONF}/${NEMO_BRAN}/trusting_${DATE} 36 36 CDOD="$CDO diffn" 37 37 38 38 ## External softs directories: -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.env
r5690 r5695 36 36 export NPROC TIME_LIMI 37 37 export ARCH_ENV 38 export CDOD -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
r5690 r5695 8 8 NEMO_TRUS=$PWD 9 9 DEBUG=0; PUBLISH=0 10 IMOD=0; ST='FAILED' ## No modules system & 'FAILED' status 10 IMOD=0; ST='FAILED' ## No modules system & 'FAILED' status 11 SVN_CMD='svn status' ## No update on directories 12 trus_help='trusting_help.txt' 11 13 12 14 … … 16 18 17 19 case $1 in 18 '-a'|'--archive') NEMO_TARF=$2 ; shift 2;; '-b'|'--branch ') NEMO_BRAN=$2 19 '-d'|'--debug ') set -vx; DEBUG=1; shift ;; '-e'|'--email ') EMAIL=$2 20 '-j'|'--job ') NPROC=$2 ; shift 2;; '-h'|'--help ') cat trusting_help.txt; exit 1;;21 '-m'|'--machine') NEMO_HPCC=$2 ; shift 2;; '-n'|'--newconf') TEST_CONF=$2 22 '-r'|'--refconf') REFE_CONF=$2 ; shift 2;; '-t'|'--time ') TIME_LIMI=$2 23 '-p'|'--publish') PUBLISH=1 ; shift ;; '-u'|'--user ') NEMO_USER=$2 24 '-v'|'--version') NEMO_VERS=$2 ; shift 2;; "*" ) cat trusting_help.txt; exit 1;;20 '-a'|'--archive') NEMO_TARF=$2 ; shift 2;; '-b'|'--branch ') NEMO_BRAN=$2 ; shift 2;; 21 '-d'|'--debug ') set -vx; DEBUG=1; shift ;; '-e'|'--email ') EMAIL=$2 ; shift 2;; 22 '-j'|'--job ') NPROC=$2 ; shift 2;; '-h'|'--help ') cat ${help_file}; exit 1;; 23 '-m'|'--machine') NEMO_HPCC=$2 ; shift 2;; '-n'|'--newconf') TEST_CONF=$2 ; shift 2;; 24 '-r'|'--refconf') REFE_CONF=$2 ; shift 2;; '-t'|'--time ') TIME_LIMI=$2 ; shift 2;; 25 '-p'|'--publish') PUBLISH=1 ; shift ;; '-u'|'--user ') NEMO_USER=$2 ; shift 2;; 26 '-v'|'--version') NEMO_VERS=$2 ; shift 2;; "*" ) cat ${help_file}; exit 1;; 25 27 esac 26 28 … … 31 33 [ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>' 32 34 35 ## If -v|--version option has been set, modify default SVN action on directories 36 if [ $( echo ${NEMO_VERS} | grep "HEAD\|up\|update" ) ]; then 37 SVN_CMD='svn update -r HEAD' 38 elif [ $( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) ]; then 39 SVN_CMD='svn update -r '$( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) 40 elif [ $( echo ${NEMO_VERS} | grep -o '[0-9]*' ) ]; then 41 SVN_CMD='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*' ) 42 fi 43 33 44 34 45 ## Initialization (super-computer & user environment) … … 37 48 38 49 if [ ! $( find ${NEMO_TRUS}/config -name ${NEMO_USER}'.cfg' ) ]; then 39 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_USER}; exit 1 50 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_USER} 51 exit 1 40 52 elif [ ! $( find ${NEMO_TRUS}/config -name ${NEMO_HPCC}'.cfg' ) ]; then 41 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_HPCC}; exit 1 53 echo 'No configuration file (.cfg) in '${NEMO_TRUS}'/config for '${NEMO_HPCC} 54 exit 1 42 55 else 43 56 . ${NEMO_TRUS}/trusting.env … … 45 58 46 59 else 47 cat ${NEMO_TRUS}/trusting_help.txt; exit 1 60 cat ${NEMO_TRUS}/${help_file} 61 exit 1 48 62 fi 49 63 … … 63 77 ## Make timestamped directory with messenger files 64 78 ##--------------------------------------------------- 65 cd ${TEST_DIR} 66 67 print_step 'Create timestamped testing directory' 68 echo ${TEST_DIR} && init_files && get_date 79 print_step 'Timestamped testing directory' 80 mkdir -p ${TEST_DIR} ${REFE_DIR} 81 cd ${TEST_DIR} && echo ${TEST_DIR} 82 init_files 83 get_date 69 84 70 85 71 86 ## Get SVN revision on XIOS & NEMO essentials directories 72 87 ##--------------------------------------------------- 73 print_step 'SVN command on NEMO directories:'; get_nemo_rev 88 print_step "SVN action on NEMO directories: ${SVN_CMD}" 89 get_nemo_rev 74 90 75 91 … … 77 93 ##--------------------------------------------------- 78 94 print_step 'Get softwares releases' 79 80 95 [ -e ${ARCH_ENV} ] && . ${ARCH_ENV} > /dev/null 81 96 [ $? -eq 0 ] && export IMOD=1 82 83 get_soft_rel;cat model.log97 get_soft_rel 98 cat model.log 84 99 85 100 86 101 ## XIOS compilation from scratch 87 102 ##--------------------------------------------------- 103 print_step 'Compile XIOS' 88 104 cd ${DIR_XIOS} 89 90 print_step 'Compile XIOS'91 105 eval ./make_xios ${xios_full} --arch ${NEMO_HPCC} --job $NPROC \ 92 106 ${stderr_redir} /dev/null 93 [ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out 1107 [ ! -e lib/libxios.a ] && get_out 1 || echo 'Success' 94 108 95 109 96 110 ## NEMO compilation from scratch 97 111 ##--------------------------------------------------- 112 print_step "Compile ${REFE_CONF} configuration" 98 113 cd ${NEMO_CONF} 99 100 print_step "Compile ${REFE_CONF} configuration" 101 102 if [[ -d ${TEST_CONF} && $DEBUG -eq 0 ]]; then 103 ./makenemo -n ${TEST_CONF} clean_config > /dev/null <<EOF 114 [[ -d ${TEST_CONF} && $DEBUG -eq 0 ]] && ./makenemo -n ${TEST_CONF} clean_config \ 115 > /dev/null <<EOF 104 116 y 105 117 EOF 106 fi107 118 108 119 eval ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m ${NEMO_HPCC} -j $NPROC \ 109 120 ${KEYS_ADD} ${KEYS_DEL} \ 110 121 ${stderr_redir} /dev/null 111 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 2 122 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 2 || echo 'Success' 112 123 113 124 114 125 ## Get all inputs for running 115 126 ##--------------------------------------------------- 127 print_step 'Set job (copying or extracting inputs)' 116 128 cd ${TEST_DIR} 117 118 print_step 'Set job (copying or extracting inputs)'119 129 cp ${NEMO_CONF}/${TEST_CONF}/cpp_* . 120 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' -exec cp {} . \;121 130 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' \ 131 -exec cp {} . \; 122 132 get_inputs 123 [ $? -ne 0 ] && get_out 3 133 [ $? -ne 0 ] && get_out 3 || echo 'Success' 124 134 [ $( find . -name '*.gz' -print -quit ) ] && find . -name '*.gz' -exec gunzip {} \; 125 135 … … 127 137 ## Check inputs 128 138 ##--------------------------------------------------- 129 print_step 'Compare inputs to benchmark files'; diff_inputs 139 print_step 'Compare inputs to benchmark files' 140 diff_inputs 130 141 131 142 132 143 ## Job submission & computation 133 144 ##--------------------------------------------------- 134 print_step 'Submit job' ; JOB_ID=$( eval ${JOB_SUBM} )135 [ $? -ne 0 ] && get_out 4 136 137 print_step 'Pending job' ; job_pending138 145 print_step 'Submit job' 146 JOB_ID=$( eval ${JOB_SUBM} ) 147 [ $? -ne 0 ] && get_out 4 || echo 'Success (job ID '${JOB_ID}')' 148 print_step 'Pending job' 149 job_pending 139 150 print_step 'Job finished' 140 151 … … 143 154 ##--------------------------------------------------- 144 155 print_step 'Test job state' 145 146 if [[ ! -e time.step || $( grep 'E R R O R' ocean.output ) ]]; then 147 echo 'Failure' 148 get_out 5 149 else 150 echo 'Success' 151 fi 152 153 print_step 'Get real CPU time'; get_cpu_time 156 [[ ! -e time.step || $( grep 'E R R O R' ocean.output ) ]] && get_out 5 || echo 'Success' 157 print_step 'Get real CPU time' 158 get_cpu_time 154 159 155 160 … … 157 162 ##--------------------------------------------------- 158 163 ST='OK' ## 'OK' by default 159 160 print_step 'Compare new outputs to benchmark files'; diff_results; diff_restart 161 [ $ST == 'FAILED' ] && get_out 8 164 print_step 'Compare new outputs to benchmark files' 165 diff_results 166 [ $ST == 'FAILED' ] && echo 'Failure' || echo 'Success' 167 print_step 'Compare new restarts to benchmark files' 168 diff_restart 169 [ $ST == 'FAILED' ] && get_out 8 || echo 'Success' 162 170 163 171 164 172 ## End, at least nothing has changed ;-) 165 173 ##--------------------------------------------------- 166 get_out 'Code is reliable'174 get_out 0 -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh
r5690 r5695 1 1 #!/bin/bash 2 2 3 4 mkdir -p ${TEST_DIR} ${REFE_DIR}5 3 6 4 ## Messenger filenames … … 45 43 46 44 get_nemo_rev() { 47 local rev_loc svn_cmd 48 local rev=0 svn_opt='status' 49 50 ## If -v|--version option has been set, modify svn command 51 if [ $( echo ${NEMO_VERS} | grep "HEAD\|up\|update" ) ]; then 52 svn_cmd='svn update -r HEAD' 53 elif [ $( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) ]; then 54 svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) 55 elif [ $( echo ${NEMO_VERS} | grep -o '[0-9]*' ) ]; then 56 svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*' ) 57 else 58 svn_cmd='svn '${svn_opt} 59 fi 45 local dir rev_loc 46 local rev=0 60 47 61 48 ## Loop on essential directories … … 75 62 fi 76 63 77 echo $dir 78 ${svn_cmd} $dir 64 echo $dir && ${SVN_CMD} $dir 79 65 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 80 66 … … 91 77 92 78 get_soft_rel() { 93 local soft _rel79 local soft soft_rel 94 80 95 81 for soft in $CDO $COMPILER $MPI $NETCDF; do … … 129 115 130 116 get_inputs() { 131 # List archive content & extract by default117 # List archive content & extract it by default 132 118 local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}" 133 119 134 ## List & copy file in case of personal inputs120 ## List & copy files in case of personal inputs 135 121 [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ." 136 122 … … 141 127 142 128 diff_inputs() { 143 local dif 129 local dif file 144 130 local files_list='' mesg='Same' 145 131 … … 152 138 153 139 ## Pass over useless file omission in benckmark directory 154 if [ -n "$dif" && "$dif" != '0']; then140 if [[ -n "$dif" && "$dif" != '0' ]]; then 155 141 mesg='Different'; echo $dif; files_list+=$file' ' 156 142 fi … … 186 172 187 173 ## Kill remaining job & stop the test if it's too long 188 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 189 ${JOB_DELE} ${JOB_ID} &> /dev/null 190 get_out 6 191 fi 174 [ ${time_elapsed} -eq ${TIME_LIMI} ] && ${JOB_DELE} ${JOB_ID} &> /dev/null && get_out 6 192 175 } 193 176 194 177 195 178 diff_results() { 179 local file 196 180 local files_list='' 197 181 … … 215 199 216 200 diff_restart() { 217 local base_name dif file list_comp list_tmsp nb_dom201 local base_name comp dif file list_comp list_tmsp nb_dom time_step tmsp 218 202 local files_list='' dift=0 219 203 220 ## Stop if no benchmark files ( time.step)204 ## Stop if no benchmark files (ie time.step) 221 205 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 7 222 223 206 time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 224 echo 'Last time step of benchmark run: '${time_step}225 207 226 208 ## Find all restart files to rebuild … … 253 235 254 236 ## UNIX `cmp` not suitable (timestamp in .nc file) 255 dif=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null\256 | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' 257 258 ## CDO can return no output onstdout with no difference237 dif=$( $CDOD $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 238 | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' ) 239 240 ## CDO can return void stdout with no difference 259 241 if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then 260 242 export ST='FAILED' && files_list+=$file' ' … … 277 259 if [ $dift -ne 0 ]; then 278 260 echo 'Restarts: '${files_list}$dift' record(s) differ<br>' | tee -a temp_${FILE_COMM} 279 else280 echo 'No restart differ'281 261 fi 282 262 … … 297 277 298 278 comments() { 299 local opt 300 local state=$1 301 302 ## 'W A R N I N G' by default 303 opt='-A2 "^ $state"' 304 [ "$state" == 'E R R O R' ] && opt="-A4 $state" 305 306 ## Select first occurence for web comment 307 line=$( grep -m1 $opt ocean.output | tr -d '\n' ) 308 309 echo $line 279 local opat 280 local line='' state=$1 281 282 if [ -e ocean.output ]; then 283 ## 'W A R N I N G' by default 284 opat="-A2 '^ $state'" 285 [ "$state" == 'E R R O R' ] && opat="-A4 '$state'" 286 287 ## Select first occurence for web comment 288 line=$( grep -m1 $opat ocean.output | tr -d '\n' ) 289 echo $line 290 fi 291 310 292 printf "$line<br>" \ 311 293 >> temp_${FILE_COMM} … … 364 346 365 347 get_out() { 348 local time_step=0 349 366 350 ERR=$1 367 368 cd ${TEST_DIR} 351 cd ${TEST_DIR} ## In case of compilation error 369 352 370 353 if [ $ST == 'FAILED' ]; then 354 echo 'Failure' 371 355 372 356 ## Error identification 373 357 case $ERR in 374 375 1) ERR='XIOS compilation failed' ;; 2) ERR='NEMO compilation failed';;358 ## Compilation 359 '1') ERR='XIOS compilation failed' ;; '2') ERR='NEMO compilation failed';; 376 360 ## Submission 377 3) ERR='Missing input files' ;; 4) ERR='Job submission error' ;;361 '3') ERR='Missing input files' ;; '4') ERR='Job submission error' ;; 378 362 ## Running 379 5) ERR='Crashed at time step' ;; 6) ERR='Exceeded time limit' ;;363 '5') ERR='Crashed at time step' ;; '6') ERR='Exceeded time limit' ;; 380 364 ## Results 381 7) ERR='Missing previous outputs';; 8) ERR='New outputs differ' ;;365 '7') ERR='Missing previous outputs';; '8') ERR='New outputs differ' ;; 382 366 ## Undefined 383 *) ERR='Unknown error' ;;367 "*") ERR='Unknown error' ;; 384 368 esac 385 369 386 fi 370 else 371 ERR='Code is reliable' 372 fi 373 374 print_step '' 387 375 388 376 ## Eventual comments from ocean.output 389 print_step 'Comments' 390 if [ -e ocean.output ]; then 391 if [ $ERR -eq 5 ]; then 392 comments 'E R R O R' 393 [ -e time.step ] && ERR+=' '$( cat time.step | tr -d ' ' ) 394 else 395 comments 'W A R N I N G' 396 [ $ERR -eq 6 ] && ERR+=' '$(( ${TIME_LIMIT}/3600 ))'h' 397 fi 398 fi 399 400 ## Share trusting result (locally or not) 377 if [ "$ERR" == 'Crashed at time step' ]; then 378 comments 'E R R O R' 379 [ -e time.step ] && time_step=$( cat time.step | tr -d ' ' ) 380 ERR+=' '$time_step 381 else 382 comments 'W A R N I N G' 383 [ "$ERR" == 'Exceeded time limit' ] && ERR+=' '$(( ${TIME_LIMIT}/3600 ))'h' 384 fi 385 401 386 export ERR 402 sed -i "2 s/.*/$ST/" ${FILE_STAT} 403 sed -i "2 s/.*/$ERR/" ${FILE_RESU} 387 sed -i "2 s/.*/$ST/" ${FILE_STAT}; sed -i "2 s/.*/$ERR/" ${FILE_RESU} 404 388 405 389 ## Save tested configuration if trusting failed in production mode (-p|--publish) 406 if [[ $ST == 'FAILED' && $PUBLISH -eq 1 && $DEBUG -eq 0]]; then390 if [[ $ST == 'FAILED' && $PUBLISH -eq 1 ]]; then 407 391 echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} 408 392 tar -czf ${REFE_DIR}/${FILE_ARCH} * ${NEMO_HOME}/CONFIG/${TEST_CONF}/MY_SRC \ … … 410 394 fi 411 395 412 ## 413 print_step 'Trusting outcome'; mesg_make; mesg_publish 396 ## Share trusting result (.txt file/mail) 397 mesg_make 398 mesg_publish 414 399 415 400 exit 0
Note: See TracChangeset
for help on using the changeset viewer.