Changeset 800 for trunk/libIGCM/libIGCM_sys/libIGCM_sys_default.ksh
- Timestamp:
- 02/18/13 17:09:13 (11 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/libIGCM/libIGCM_sys/libIGCM_sys_default.ksh
r785 r800 23 23 #D- 24 24 #D- This ksh library if a layer under some usefull 25 #D- 26 #D- 27 #D- 28 #D- 25 #D-environment variables and shell commands. 26 #D-All those definitions depend on host particularities. 27 #D-It manages a stack mechanism and test validity of operations. 28 #D-All function described bellow must be prefixed by IGCM_sys. 29 29 30 30 #==================================================== … … 33 33 34 34 #==================================================== 35 # set DEBUG_sys to true to output calls of function35 # set DEBUG_sys to true to output calls of function 36 36 typeset -r DEBUG_sys=${DEBUG_sys:=true} 37 37 … … 56 56 57 57 #===================================================== 58 # Global Variables :58 # Global Variables : 59 59 #===================================================== 60 60 # Language : "fr" or "en" … … 79 79 # rsync with path 80 80 typeset -r RSYNC=/usr/bin/rsync 81 # RSYNC_opt args to rsync81 # RSYNC_opt args to rsync 82 82 typeset -r RSYNC_opt="-va" 83 #"-Lt -v"84 83 # ie storage filesystem 85 84 typeset -r STOREHOST=${MASTER} … … 99 98 100 99 #==================================================== 101 #- libIGCM_POST 100 #- libIGCM_POST for frontend 102 101 typeset -r libIGCM_POST=${libIGCM} 103 102 … … 107 106 108 107 #==================================================== 109 #- SCRATCHDIR (=> ${R_DONNEES})110 typeset -r SCRATCHDIR=/tmp111 112 #====================================================113 108 #- SUBMIT_DIR : submission dir 114 109 typeset SUBMIT_DIR=${SUBMIT_DIR:=${PWD}} … … 117 112 #- IN 118 113 typeset -r R_IN=${R_IN:=/home/${LOGIN}/IGCM} 114 typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/home/${LOGIN}/IGCM} 119 115 120 116 #==================================================== … … 127 123 128 124 #==================================================== 129 #- OUT 130 IGCM_OUT=${IGCM_OUT:=IGCM_OUT} 131 typeset -r R_OUT=${ARCHIVE}/${IGCM_OUT} 125 #- R_OUT 126 typeset R_OUT=${ARCHIVE}/IGCM_OUT 132 127 133 128 #==================================================== 134 129 #- R_FIG (hosting figures : monitoring and atlas, and/or small files) 135 typeset -rR_FIG=${STORAGE}/IGCM_OUT130 typeset R_FIG=${STORAGE}/IGCM_OUT 136 131 137 132 #==================================================== 138 133 #- R_BUF (ONLY FOR double copy an scratch) 139 typeset -r R_BUF=${S CRATCHDIR}/IGCM_OUT134 typeset -r R_BUF=${STORAGE}/IGCM_OUT 140 135 141 136 #==================================================== 142 137 #- BIG_DIR : BIG_DIR to store files waiting for rebuild 143 typeset -r BIG_DIR=${BIG_DIR:=${S CRATCHDIR}/REBUILD}138 typeset -r BIG_DIR=${BIG_DIR:=${STORAGE}/REBUILD} 144 139 145 140 #==================================================== 146 141 #- RUN_DIR_PATH : Temporary working directory (=> TMP) 147 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${S CRATCHDIR}/tmp$$}142 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${STORAGE}/RUN_DIR/tmp$$} 148 143 149 144 #==================================================== 150 145 #- HOST_MPIRUN_COMMAND 151 146 typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"} 152 153 #=========================================================154 #- Add "time" before mpirun command155 MPIRUN_COMMAND="time "${MPIRUN_COMMAND}156 echo ${MPIRUN_COMMAND}157 147 158 148 #==================================================== … … 186 176 #D-#================================================== 187 177 #D-function IGCM_sys_RshMaster 188 #D-* Purpose: Just a fake command to wrapp 189 #D- IGCM_card call in post-treatment 190 #D- Ulam do not see brodie filesystem 191 #D- Cesium do not see all mercure filesystem 192 #D- That's why we need this hack. 178 #D-* Purpose: Connection to frontend machine. 193 179 #D-* Examples: 194 180 #D- 195 181 function IGCM_sys_RshMaster { 196 182 IGCM_debug_PushStack "IGCM_sys_RshMaster" $@ 197 ssh ${HOST} /bin/ksh<<-EOF183 /bin/ksh <<-EOF 198 184 export libIGCM=${libIGCM} 199 185 export DEBUG_debug=${DEBUG_debug} … … 216 202 function IGCM_sys_RshArchive { 217 203 IGCM_debug_PushStack "IGCM_sys_RshArchive" $@ 218 /bin/ksh 204 /bin/ksh <<-EOF 219 205 ${@} 220 206 EOF … … 231 217 #D-* Examples: 232 218 #D- 233 (( RshPNum = 0 ))234 219 function IGCM_sys_RshPost { 235 220 IGCM_debug_PushStack "IGCM_sys_RshPost" $@ … … 237 222 echo "IGCM_sys_RshPost :" $@ 238 223 fi 239 #/bin/ksh -vx $@ > out_RshPost.${RshPNum} 240 ssh ${HOST} /bin/ksh $@ > out_RshPost.${RshPNum} 224 225 #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL 226 #cat tmp_IGCM_sys_RshPost_$$ 227 # keep standard input (stdin) for the loop onto temporary file 228 cat >tmp_IGCM_sys_RshPost_$$ 229 230 /bin/ksh <tmp_IGCM_sys_RshPost_$$ 241 231 if [ $? -gt 0 ] ; then 242 232 echo "IGCM_sys_RshPost : erreur." 243 233 IGCM_debug_Exit "IGCM_sys_RshPost" 244 234 fi 245 (( RshPNum = RshPNum + 1 )) 235 # delete temporary file 236 \rm tmp_IGCM_sys_RshPost_$$ 237 246 238 IGCM_debug_PopStack "IGCM_sys_RshPost" 247 239 } … … 263 255 status=completed 264 256 fi 257 265 258 cat << END_MAIL > job_end.mail 266 259 Dear ${LOGIN}, … … 270 263 Job ended : ${DateEnd} 271 264 Output files are available in ${R_SAVE} 265 Files to be rebuild are temporarily available in ${REBUILD_DIR} 266 Pre-packed files are temporarily available in ${R_BUFR} 272 267 Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR} 273 268 END_MAIL … … 303 298 fi 304 299 fi 305 300 # vérification : 306 301 if [ ! -d ${1} ] ; then 307 302 echo "IGCM_sys_Mkdir : erreur." … … 321 316 echo "IGCM_sys_MkdirArchive :" $@ 322 317 fi 323 318 #- creation de repertoire sur le serveur fichier 324 319 if [ ! -d ${1} ]; then 325 320 \mkdir -p $1 … … 342 337 echo "IGCM_sys_MkdirWork :" $@ 343 338 fi 344 339 #- creation de repertoire sur le serveur fichier 345 340 if [ ! -d ${1} ]; then 346 341 \mkdir -p $1 … … 446 441 447 442 #D-#================================================== 443 #D-function IGCM_sys_IsFileArchived 444 #D-* Purpose: Test file that must NOT EXISTS on Archive 445 #D-* Examples: 446 #D- 447 function IGCM_sys_IsFileArchived { 448 IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@ 449 if ( $DEBUG_sys ) ; then 450 echo "IGCM_sys_IsFileArchived :" $@ 451 fi 452 typeset IsArchivedFlag 453 # Never archived for this system 454 IsArchivedFlag=1 455 IGCM_debug_PopStack "IGCM_sys_IsFileArchived" 456 457 return ${IsArchivedFlag} 458 } 459 460 #D-#================================================== 448 461 #D-function IGCM_sys_TestFileArchive 449 462 #D-* Purpose: Test file that must NOT EXISTS on Archive … … 563 576 echo "IGCM_sys_Qsub :" $@ 564 577 fi 565 echo " ssh -l ${LOGIN} ${HOST} ""cd ${SUBMIT_DIR} ; ${@}"" >${Script_Output} 2>&1 &"566 ssh -l ${LOGIN} ${HOST} "cd ${SUBMIT_DIR} ; ${@}" >${Script_Output} 2>&1 &578 echo "${@} > ${SUBMIT_DIR}/${Script_Output} 2>&1 &" 579 ${@} > ${SUBMIT_DIR}/${Script_Output} 2>&1 & 567 580 if [ $? -gt 0 ] ; then 568 echo "IGCM_sys_Qsub : erreur ."581 echo "IGCM_sys_Qsub : erreur ${options} $1" 569 582 IGCM_debug_Exit "IGCM_sys_Qsub" 570 583 fi … … 582 595 echo "IGCM_sys_QsubPost :" $@ 583 596 fi 584 #echo "ssh -x -l ${LOGIN} ${HOST} ${3} > ${2} 2>&1 &"585 # ssh -x -l ${LOGIN} ${HOST}586 597 echo "${libIGCM_POST}/$1.job > ${POST_DIR}/$1.${PeriodDateEnd}.out 2>&1 &" 587 598 ${libIGCM_POST}/$1.job > ${POST_DIR}/${Script_Post_Output}.out 2>&1 & 588 # ${3} > ${2} 2>&1 &589 599 if [ $? -gt 0 ] ; then 590 600 echo "IGCM_sys_QsubPost : erreur " $@ … … 602 612 #D-function IGCM_sys_Rsync_out 603 613 #D-* Purpose: treat return val of rsync 604 #D-* Examples: 614 #D-* Examples: IGCM_sys_Rsync_out out_RET_rsync 605 615 #D- Error values and explanations can depend on your system version. 606 616 function IGCM_sys_Rsync_out { 607 RET=$1608 if [ ! $ RET] ; then617 status=$1 618 if [ ! $status ] ; then 609 619 echo "rsync error !" 610 620 fi 611 621 612 622 if [ $MYLANG = "fr" ]; then 613 case $ RETin623 case $status in 614 624 0) return ;; 615 625 1) echo "Erreur de rsync ; RERR_SYNTAX : " … … 659 669 echo "Temps d'attente écoulé dans l'envoi/réception de données" 660 670 return;; 661 *) echo "Erreur de rsync : code de retour de rsync inconnu :" $ RET671 *) echo "Erreur de rsync : code de retour de rsync inconnu :" $status 662 672 return;; 663 673 esac 664 674 elif [ $MYLANG = "en" ] ; then 665 case $ RETin675 case $status in 666 676 0) return;; 667 677 1) echo "rsync error : Syntax or usage error " … … 700 710 30) echo "rsync error : Timeout in data send/receive " 701 711 return;; 702 *) echo "rsync error : return code of rsync unknown :" $ RET712 *) echo "rsync error : return code of rsync unknown :" $status 703 713 return;; 704 714 esac … … 773 783 if ( $DEBUG_sys ) ; then 774 784 echo "IGCM_sys_RmRunDir :" $@ 775 echo "Dummy call, let the scheduler do that."785 echo "Dummy call, let the user do that." 776 786 # IGCM_sys_Rm -rf ${RUN_DIR} 777 787 fi … … 870 880 typeset RET 871 881 872 882 #USUAL WAY 873 883 \cp -R $1 $2 > out_rsync 2>&1 874 884 RET=$? … … 906 916 typeset RET 907 917 908 918 #USUAL WAY 909 919 cp -RL $1 $2 > out_rsync 2>&1 910 920 RET=$? … … 1041 1051 1042 1052 typeset RET 1043 1053 # 1044 1054 if [ X${JobType} = XRUN ] ; then 1045 1055 if [ X${3} = X ] ; then … … 1047 1057 fi 1048 1058 fi 1049 1059 # 1050 1060 1051 1061 echo ${RSYNC} ${RSYNC_opt} $1 $2 > out_rsync 2>&1 … … 1138 1148 eval DEST=\${${#}} 1139 1149 1140 1150 # test if the (first) file is present in the old computation : 1141 1151 IGCM_sys_TestFileArchive ${dm_liste[0]} 1142 1152 RET=$? … … 1238 1248 eval ${2}=${dateF[5]} 1239 1249 1240 1250 # donne la date filesys d'un fichier sur la machine work 1241 1251 IGCM_debug_PopStack "IGCM_sys_FichWork" 1242 1252 } … … 1270 1280 return 0 1271 1281 } 1272 1273 1282 1274 1283 #D-#================================================== … … 1332 1341 list_opt=$@ 1333 1342 1334 # Invert Axis : t,x -> x,t 1343 # Invert Axis : t,x -> x,t 1335 1344 # t,pres,x -> x,t,pres 1336 1345 # So that we can concatenate along x … … 1395 1404 # j3=${proc_stn[${iStation}]} 1396 1405 # Extract station 1397 # Invert Axis : t,x -> x,t 1406 # Invert Axis : t,x -> x,t 1398 1407 # t,pres,x -> x,t,pres 1399 1408 # So that we can concatenate along x … … 1438 1447 1439 1448 ############################################################ 1440 # Build run file1449 # Build MPI/OMP scripts run file (dummy function) 1441 1450 1442 1451 function IGCM_sys_build_run_file { … … 1740 1749 \rm out_rsync 1741 1750 IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1751 sleep $DELAI 1742 1752 else 1743 1753 \rm out_rsync … … 1776 1786 \rm out_rsync 1777 1787 IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1788 sleep $DELAI 1778 1789 else 1779 1790 \rm out_rsync … … 1812 1823 \rm out_rsync 1813 1824 IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1825 sleep $DELAI 1814 1826 else 1815 1827 \rm out_rsync … … 1848 1860 \rm out_rsync 1849 1861 IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1862 sleep $DELAI 1850 1863 else 1851 1864 \rm out_rsync … … 1884 1897 \rm out_rsync 1885 1898 IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1899 sleep $DELAI 1886 1900 else 1887 1901 \rm out_rsync … … 1920 1934 \rm out_rsync 1921 1935 IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1936 sleep $DELAI 1922 1937 else 1923 1938 \rm out_rsync … … 1956 1971 \rm out_rsync 1957 1972 IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 1973 sleep $DELAI 1958 1974 else 1959 1975 \rm out_rsync … … 1992 2008 \rm out_rsync 1993 2009 IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 2010 sleep $DELAI 1994 2011 else 1995 2012 \rm out_rsync … … 2028 2045 \rm out_rsync 2029 2046 IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 2047 sleep $DELAI 2030 2048 else 2031 2049 \rm out_rsync … … 2064 2082 \rm out_rsync 2065 2083 IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 2084 sleep $DELAI 2066 2085 else 2067 2086 \rm out_rsync … … 2100 2119 \rm out_rsync 2101 2120 IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 2121 sleep $DELAI 2102 2122 else 2103 2123 \rm out_rsync … … 2136 2156 \rm out_rsync 2137 2157 IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 2158 sleep $DELAI 2138 2159 else 2139 2160 \rm out_rsync … … 2172 2193 \rm out_rsync 2173 2194 IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again." 2195 sleep $DELAI 2174 2196 else 2175 2197 \rm out_rsync
Note: See TracChangeset
for help on using the changeset viewer.