source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ciclad.ksh @ 1210

Last change on this file since 1210 was 1210, checked in by sdipsl, 9 years ago
  • we don't want the module list when running checker like tools
  • Property svn:keywords set to Revision Date Author
File size: 35.8 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for ciclad
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=${MASTER:=$( hostname )}
71# project name
72typeset PROJECT=NONE
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${PBS_WALLTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=qsub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89
90#====================================================
91# Access to module command
92#====================================================
93. /usr/share/Modules/init/ksh
94
95#====================================================
96# Set environment tools (ferret, nco, cdo, rebuild, ...)
97#====================================================
98. /home/igcmg/.atlas_env_ciclad_ksh
99[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
100[ ! X${TaskType} = Xchecking ] && module list
101
102export PATH=${PATH}:/home/igcmg/rebuild/src_X64_CICLAD/modipsl_v2_2_3_netcdf4.2/bin/
103
104#====================================================
105# Host specific DIRECTORIES
106#====================================================
107
108#====================================================
109#- MirrorlibIGCM for frontend
110typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
111
112#====================================================
113#- libIGCM_POST for frontend
114typeset -r libIGCM_POST=${libIGCM}
115
116#====================================================
117#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
118typeset -r R_EXE="${MODIPSL}/bin"
119
120#====================================================
121#- SUBMIT_DIR : submission dir
122if [ X${PBS_O_WORKDIR} != X ] ; then
123  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
124else
125  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
126fi
127
128#====================================================
129#- IN
130typeset -r R_IN=${R_IN:=/prodigfs/ipslfs/igcmg/IGCM}
131typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/prodigfs/ipslfs/igcmg/IGCM}
132
133#====================================================
134#- RUN_DIR_PATH : Temporary working directory (=> TMP)
135if [ X${PBS_JOBID} != X ] ; then
136  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/data/${LOGIN}/RUN_DIR/${PBS_JOBID}_${$}}
137else
138  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
139fi
140
141#====================================================
142#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
143typeset -r OUTCOMMAND_PATH=/tmp
144
145#====================================================
146#- HOST_MPIRUN_COMMAND
147typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
148
149#====================================================
150#- Max number of arguments passed to nco operator or demigration command
151UNIX_MAX_LIMIT=360
152
153#====================================================
154#- set PackDefault to false on ciclad
155PackDefault=false
156
157#====================================================
158#- Number of core per node (max number of OpenMP task)
159NUM_COREPERNODE=1
160
161#====================================================
162#- Default number of MPI task for IPSL coupled model
163#- required for backward compatibility
164#-
165DEFAULT_NUM_PROC_OCE=1
166DEFAULT_NUM_PROC_CPL=1
167(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
168DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
169
170#D-#==================================================
171#D-function IGCM_sys_defineArchives
172#D-* Purpose:
173#D-* Define ARCHIVE : Dedicated to large files
174#D-* Define STORAGE : Dedicated to small/medium files
175#D-* Define R_OUT   : Output tree located on ARCHIVE
176#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
177#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
178#D-* if SpaceName=TEST nothing special will hapen
179#D-* Examples:
180#D-
181function IGCM_sys_defineArchives {
182  IGCM_debug_PushStack "IGCM_sys_defineArchives"
183
184  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
185    #====================================================
186    #- ARCHIVE (dedicated to large files)
187    ARCHIVE=${config_UserChoices_ARCHIVE}
188  else
189    #====================================================
190    #- ARCHIVE (dedicated to large files)
191    ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
192  fi
193
194  if [ ! X${config_UserChoices_STORAGE} = X ]; then
195    #====================================================
196    #- STORAGE (dedicated to small/medium files)
197    STORAGE=${config_UserChoices_STORAGE}
198  else
199    #====================================================
200    #- STORAGE (dedicated to small/medium files)
201    STORAGE=${ARCHIVE}
202  fi
203
204  # ON CICLAD NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
205
206  #====================================================
207  #- R_OUT
208  R_OUT=${ARCHIVE}/IGCM_OUT
209
210  #====================================================
211  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
212  R_FIG=${STORAGE}/IGCM_OUT
213
214  #====================================================
215  #- R_BUF (ONLY FOR double copy an scratch)
216  R_BUF=${STORAGE}/IGCM_OUT
217
218  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
219  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
220  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
221
222  IGCM_debug_PopStack "IGCM_sys_defineArchives"
223}
224
225#D-#==================================================
226#D-function IGCM_sys_RshArchive
227#D-* Purpose: Archive rsh command
228#D-* Examples:
229#D-
230function IGCM_sys_RshArchive {
231  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
232  /bin/ksh <<-EOF
233    ${@}
234EOF
235  status=$?
236  if [ ${status} -gt 0 ] ; then
237    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
238    IGCM_debug_Exit "IGCM_sys_RshArchive"
239  fi
240  IGCM_debug_PopStack "IGCM_sys_RshArchive"
241}
242
243#D-#==================================================
244#D-function IGCM_sys_RshArchive_NoError
245#D-* Purpose: Archive rsh command, without error
246#D-*          used only in monitoring.job
247#D-* Examples:
248#D-
249function IGCM_sys_RshArchive_NoError {
250  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
251  /bin/ksh <<-EOF
252    ${@} 2> /dev/null
253EOF
254  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
255}
256
257#D-#==================================================
258#D-function IGCM_sys_MkdirArchive
259#D-* Purpose: Mkdir on Archive
260#D-* Examples:
261#D-
262function IGCM_sys_MkdirArchive {
263  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
264  if ( $DEBUG_sys ) ; then
265    echo "IGCM_sys_MkdirArchive :" $@
266  fi
267  #- creation de repertoire sur le serveur fichier
268  if [ ! -d ${1} ]; then
269    \mkdir -p $1
270    status=$?
271
272    if [ ${status} -gt 0 ] ; then
273      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
274      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
275    fi
276  fi
277  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
278}
279
280#D-#==================================================
281#D-function IGCM_sys_TestDirArchive
282#D-* Purpose: Test Directory that must exists on Archive
283#D-* Examples:
284#D-
285function IGCM_sys_TestDirArchive {
286  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
287  if ( $DEBUG_sys ) ; then
288    echo "IGCM_sys_TestDirArchive :" $@
289  fi
290  typeset ExistFlag
291  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
292  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
293  return ${ExistFlag}
294}
295
296#D-#==================================================
297#D-function IGCM_sys_IsFileArchived
298#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
299#D-* Examples:
300#D-
301function IGCM_sys_IsFileArchived {
302  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
303  if ( $DEBUG_sys ) ; then
304    echo "IGCM_sys_IsFileArchived :" $@
305  fi
306  typeset IsArchivedFlag
307  # Never archived for this system
308  IsArchivedFlag=1
309  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
310
311  return ${IsArchivedFlag}
312}
313
314#D-#==================================================
315#D-function IGCM_sys_TestFileArchive
316#D-* Purpose: Test file that must NOT EXISTS on Archive
317#D-* Examples:
318#D-
319function IGCM_sys_TestFileArchive {
320  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
321  typeset ExistFlag
322  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
323  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
324
325  return ${ExistFlag}
326}
327
328#D-#==================================================
329#D-function IGCM_sys_CountFileArchive
330#D-* Purpose: Count files on Archive filesystem
331#D-* Examples:
332#D-
333function IGCM_sys_CountFileArchive {
334  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
335  ls ${@} 2>/dev/null | wc -l
336  if [ $? -gt 0 ] ; then
337    echo "IGCM_sys_CountFileArchive : erreur."
338  fi
339  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
340}
341
342#D-#==================================================
343#D-function IGCM_sys_Tree
344#D-* Purpose: Tree directories with files on ${ARCHIVE}
345#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
346#D-
347function IGCM_sys_Tree {
348  IGCM_debug_PushStack "IGCM_sys_Tree" $@
349  if ( $DEBUG_sys ) ; then
350    echo "IGCM_sys_Tree :" $@
351  fi
352
353  \tree -f $@
354
355  IGCM_debug_PopStack "IGCM_sys_Tree"
356}
357
358#D-#==================================================
359#D-function IGCM_sys_Qsub
360#D-* Purpose: Qsub new job
361#D-* Examples:
362#D-
363function IGCM_sys_Qsub {
364  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
365  if ( $DEBUG_sys ) ; then
366    echo "IGCM_sys_Qsub :" $@
367  fi
368  typeset options status
369  options="-o ${SUBMIT_DIR}/${Script_Output}"
370  /usr/bin/qsub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
371  status=$?
372
373  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
374  if [ ${status} -gt 0 ] ; then
375    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
376    IGCM_debug_Exit "IGCM_sys_Qsub"
377  else
378    JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
379  fi
380  IGCM_debug_PopStack "IGCM_sys_Qsub"
381}
382
383#D-#==================================================
384#D-function IGCM_sys_QsubPost
385#D-* Purpose: Qsub new job on scalaire
386#D-* Examples:
387#D-
388function IGCM_sys_QsubPost {
389  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
390  if ( $DEBUG_sys ) ; then
391    echo "IGCM_sys_QsubPost :" $@
392  fi
393  typeset options status
394  options="-o ${POST_DIR}/${Script_Post_Output}.out -v ${listVarEnv}"
395  /usr/bin/qsub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
396  status=$?
397
398  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
399  if [ ${status} -gt 0 ] ; then
400    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
401    IGCM_debug_Exit "IGCM_sys_QsubPost"
402  else
403    JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
404  fi
405  IGCM_debug_PopStack "IGCM_sys_QsubPost"
406}
407
408#D-*************************
409#D- File transfer functions
410#D-*************************
411#D-
412
413#D-#==================================================
414#D-function IGCM_sys_RmRunDir
415#D-* Purpose: rm tmpdir (dummy function most of the time batch
416#D-                      scheduler will do the job)
417#D-* Examples:
418#D-
419function IGCM_sys_RmRunDir {
420  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
421  if ( $DEBUG_sys ) ; then
422    echo "IGCM_sys_RmRunDir :" $@
423  fi
424
425  typeset status
426
427  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
428  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
429  status=$?
430
431  if [ ${status} -gt 0 ] ; then
432    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
433    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
434    IGCM_debug_Exit "IGCM_sys_RmRunDir"
435  else
436    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
437  fi
438  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
439}
440
441#D-#==================================================
442#D-function IGCM_sys_Put_Dir
443#D-* Purpose: Copy a complete directory on $(ARCHIVE)
444#D-* Examples:
445#D-
446function IGCM_sys_Put_Dir {
447  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
448  if ( $DEBUG_sys ) ; then
449    echo "IGCM_sys_Put_Dir :" $@
450  fi
451  if [ $DRYRUN = 0 ]; then
452    if [ ! -d ${1} ] ; then
453      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
454      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
455      return
456    fi
457
458    typeset status
459
460    # Only if we use rsync
461    #IGCM_sys_TestDirArchive $( dirname $2 )
462    #
463    #USUAL WAY
464    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
465    status=$?
466
467    if [ ${status} -gt 0 ] ; then
468      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
469      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
470      IGCM_debug_Exit "IGCM_sys_Put_Dir"
471    else
472      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
473    fi
474  fi
475  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
476}
477
478#D-#==================================================
479#D-function IGCM_sys_Get_Dir
480#D-* Purpose: Copy a complete directory from ${ARCHIVE}
481#D-* Examples:
482#D-
483function IGCM_sys_Get_Dir {
484  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
485  if ( $DEBUG_sys ) ; then
486    echo "IGCM_sys_Get_Dir :" $@
487  fi
488  if [ $DRYRUN = 0 ]; then
489    typeset NB_ESSAI DELAI status i
490    # number of tentative
491    NB_ESSAI=3
492    # time delay between tentative
493    DELAI=2
494
495    #
496    # USUAL WAY
497    i=0
498    while [ $i -lt $NB_ESSAI ] ; do
499      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
500      status=$?
501      if [ ${status} -gt 0 ] ; then
502        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
503        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
504        sleep $DELAI
505      else
506        break
507      fi
508      (( i = i + 1 ))
509    done
510
511    if [ ${status} -gt 0 ] ; then
512      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
513      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514      IGCM_debug_Exit "IGCM_sys_Get_Dir"
515    else
516      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
517    fi
518  fi
519  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
520}
521
522#D-#==================================================
523#D-function IGCM_sys_Put_Rest
524#D-* Purpose: Put computied restarts on ${ARCHIVE}.
525#D-           File and target directory must exist.
526#D-* Examples:
527#D-
528function IGCM_sys_Put_Rest {
529  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
530  if ( $DEBUG_sys ) ; then
531    echo "IGCM_sys_Put_Rest :" $@
532  fi
533  if [ $DRYRUN = 0 ]; then
534    if [ ! -f ${1} ] ; then
535      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
536      IGCM_debug_Exit "IGCM_sys_Put_Rest"
537    fi
538
539    typeset status
540    #
541    # USUAL WAY
542    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
543    status=$?
544
545#       #RSYNC WITH NETWORK SSH CALL
546#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
547#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
548
549#       #RSYNC WITH NFS USE
550#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
551#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
552
553#       status=$?
554#       IGCM_sys_Rsync_out $status
555
556#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
557#       (( status=status+$? ))
558
559    if [ ${status} -gt 0 ] ; then
560      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
561      [ -f ${1} ] && ls -l ${1}
562      [ -f ${2} ] && ls -l ${2}
563      [ -f ${2}/${1} ] && ls -l ${2}/${1}
564      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
565      IGCM_debug_Exit "IGCM_sys_Put_Rest"
566    else
567
568      if [ X${JobType} = XRUN ] ; then
569        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
570        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
571      fi
572
573      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
574    fi
575  fi
576  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
577}
578
579#D-#==================================================
580#D-function IGCM_sys_Put_Out
581#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
582#D-* Examples:
583#D-
584function IGCM_sys_Put_Out {
585  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
586  if ( $DEBUG_sys ) ; then
587    echo "IGCM_sys_Put_Out :" $@
588  fi
589
590  typeset status
591
592  if [ $DRYRUN = 0 ]; then
593    if [ ! -f ${1} ] ; then
594      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
595      IGCM_debug_PopStack "IGCM_sys_Put_Out"
596      return 1
597    fi
598    #
599    IGCM_sys_MkdirArchive $( dirname $2 )
600    #
601    if [ X${JobType} = XRUN ] ; then
602      if [ X${3} = X ] ; then
603        IGCM_sys_Chmod 444 ${1}
604      fi
605    fi
606    #
607
608    echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
609    ${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
610    status=$?
611    IGCM_sys_Rsync_out $status
612
613    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
614    (( status=status+$? ))
615
616    if [ ${status} -gt 0 ] ; then
617      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
618      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
619      IGCM_debug_Exit "IGCM_sys_Put_Out"
620    else
621      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
622    fi
623  fi
624  IGCM_debug_PopStack "IGCM_sys_Put_Out"
625  return 0
626}
627
628#D-#==================================================
629#D-function IGCM_sys_Get
630#D-* Purpose: Get a file from ${ARCHIVE}
631#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
632#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
633function IGCM_sys_Get {
634  IGCM_debug_PushStack "IGCM_sys_Get" $@
635
636  typeset DEST status dm_liste
637
638  if ( $DEBUG_sys ) ; then
639    echo "IGCM_sys_Get :" $@
640  fi
641  if [ $DRYRUN -le 2 ]; then
642    if [ X${1} = X'/l' ] ; then
643      eval set +A dm_liste \${${2}}
644    else
645      dm_liste=${1}
646    fi
647    eval DEST=\${${#}}
648
649    # test if the (first) file is present in the old computation :
650    IGCM_sys_TestFileArchive ${dm_liste[0]}
651    status=$?
652    if [ ${status} -gt 0 ] ; then
653      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
654      IGCM_debug_Exit "IGCM_sys_Get"
655      return
656    fi
657
658    #USUAL WAY
659    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
660    status=$?
661
662    if [ ${status} -gt 0 ] ; then
663      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
664      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
665      IGCM_debug_Exit "IGCM_sys_Get"
666    else
667      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
668    fi
669  fi
670  IGCM_debug_PopStack "IGCM_sys_Get"
671}
672
673#D-#==================================================
674#D-function IGCM_sys_GetDate_Monitoring
675#D-* Purpose: get the last year for which the monitoring has been computed
676#D-* Examples:
677#D-
678function IGCM_sys_GetDate_Monitoring {
679  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
680  if ( $DEBUG_sys ) ; then
681    echo "IGCM_sys_GetDate_Monitoring :" $@
682  fi
683
684  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
685
686  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
687}
688
689#D-#==================================================
690#D-function IGCM_sys_Dods_Rm
691#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
692#D-* Examples:
693#D-
694function IGCM_sys_Dods_Rm {
695  if ( $DEBUG_sys ) ; then
696    echo "IGCM_sys_Dods_Rm :" $@
697  fi
698  return 0
699}
700
701#D-#==================================================
702#D-function IGCM_sys_Dods_Cp
703#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
704#D-* Examples:
705#D-
706function IGCM_sys_Dods_Cp {
707  if ( $DEBUG_sys ) ; then
708    echo "IGCM_sys_Dods_Cp :" $@
709  fi
710  return 0
711}
712
713#D-#==================================================
714#D-function IGCM_sys_Put_Dods
715#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
716#D-* Examples:
717#D-
718function IGCM_sys_Put_Dods {
719  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
720  if ( $DEBUG_sys ) ; then
721    echo "IGCM_sys_Put_Dods :" $@
722  fi
723  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
724}
725
726##############################################################
727# REBUILD OPERATOR
728
729#D-#==================================================
730#D-function IGCM_sys_sync
731#D-* Purpose: flush buffer on disk (dummy function on Ada)
732#D-* Examples:
733#D-
734function IGCM_sys_sync {
735  IGCM_debug_PushStack "IGCM_sys_sync" $@
736  if ( $DEBUG_sys ) ; then
737    echo "IGCM_sys_sync :" $@
738    echo "Dummy call, let the system do that."
739  fi
740  IGCM_debug_PopStack "IGCM_sys_sync"
741}
742
743############################################################
744# Activate Running Environnment Variables
745
746#D-#==================================================
747#D-function IGCM_sys_desactiv_variables
748#D-* Purpose: set environement variables prior to execution
749#D-* Examples:
750#D-
751function IGCM_sys_activ_variables {
752  IGCM_debug_PushStack "IGCM_sys_activ_variables"
753  if ( $DEBUG_sys ) ; then
754    echo "IGCM_sys_activ_variables"
755  fi
756
757# --------------------------------------------------------------------
758#D- MPI specifications
759# --------------------------------------------------------------------
760
761# --------------------------------------------------------------------
762#D- Other specifications
763# --------------------------------------------------------------------
764
765  ulimit -s unlimited
766
767  IGCM_debug_PopStack "IGCM_sys_activ_variables"
768}
769
770############################################################
771# Desactivate Running Environnment Variables
772
773#D-#==================================================
774#D-function IGCM_sys_desactiv_variables
775#D-* Purpose: unset environement variables after execution
776#D-* Examples:
777#D-
778function IGCM_sys_desactiv_variables {
779  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
780  if ( $DEBUG_sys ) ; then
781    echo "IGCM_sys_desactiv_variables"
782  fi
783# --------------------------------------------------------------------
784#D- MPI specifications
785# --------------------------------------------------------------------
786
787# --------------------------------------------------------------------
788#D- Other specifications
789# --------------------------------------------------------------------
790
791  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
792}
793
794############################################################
795# Build MPI/OMP scripts run file (dummy function)
796
797#D-#==================================================
798#D-function IGCM_sys_build_run_file
799#D-* Purpose: build run file (deprecated)
800#D-* Examples:
801#D-
802function IGCM_sys_build_run_file {
803
804  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
805
806}
807
808############################################################
809# Build MPI/OMP scripts
810
811#D-#==================================================
812#D-function IGCM_sys_build_execution_scripts
813#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
814#D-* Examples:
815#D-
816function IGCM_sys_build_execution_scripts
817{
818  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
819  if ( $DEBUG_sys ) ; then
820    echo "IGCM_sys_build_execution_scripts " $@
821  fi
822
823  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
824  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
825  typeset num_corempi nombre_restant_node nombre_restant_comp
826
827  if ( ${OK_PARA_MPMD} ) ; then
828
829    if [ -f run_file ] ; then
830      IGCM_sys_Rm -f run_file
831    fi
832    touch run_file
833
834    if ( ${OK_PARA_OMP} ) ; then
835
836      #  Hosts treatment
837
838      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
839
840      i=0
841      rm -f hosts
842      IGCM_debug_Print 1 "sys Ciclad, Hosts avaible :"
843      for nodes in `cat hosts.tmp` ; do
844        host[$i]=$nodes
845        echo "${host[$i]} slots=1 max_slots=1" >> hosts
846        IGCM_debug_Print 1 ${host[$i]}
847        i=$((i+1))
848      done
849      rm -f hosts.tmp
850
851      listnodes=${host[*]}
852
853      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
854
855      # Initialisation
856
857      init_node=y
858      node_num_current=0
859      start_num=0
860      init_exec=n
861
862      # Build run_file
863
864      # First loop on the components for the coupler ie oasis (only if oasis3)
865      # the coupler ie oasis3 must be the first one
866      for comp in ${config_ListOfComponents[*]} ; do
867
868        if [ "X${comp}" = "XCPL" ]  ; then
869
870          eval ExeNameIn=\${config_Executable_${comp}[0]}
871          eval ExeNameOut=\${config_Executable_${comp}[1]}
872
873          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
874          echo ""  >> script_${ExeNameOut}.ksh
875          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
876          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
877          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
878          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
879          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
880
881          init_node=n
882
883          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
884          node_num_current=0
885          node_current=${host[${node_num_current}]}
886
887          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
888
889          init_exec=y
890          start_num=1
891
892        fi
893
894      done
895
896      # Then loop on the components (except for oasis)
897
898      for comp in ${config_ListOfComponents[*]} ; do
899
900        eval ExeNameIn=\${config_Executable_${comp}[0]}
901        eval ExeNameOut=\${config_Executable_${comp}[1]}
902
903        # Only if we really have an executable for the component :
904        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
905
906          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
907          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
908
909          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
910          # echo "set -vx" >> script_${ExeNameOut}.ksh
911          echo ""  >> script_${ExeNameOut}.ksh
912          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
913          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
914          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
915          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
916          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
917          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
918          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
919
920          node_num=0
921
922          # We define the number of MPI process to be assigned for the component
923
924          nombre_restant_comp=${comp_proc_mpi_loc}
925
926          # Loop on the allocated nodes
927
928          for node in ${listnodes} ; do
929
930            # We go to the current node
931            if [ ${node_num} = ${node_num_current} ] ; then
932
933              node_current=${host[${node_num_current}]}
934
935              # If first time on the node : initialisation
936
937              if [ ${init_node} = y ] ; then
938                nombre_restant_node=${NUM_COREPERNODE}
939              fi
940
941              # Test on the number of OMP threads
942
943              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
944                (( node_num = node_num + 1 ))
945                node_num_current=${node_num}
946                init_node=y
947                continue
948              fi
949
950              # Number of MPI process to assign
951
952              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
953
954              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
955                num_corempi=${nombre_restant_comp}
956              fi
957
958              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
959              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
960
961              if [ ${init_exec} = y ] ; then
962                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
963              else
964                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
965                init_exec=y
966              fi
967
968              ((  start_num = num_corempi + start_num ))
969
970            else
971
972              (( node_num = node_num + 1 ))
973              continue
974            fi
975
976            # Test on the number of core/process remaining on the node/component
977
978            if [ ${nombre_restant_node} = 0 ] ; then
979              (( node_num = node_num + 1 ))
980              node_num_current=${node_num}
981              init_node=y
982
983              if [ ${nombre_restant_comp} = 0 ] ; then
984                break 1
985              fi
986            else
987
988              node_num_current=${node_num}
989              init_node=n
990
991              if [ ${nombre_restant_comp} = 0 ] ; then
992                break 1
993              fi
994            fi
995          done
996        fi
997      done
998
999    else
1000
1001      # Then first loop on the components for the coupler ie oasis
1002
1003      ## the coupler ie oasis must be the first one
1004      for comp in ${config_ListOfComponents[*]} ; do
1005
1006        eval ExeNameOut=\${config_Executable_${comp}[1]}
1007
1008        # for CPL component only
1009        if [ "X${comp}" = "XCPL" ] ; then
1010          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1011          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1012        fi
1013      done
1014
1015      # Then second loop on the components
1016
1017      for comp in ${config_ListOfComponents[*]} ; do
1018
1019        eval ExeNameOut=\${config_Executable_${comp}[1]}
1020
1021        # Only if we really have an executable for the component and not the coupler ie oasis:
1022        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1023          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1024          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1025        fi
1026      done
1027      IGCM_sys_Chmod u+x run_file
1028
1029      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1030
1031    fi
1032
1033  else # Only one executable. launch it.
1034
1035    for comp in ${config_ListOfComponents[*]} ; do
1036
1037      # Only if we really have an executable for the component :
1038      eval ExeNameOut=\${config_Executable_${comp}[1]}
1039      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1040
1041        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1042        echo ""  >> script_${ExeNameOut}.ksh
1043        if ( ${OK_PARA_OMP} ) ; then
1044          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1045          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1046        fi
1047        if  ( ${OK_PARA_MPI} ) ; then
1048          # Default : mpirun used if nb_proc gt 1
1049          # pour sortie out/err par process
1050          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
1051          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1052          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1053          EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1054        else
1055          # Default : mpirun is NOT used if nb_proc eq 1
1056          # pour sortie out/err par process
1057          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1058          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1059          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1060          EXECUTION="time ./script_${ExeNameOut}.ksh"
1061        fi
1062
1063        IGCM_debug_Print 1 "sys Ciclad : script_${ExeNameOut}.ksh contains"
1064        cat script_${ExeNameOut}.ksh
1065
1066      fi
1067
1068    done
1069
1070  fi
1071
1072  IGCM_debug_Print 1 "sys Ciclad : execution command is"
1073  IGCM_debug_Print 1 "$EXECUTION"
1074
1075  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1076}
1077
1078#D-#==================================================
1079#D-function IGCM_sys_check_path
1080#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1081#D-* do not point to an important use directory. Stop immediately in that case.
1082#D-* Examples:
1083#D-
1084function IGCM_sys_check_path {
1085  IGCM_debug_PushStack "IGCM_sys_check_path"
1086  if ( $DEBUG_sys ) ; then
1087    echo "IGCM_sys_check_path"
1088  fi
1089
1090  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1091    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1092    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1093    IGCM_debug_Exit "This will stop the job"
1094  fi
1095  IGCM_debug_PopStack "IGCM_sys_check_path"
1096}
1097
1098#D-#==================================================
1099#D-function IGCM_sys_check_quota. Dummy call here
1100#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1101#D-* Examples:
1102#D-
1103function IGCM_sys_check_quota {
1104  IGCM_debug_PushStack "IGCM_sys_check_quota"
1105  if ( $DEBUG_sys ) ; then
1106    echo "IGCM_sys_check_quota"
1107  fi
1108
1109  # TO BE IMPLEMENTED
1110
1111  IGCM_debug_PopStack "IGCM_sys_check_quota"
1112}
1113
1114#D-#==================================================
1115#D-function IGCM_sys_GetJobID
1116#D-* Purpose: Check if job_name is currently
1117#D-  running or in queue
1118#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1119#D-
1120function IGCM_sys_GetJobID {
1121  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1122  if ( $DEBUG_sys ) ; then
1123    echo "IGCM_sys_GetJobID"
1124  fi
1125
1126  # With -f option, the full job name is given in the last column
1127  ID="$( qstat -u $2 | grep -w $1 | gawk '-F ' '{print $10}' )"
1128
1129  eval ${3}=${ID}
1130  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1131}
1132
1133#D-#==================================================
1134#D-function IGCM_sys_CountJobInQueue
1135#D-* Purpose: Check if job_name is currently
1136#D-  running or in queue
1137#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1138#D-
1139function IGCM_sys_CountJobInQueue {
1140  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1141  if ( $DEBUG_sys ) ; then
1142    echo "IGCM_sys_CountJobInQueue"
1143  fi
1144  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1145}
Note: See TracBrowser for help on using the repository browser.