source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1208

Last change on this file since 1208 was 1208, checked in by sdipsl, 9 years ago
  • Print list of loaded modules
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 51.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Curie
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=curie
71# add default project on curie
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
96# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
97# uncomment 2 lines :
98#  module unload netcdf
99#  module load netcdf/3.6.3
100else
101  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
102  PCMDI_MP=/ccc/work/cont003/dsm/p86ipsl/PCMDI-MP
103fi
104IGCM_debug_Print 1 "List of loaded modules:"
105module list
106
107export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
108export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
109export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
110
111#====================================================
112# Specific for ocean additionnal diagnostic
113export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
114export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
115
116#====================================================
117# Host specific DIRECTORIES
118#====================================================
119
120# ============ CESIUM START ============ #
121
122#====================================================
123#- Mirror libIGCM from titane to cesium if needed
124#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
125#if [ ! ${ROOTSYS} = "home" ] ; then
126#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
127#else
128#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
129#fi
130
131#====================================================
132#- libIGCM_POST
133#if ( ${MirrorlibIGCM} ) ; then
134#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
135#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
136#else
137#  typeset -r libIGCM_POST=${libIGCM}
138#fi
139
140# ============ CESIUM  END  ============ #
141
142#====================================================
143#- MirrorlibIGCM for frontend
144typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
145
146#====================================================
147#- libIGCM_POST for frontend
148typeset -r libIGCM_POST=${libIGCM}
149
150#====================================================
151#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
152typeset -r R_EXE="${MODIPSL}/bin"
153
154#====================================================
155#- SUBMIT_DIR : submission dir
156typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
157
158#====================================================
159#- IN
160typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
161typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
162
163#====================================================
164#- RUN_DIR_PATH : Temporary working directory (=> TMP)
165typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
166
167#====================================================
168#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
169typeset -r OUTCOMMAND_PATH=/tmp
170
171#====================================================
172#- HOST_MPIRUN_COMMAND
173typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
174
175#====================================================
176#- Max number of arguments passed to nco operator or demigration command
177UNIX_MAX_LIMIT=360
178
179#====================================================
180#- set PackDefault to true on curie
181PackDefault=true
182
183#====================================================
184#- Number of core per node (max number of OpenMP task)
185NUM_COREPERNODE=8
186
187#====================================================
188#- Default number of MPI task for IPSL coupled model
189#- required for backward compatibility
190#-
191DEFAULT_NUM_PROC_OCE=5
192DEFAULT_NUM_PROC_CPL=1
193(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
194DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
195
196#D-#==================================================
197#D-function IGCM_sys_defineArchives
198#D-* Purpose:
199#D-* Define ARCHIVE : Dedicated to large files
200#D-* Define STORAGE : Dedicated to small/medium files
201#D-* Define R_OUT   : Output tree located on ARCHIVE
202#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
203#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
204#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
205#D-* Examples:
206#D-
207function IGCM_sys_defineArchives {
208  IGCM_debug_PushStack "IGCM_sys_defineArchives"
209
210  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
211    #====================================================
212    #- ARCHIVE (dedicated to large files)
213    ARCHIVE=${config_UserChoices_ARCHIVE}
214  else
215    #====================================================
216    #- ARCHIVE (dedicated to large files)
217    ARCHIVE=${CCCSTOREDIR}
218  fi
219
220  if [ ! X${config_UserChoices_STORAGE} = X ]; then
221    #====================================================
222    #- STORAGE (dedicated to small/medium files)
223    STORAGE=${config_UserChoices_STORAGE}
224  else
225    #====================================================
226    #- STORAGE (dedicated to small/medium files)
227    STORAGE=${CCCWORKDIR}
228  fi
229
230  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
231    #====================================================
232    #- R_OUT
233    R_OUT=${SCRATCHDIR}/IGCM_OUT
234
235    #====================================================
236    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
237    R_FIG=${SCRATCHDIR}/IGCM_OUT
238
239    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
240
241  else
242    #====================================================
243    #- R_OUT
244    R_OUT=${ARCHIVE}/IGCM_OUT
245
246    #====================================================
247    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
248    R_FIG=${STORAGE}/IGCM_OUT
249  fi
250
251  #====================================================
252  #- R_BUF (ONLY FOR double copy an scratch)
253  R_BUF=${SCRATCHDIR}/IGCM_OUT
254
255  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
256  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
257  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
258
259  IGCM_debug_PopStack "IGCM_sys_defineArchives"
260}
261
262#D-#==================================================
263#D-function IGCM_sys_RshArchive
264#D-* Purpose: Archive rsh command
265#D-* Examples:
266#D-
267function IGCM_sys_RshArchive {
268  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
269  /bin/ksh <<-EOF
270    ${@}
271EOF
272  status=$?
273  if [ ${status} -gt 0 ] ; then
274    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
275    IGCM_debug_Exit "IGCM_sys_RshArchive"
276  fi
277  IGCM_debug_PopStack "IGCM_sys_RshArchive"
278}
279
280#D-#==================================================
281#D-function IGCM_sys_RshArchive_NoError
282#D-* Purpose: Archive rsh command, without error
283#D-*          used only in monitoring.job
284#D-* Examples:
285#D-
286function IGCM_sys_RshArchive_NoError {
287  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
288  /bin/ksh <<-EOF
289    ${@} 2> /dev/null
290EOF
291  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
292}
293
294#D-#==================================================
295#D-function IGCM_sys_MkdirArchive
296#D-* Purpose: Mkdir on Archive
297#D-* Examples:
298#D-
299function IGCM_sys_MkdirArchive {
300  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
301  if ( $DEBUG_sys ) ; then
302    echo "IGCM_sys_MkdirArchive :" $@
303  fi
304  #- creation de repertoire sur le serveur fichier
305  if [ ! -d ${1} ]; then
306    \mkdir -p $1
307    status=$?
308
309    if [ ${status} -gt 0 ] ; then
310      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
311      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
312    fi
313  fi
314  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
315}
316
317#D-#==================================================
318#D-function IGCM_sys_TestDirArchive
319#D-* Purpose: Test Directory that must exists on Archive
320#D-* Examples:
321#D-
322function IGCM_sys_TestDirArchive {
323  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
324  if ( $DEBUG_sys ) ; then
325    echo "IGCM_sys_TestDirArchive :" $@
326  fi
327  typeset ExistFlag
328  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
329  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
330  return ${ExistFlag}
331}
332
333#D-#==================================================
334#D-function IGCM_sys_IsFileArchived
335#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
336#D-* Examples:
337#D-
338function IGCM_sys_IsFileArchived {
339  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
340  if ( $DEBUG_sys ) ; then
341    echo "IGCM_sys_IsFileArchived :" $@
342  fi
343  typeset IsArchivedFlag
344  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
345  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
346
347  return ${IsArchivedFlag}
348}
349
350#D-#==================================================
351#D-function IGCM_sys_TestFileArchive
352#D-* Purpose: Test file that must NOT EXISTS on Archive
353#D-* Examples:
354#D-
355function IGCM_sys_TestFileArchive {
356  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
357  typeset ExistFlag
358  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
359  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
360
361  return ${ExistFlag}
362}
363
364#D-#==================================================
365#D-function IGCM_sys_CountFileArchive
366#D-* Purpose: Count files on Archive filesystem
367#D-* Examples:
368#D-
369function IGCM_sys_CountFileArchive {
370  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
371  ls ${@} 2>/dev/null | wc -l
372  if [ $? -gt 0 ] ; then
373    echo "IGCM_sys_CountFileArchive : erreur."
374  fi
375  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
376}
377
378#D-#==================================================
379#D-function IGCM_sys_Tree
380#D-* Purpose: Tree directories with files on ${ARCHIVE}
381#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
382#D-
383function IGCM_sys_Tree {
384  IGCM_debug_PushStack "IGCM_sys_Tree" $@
385  if ( $DEBUG_sys ) ; then
386    echo "IGCM_sys_Tree :" $@
387  fi
388
389  \ls -lR ${@}
390
391  IGCM_debug_PopStack "IGCM_sys_Tree"
392}
393
394#D-#==================================================
395#D-function IGCM_sys_Qsub
396#D-* Purpose: Qsub new job
397#D-* Examples:
398#D-
399function IGCM_sys_Qsub {
400  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
401  if ( $DEBUG_sys ) ; then
402    echo "IGCM_sys_Qsub :" $@
403  fi
404  typeset options status
405  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
406
407  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
408  status=$?
409
410  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
411  if [ ${status} -gt 0 ] ; then
412    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
413    IGCM_debug_Exit "IGCM_sys_Qsub"
414  else
415    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
416  fi
417  IGCM_debug_PopStack "IGCM_sys_Qsub"
418}
419
420#D-#==================================================
421#D-function IGCM_sys_QsubPost
422#D-* Purpose: Qsub new job on scalaire
423#D-* Examples:
424#D-
425function IGCM_sys_QsubPost {
426  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
427  if ( $DEBUG_sys ) ; then
428    echo "IGCM_sys_QsubPost :" $@
429  fi
430  typeset options status
431
432  # EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
433  case $( hostname -s ) in
434  curie*)
435    options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
436  airain*)
437    options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
438  esac
439
440  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
441  status=$?
442
443  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
444  if [ ${status} -gt 0 ] ; then
445    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
446    IGCM_debug_Exit "IGCM_sys_QsubPost"
447  else
448    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
449  fi
450  IGCM_debug_PopStack "IGCM_sys_QsubPost"
451}
452
453#D-*************************
454#D- File transfer functions
455#D-*************************
456#D-
457
458#D-#==================================================
459#D-function IGCM_sys_RmRunDir
460#D-* Purpose: rm tmpdir (dummy function most of the time batch
461#D-                      scheduler will do the job)
462#D-* Examples:
463#D-
464function IGCM_sys_RmRunDir {
465  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
466  if ( $DEBUG_sys ) ; then
467    echo "IGCM_sys_RmRunDir :" $@
468  fi
469
470  typeset status
471
472  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
473  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
474  status=$?
475
476  if [ ${status} -gt 0 ] ; then
477    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
478    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
479    IGCM_debug_Exit "IGCM_sys_RmRunDir"
480  else
481    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
482  fi
483  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
484}
485
486#D-#==================================================
487#D-function IGCM_sys_Put_Dir
488#D-* Purpose: Copy a complete directory on $(ARCHIVE)
489#D-* Examples:
490#D-
491function IGCM_sys_Put_Dir {
492  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
493  if ( $DEBUG_sys ) ; then
494    echo "IGCM_sys_Put_Dir :" $@
495  fi
496  if [ $DRYRUN = 0 ]; then
497    if [ ! -d ${1} ] ; then
498      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
499      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
500      return
501    fi
502
503    typeset status
504
505    # Only if we use rsync
506    #IGCM_sys_TestDirArchive $( dirname $2 )
507    #
508    #USUAL WAY
509    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
510    status=$?
511
512    if [ ${status} -gt 0 ] ; then
513      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
514      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
515      IGCM_debug_Exit "IGCM_sys_Put_Dir"
516    else
517      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
518    fi
519  fi
520  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
521}
522
523#D-#==================================================
524#D-function IGCM_sys_Get_Dir
525#D-* Purpose: Copy a complete directory from ${ARCHIVE}
526#D-* Examples:
527#D-
528function IGCM_sys_Get_Dir {
529  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
530  if ( $DEBUG_sys ) ; then
531    echo "IGCM_sys_Get_Dir :" $@
532  fi
533  if [ $DRYRUN = 0 ]; then
534    typeset NB_ESSAI DELAI status i
535    # number of tentative
536    NB_ESSAI=3
537    # time delay between tentative
538    DELAI=2
539
540    #
541    # USUAL WAY
542    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
543    ccc_hsm get -r $1
544
545    i=0
546    while [ $i -lt $NB_ESSAI ] ; do
547      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
548      status=$?
549      if [ ${status} -gt 0 ] ; then
550        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
551        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
552        sleep $DELAI
553      else
554        break
555      fi
556      (( i = i + 1 ))
557    done
558
559    if [ ${status} -gt 0 ] ; then
560      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
561      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
562      IGCM_debug_Exit "IGCM_sys_Get_Dir"
563    else
564      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
565    fi
566  fi
567  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
568}
569
570#D-#==================================================
571#D-function IGCM_sys_Put_Rest
572#D-* Purpose: Put computied restarts on ${ARCHIVE}.
573#D-           File and target directory must exist.
574#D-* Examples:
575#D-
576function IGCM_sys_Put_Rest {
577  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
578  if ( $DEBUG_sys ) ; then
579    echo "IGCM_sys_Put_Rest :" $@
580  fi
581  if [ $DRYRUN = 0 ]; then
582    if [ ! -f ${1} ] ; then
583      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
584      IGCM_debug_Exit "IGCM_sys_Put_Rest"
585    fi
586
587    typeset status
588    #
589    # USUAL WAY
590    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
591    status=$?
592
593#       #RSYNC WITH NETWORK SSH CALL
594#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
595#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
596
597#       #RSYNC WITH NFS USE
598#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
599#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
600
601#       status=$?
602#       IGCM_sys_Rsync_out $status
603
604#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
605#       (( status=status+$? ))
606
607    if [ ${status} -gt 0 ] ; then
608      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
609      [ -f ${1} ] && ls -l ${1}
610      [ -f ${2} ] && ls -l ${2}
611      [ -f ${2}/${1} ] && ls -l ${2}/${1}
612      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
613      IGCM_debug_Exit "IGCM_sys_Put_Rest"
614    else
615
616      if [ X${JobType} = XRUN ] ; then
617        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
618        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
619      fi
620
621      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
622    fi
623  fi
624  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
625}
626
627#D-#==================================================
628#D-function IGCM_sys_Put_Out
629#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
630#D-* Examples:
631#D-
632function IGCM_sys_Put_Out {
633  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
634  if ( $DEBUG_sys ) ; then
635    echo "IGCM_sys_Put_Out :" $@
636  fi
637
638  typeset NB_ESSAI DELAI status i exist skip
639
640  # number of tentative
641  NB_ESSAI=3
642  # time delay between tentative
643  DELAI=2
644
645  if [ $DRYRUN = 0 ]; then
646    if [ ! -f ${1} ] ; then
647      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
648      IGCM_debug_PopStack "IGCM_sys_Put_Out"
649      return 1
650    fi
651    #
652    IGCM_sys_MkdirArchive $( dirname $2 )
653    #
654    exist=false
655    skip=false
656    if [ -f $2 ] ; then
657      IGCM_debug_Print 1 "$2 already exist"
658      ccc_hsm get $2
659      exist=true
660      if [ "X$( diff $1 $2 )" = X ] ; then
661        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
662        skip=true
663      else
664        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
665        skip=false
666      fi
667    fi
668    #
669    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
670      IGCM_sys_Chmod u+w $2
671    fi
672
673    if [ X${skip} = Xfalse ] ; then
674      i=0
675      while [ $i -lt $NB_ESSAI ] ; do
676        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
677          # USUAL WAY
678          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
679          status=$?
680        else
681          # NOT SO USUAL WAY
682          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
683          status=$?
684        fi
685        if [ ${status} -gt 0 ]; then
686          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
687          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
688          [ -f ${1} ] && ls -l ${1}
689          [ -f ${2} ] && ls -l ${2}
690          [ -f ${2}/${1} ] && ls -l ${2}/${1}
691          sleep $DELAI
692        else
693          break
694        fi
695        (( i = i + 1 ))
696      done
697    fi
698
699#       #RSYNC WITH NETWORK SSH CALL
700#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
701#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
702
703#       #RSYNC WITH NFS USE
704#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
705#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
706
707#       status=$?
708#       IGCM_sys_Rsync_out $status
709
710#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
711#       (( status=status+$? ))
712
713    if [ ${status} -gt 0 ] ; then
714      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
715      [ -f ${1} ] && ls -l ${1}
716      [ -f ${2} ] && ls -l ${2}
717      [ -f ${2}/${1} ] && ls -l ${2}/${1}
718      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
719      IGCM_debug_Exit "IGCM_sys_Put_Out"
720    else
721
722      if [ X${JobType} = XRUN ] ; then
723        if [ X${3} = X ] ; then
724          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
725          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
726        fi
727      fi
728
729      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
730    fi
731  fi
732  IGCM_debug_PopStack "IGCM_sys_Put_Out"
733  return 0
734}
735
736#D-#==================================================
737#D-function IGCM_sys_Get
738#D-* Purpose: Get a file from ${ARCHIVE}
739#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
740#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
741function IGCM_sys_Get {
742  IGCM_debug_PushStack "IGCM_sys_Get" $@
743
744  typeset DEST dm_liste target file_work
745  typeset NB_ESSAI DELAI status i
746
747  if ( $DEBUG_sys ) ; then
748    echo "IGCM_sys_Get :" $@
749  fi
750
751  # number of tentative
752  NB_ESSAI=3
753  # time delay between tentative
754  DELAI=2
755
756  if [ $DRYRUN -le 2 ]; then
757    if [ X${1} = X'/l' ] ; then
758      eval set +A dm_liste \${${2}}
759    else
760      eval set +A dm_liste ${1}
761    fi
762    eval DEST=\${${#}}
763    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
764    status=$?
765    if [ ${status} -gt 0 ] ; then
766      echo "WARNING IGCM_sys_Get : error code ${status}"
767      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
768      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
769    fi
770
771    #if [ ${status} -gt 0 ] ; then
772    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
773    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
774    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
775    #    sleep 30
776    #    echo "We try another time"
777    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
778    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
779    #    status=$?
780    #    if [ ${status} -gt 0 ] ; then
781    #      echo "ERROR IGCM_sys_Get : again demigration error :"
782    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
783    #      IGCM_debug_Exit "IGCM_sys_Get"
784    #    fi
785    #  else
786    #    echo "ERROR IGCM_sys_Get : demigration error :"
787    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
788    #    IGCM_debug_Exit "IGCM_sys_Get"
789    #  fi
790    #fi
791
792    #   #RSYNC WITH NETWORK SSH CALL
793    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
794    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
795
796    #   #RSYNC WITH NFS USE
797    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
798    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
799
800    #   status=$?
801    #   IGCM_sys_Rsync_out $status
802
803    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
804    #   (( status=status+$? ))
805
806    #USUAL WAY
807    if [ X${1} = X'/l' ] ; then
808      for target in ${dm_liste[*]} ; do
809        local_file=$( basename ${target} )
810        # test if the target file is present before the loop
811        IGCM_sys_TestFileArchive ${target}
812        status=$?
813        if [ ${status} -gt 0 ] ; then
814          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
815          IGCM_debug_Exit "IGCM_sys_Get"
816        else
817          i=0
818          while [ $i -lt $NB_ESSAI ] ; do
819            #if [ X${DoLink} = Xtrue ] ; then
820            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
821            #  status=$?
822            #  else
823            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824            #  status=$?
825            #fi
826            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
827            status=$?
828            if [ ${status} -gt 0 ]; then
829              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
830              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
831              sleep $DELAI
832            else
833              break
834            fi
835            (( i = i + 1 ))
836          done
837          if [ ${status} -gt 0 ] ; then
838            echo "IGCM_sys_Get : error"
839            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
840            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
841            IGCM_debug_Exit "IGCM_sys_Get"
842          else
843            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
844          fi
845        fi
846      done
847    else
848      i=0
849      while [ $i -lt $NB_ESSAI ] ; do
850        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
851        status=$?
852        if [ ${status} -gt 0 ]; then
853          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
854          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
855          sleep $DELAI
856        else
857          break
858        fi
859        (( i = i + 1 ))
860      done
861      if [ ${status} -gt 0 ] ; then
862        echo "IGCM_sys_Get : error"
863        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
864        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
865        IGCM_debug_Exit "IGCM_sys_Get"
866      else
867        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
868      fi
869    fi
870  fi
871  IGCM_debug_PopStack "IGCM_sys_Get"
872}
873
874#D-#==================================================
875#D-function IGCM_sys_GetDate_Monitoring
876#D-* Purpose: get the last year for which the monitoring has been computed
877#D-* Examples:
878#D-
879function IGCM_sys_GetDate_Monitoring {
880  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
881  if ( $DEBUG_sys ) ; then
882    echo "IGCM_sys_GetDate_Monitoring :" $@
883  fi
884
885  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
886
887  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
888}
889
890#D-#==================================================
891#D-function IGCM_sys_Dods_Rm
892#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
893#D-* Examples:
894#D-
895function IGCM_sys_Dods_Rm {
896  if ( $DEBUG_sys ) ; then
897    echo "IGCM_sys_Dods_Rm :" $@
898  fi
899  typeset status
900  if [ $DRYRUN = 0 ]; then
901
902#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
903#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
904#      echo "Nothing has been done."
905#      return
906#    fi
907
908    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
909    status=$?
910
911#    if [ ${status} -gt 0 ] ; then
912#      echo "IGCM_sys_Dods_Rm : error."
913#      cat out_dods_rm
914#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
915#    else
916#      rm out_dods_rm
917#    fi
918
919  fi
920  return $status
921}
922
923#D-#==================================================
924#D-function IGCM_sys_Dods_Cp
925#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
926#D-* Examples:
927#D-
928function IGCM_sys_Dods_Cp {
929  if ( $DEBUG_sys ) ; then
930    echo "IGCM_sys_Dods_Cp :" $@
931  fi
932  typeset status
933  if [ $DRYRUN = 0 ]; then
934
935#    if [ ! -d ${R_SAVE}/${1} ] ; then
936#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
937#      echo "Nothing has been done."
938#      return
939#    fi
940
941    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
942    status=$?
943
944#       if [ ${status} -gt 0 ] ; then
945#           echo "IGCM_sys_Dods_Cp : error."
946#           cat out_dods_cp
947#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
948#       else
949#           rm out_dods_cp
950#       fi
951
952  fi
953  return $status
954}
955
956#D-#==================================================
957#D-function IGCM_sys_Put_Dods
958#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
959#D-* Examples:
960#D-
961function IGCM_sys_Put_Dods {
962  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
963  if ( $DEBUG_sys ) ; then
964    echo "IGCM_sys_Put_Dods :" $@
965  fi
966  typeset status
967  if [ $DRYRUN = 0 ]; then
968    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
969      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
970      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
971      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
972      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
973      return
974    fi
975
976    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
977      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
978      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
979      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
980      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
981      return
982    fi
983    #
984    if [ -d ${R_SAVE}/${1} ] ; then
985      cd ${R_SAVE}
986    elif [ -d ${R_FIGR}/${1} ] ; then
987      cd ${R_FIGR}
988    fi
989
990    IGCM_sys_Dods_Rm ${1}
991    IGCM_sys_Dods_Cp ${1}
992    status=0
993
994    if [ ${status} -gt 0 ] ; then
995      echo "IGCM_sys_Put_Dods : error."
996      IGCM_debug_Exit "IGCM_sys_Put_Dods"
997    fi
998  fi
999  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1000}
1001
1002##############################################################
1003# REBUILD OPERATOR
1004
1005#D-#==================================================
1006#D-function IGCM_sys_sync
1007#D-* Purpose: flush buffer on disk
1008#D-* Examples:
1009#D-
1010function IGCM_sys_sync {
1011  IGCM_debug_PushStack "IGCM_sys_sync" $@
1012  if ( $DEBUG_sys ) ; then
1013    echo "IGCM_sys_sync :" $@
1014  fi
1015
1016  /bin/sync
1017
1018  IGCM_debug_PopStack "IGCM_sys_sync"
1019}
1020
1021############################################################
1022# Activate Running Environnment Variables
1023
1024#D-#==================================================
1025#D-function IGCM_sys_desactiv_variables
1026#D-* Purpose: set environement variables prior to execution
1027#D-* Examples:
1028#D-
1029function IGCM_sys_activ_variables {
1030  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1031  if ( $DEBUG_sys ) ; then
1032    echo "IGCM_sys_activ_variables"
1033  fi
1034
1035# --------------------------------------------------------------------
1036#D- MPI specifications
1037# --------------------------------------------------------------------
1038
1039# --------------------------------------------------------------------
1040#D- Other specifications
1041# --------------------------------------------------------------------
1042
1043  ulimit -s unlimited
1044
1045  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1046}
1047
1048############################################################
1049# Desactivate Running Environnment Variables
1050
1051#D-#==================================================
1052#D-function IGCM_sys_desactiv_variables
1053#D-* Purpose: unset environement variables after execution
1054#D-* Examples:
1055#D-
1056function IGCM_sys_desactiv_variables {
1057  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1058  if ( $DEBUG_sys ) ; then
1059    echo "IGCM_sys_desactiv_variables"
1060  fi
1061# --------------------------------------------------------------------
1062#D- MPI specifications
1063# --------------------------------------------------------------------
1064
1065# --------------------------------------------------------------------
1066#D- Other specifications
1067# --------------------------------------------------------------------
1068
1069  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1070}
1071
1072############################################################
1073# Build MPI/OMP scripts run file (dummy function)
1074
1075#D-#==================================================
1076#D-function IGCM_sys_build_run_file
1077#D-* Purpose: build run file (deprecated)
1078#D-* Examples:
1079#D-
1080function IGCM_sys_build_run_file {
1081
1082  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1083
1084}
1085
1086############################################################
1087# Build MPI/OMP scripts
1088
1089#D-#==================================================
1090#D-function IGCM_sys_build_execution_scripts
1091#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1092#D-* Examples:
1093#D-
1094function IGCM_sys_build_execution_scripts
1095{
1096  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1097  if ( $DEBUG_sys ) ; then
1098    echo "IGCM_sys_build_execution_scripts " $@
1099  fi
1100
1101  EXECUTION=${HOST_MPIRUN_COMMAND}
1102
1103  # MPMD mode
1104  if ( ${OK_PARA_MPMD} ) ; then
1105
1106    # Only MPI (MPMD)
1107    if  ( ! ${OK_PARA_OMP} ) ; then
1108
1109      if [ -f run_file ] ; then
1110        IGCM_sys_Rm -f run_file
1111      fi
1112      touch run_file
1113
1114      # Build run_file
1115
1116      # First loop on the components for the coupler ie oasis (only if oasis3)
1117      # the coupler ie oasis3 must be the first one
1118      for comp in ${config_ListOfComponents[*]} ; do
1119
1120        eval ExeNameIn=\${config_Executable_${comp}[0]}
1121        eval ExeNameOut=\${config_Executable_${comp}[1]}
1122
1123        # for CPL component only
1124        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1125
1126          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1127          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1128          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1129        fi
1130      done
1131
1132      # Then second loop on the components
1133      for comp in ${config_ListOfComponents[*]} ; do
1134
1135        eval ExeNameIn=\${config_Executable_${comp}[0]}
1136        eval ExeNameOut=\${config_Executable_${comp}[1]}
1137
1138        # Only if we really have an executable for the component and not the coupler ie oasis:
1139        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1140
1141          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1142          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1143          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1144        fi
1145      done
1146
1147      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1148
1149      IGCM_sys_Chmod u+x run_file
1150      if ( $DEBUG_sys ) ; then
1151        echo "run_file contains : "
1152        cat run_file
1153      fi
1154
1155    # MPI-OpenMP (MPMD)
1156    else
1157
1158      # Use of mpirun instead of ccc_mprun
1159      EXECUTION="time mpirun"
1160
1161      #  Hosts treatment
1162      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1163
1164      i=0
1165      rm -f hosts rankfile
1166      IGCM_debug_Print 1 "sys Curie, Hosts available :"
1167      for nodes in `cat hosts.tmp`
1168      do
1169        host[$i]=$nodes
1170        echo "${host[$i]}" >> hosts
1171        IGCM_debug_Print 1 ${host[$i]}
1172        i=$((i+1))
1173      done
1174      rm -f hosts.tmp
1175
1176      listnodes=${host[*]}
1177
1178      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1179
1180      # Initialisation
1181      rank=0
1182      current_core=0
1183      core_per_node=16
1184      init_exec=n
1185
1186      # Loop on the components
1187      for comp in ${config_ListOfComponents[*]} ; do
1188
1189        eval ExeNameIn=\${config_Executable_${comp}[0]}
1190        eval ExeNameOut=\${config_Executable_${comp}[1]}
1191
1192        # Not possible if oasis has an executable (i.e old version of oasis3)
1193        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1194          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1195          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1196          IGCM_debug_Verif_Exit
1197        fi
1198
1199        # Only if we really have an executable for the component :
1200        if [ "X${ExeNameOut}" != X\"\" ] ; then
1201
1202          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1203          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1204
1205          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1206          echo ""  >> script_${ExeNameOut}.ksh
1207          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1208
1209            # Check if the number of threads is correct
1210            case ${comp_proc_omp_loc} in
1211            2|4|8|16)
1212              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1213              ;;
1214            *)
1215              IGCM_debug_Exit "ERROR with OMP parameters !"
1216              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1217              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1218              IGCM_debug_Verif_Exit
1219              ;;
1220            esac
1221            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1222            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1223            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1224            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1225          fi
1226          #echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1227          #echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1228          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1229          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1230
1231          if [ ${init_exec} = y ] ; then
1232            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1233          else
1234            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1235            init_exec=y
1236          fi
1237
1238# Build rankfile : method used to assign cores and nodes for the MPI process
1239# Ex :
1240#rank 0=curie5296 slot=0,1,2,3
1241#rank 1=curie5296 slot=4,5,6,7
1242# Example of final command :
1243# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1244# with script_lmdz.x.ksh :
1245# #!/bin/ksh
1246#export KMP_STACKSIZE=3g
1247#export KMP_LIBRARY=turnaround
1248#export MKL_SERIAL=YES
1249#OMP_NUM_THREADS=4
1250#./lmdz.x
1251#
1252
1253          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1254            (( index_host = current_core / core_per_node ))
1255            host_value=${host[${index_host}]}
1256            (( slot =  current_core % core_per_node ))
1257            virg=","
1258            string_final=""
1259            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1260              string=$index$virg
1261              string_final=$string_final$string
1262            done
1263            string_final=$( echo $string_final | sed "s/.$//" )
1264            echo "rank $rank=$host_value slot=$string_final" >> rankfile
1265            (( rank = rank + 1 ))
1266            (( current_core = current_core + comp_proc_omp_loc ))
1267          done
1268        fi
1269
1270      done
1271    fi
1272
1273  # Only one executable (SPMD mode).
1274  else
1275
1276    for comp in ${config_ListOfComponents[*]} ; do
1277
1278      # Only if we really have an executable for the component :
1279      eval ExeNameOut=\${config_Executable_${comp}[1]}
1280      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1281
1282        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1283        echo ""  >> script_${ExeNameOut}.ksh
1284        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1285
1286        if ( ${OK_PARA_OMP} ) ; then
1287          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1288          echo ""  >> script_${ExeNameOut}.ksh
1289          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1290          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1291          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1292          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1293        fi
1294
1295        if  ( ${OK_PARA_MPI} ) ; then
1296          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1297          # Default : ccc_mprun used if nb_proc gt 1
1298          # to have out/err per process on different files
1299          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1300          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1301          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1302        else
1303          # Default : ccc_mprun is NOT used if nb_proc eq 1
1304          # to have out/err per process on different files
1305          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1306          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1307          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1308        fi
1309
1310        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1311        cat script_${ExeNameOut}.ksh
1312
1313      fi
1314
1315    done
1316
1317  fi
1318
1319  IGCM_debug_Print 1 "sys Curie : execution command is "
1320  IGCM_debug_Print 1 "$EXECUTION"
1321
1322  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1323}
1324
1325#D-#==================================================
1326#D-function IGCM_sys_check_path
1327#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1328#D-* do not point to an important use directory. Stop immediately in that case.
1329#D-* Examples:
1330#D-
1331function IGCM_sys_check_path {
1332  IGCM_debug_PushStack "IGCM_sys_check_path"
1333  if ( $DEBUG_sys ) ; then
1334    echo "IGCM_sys_check_path"
1335  fi
1336
1337  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1338    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1339    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1340    IGCM_debug_Exit "This will stop the job"
1341  fi
1342  IGCM_debug_PopStack "IGCM_sys_check_path"
1343}
1344
1345#D-#==================================================
1346#D-function IGCM_sys_check_quota
1347#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1348#D-* Examples:
1349#D-
1350function IGCM_sys_check_quota {
1351  IGCM_debug_PushStack "IGCM_sys_check_quota"
1352  if ( $DEBUG_sys ) ; then
1353    echo "IGCM_sys_check_quota"
1354  fi
1355  # Limit of quota (in %)
1356  limit_quota=90
1357
1358  # Check of the volume
1359  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1360  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1361
1362  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1363
1364    unit_avail=${volume_avail: -1}
1365    unit_quota=${volume_quota: -1}
1366
1367    if [ "${unit_quota}" = "*" ] ; then
1368      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1369      IGCM_debug_Print 1 "More than 100% of your quota is used"
1370      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1371      IGCM_debug_Print 1 "You must have more than 10% available to run"
1372      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1373      IGCM_debug_Verif_Exit
1374    fi
1375
1376    temp_avail=${volume_avail%%${volume_avail: -1}*}
1377    temp_quota=${volume_quota%%${volume_quota: -1}*}
1378
1379    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1380
1381    # Convertion
1382      if [ ${volume_avail: -1} = "T" ] ; then
1383        (( temp_avail = temp_avail * 1000000000000 ))
1384      elif [ ${volume_avail: -1} = "G" ] ; then
1385        (( temp_avail = temp_avail * 1000000000 ))
1386      elif [ ${volume_avail: -1} = "M" ] ; then
1387        (( temp_avail = temp_avail * 1000000 ))
1388      elif [ ${volume_avail: -1} = "k" ] ; then
1389        (( temp_avail = temp_avail * 1000 ))
1390      else
1391        (( temp_avail = volume_avail ))
1392      fi
1393      if [ ${volume_quota: -1} = "T" ] ; then
1394        (( temp_quota = temp_quota * 1000000000000 ))
1395      elif [ ${volume_quota: -1} = "G" ] ; then
1396        (( temp_quota = temp_quota * 1000000000 ))
1397      elif [ ${volume_quota: -1} = "M" ] ; then
1398        (( temp_quota = temp_quota * 1000000 ))
1399      elif [ ${volume_quota: -1} = "k" ] ; then
1400        (( temp_quota = temp_quota * 1000 ))
1401      else
1402        (( temp_quota = volume_quota ))
1403      fi
1404    fi
1405
1406    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1407#    echo "volume ratio is " $quota_volume
1408
1409    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1410      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1411      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1412      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1413      IGCM_debug_Print 1 "You must have more than 10% available to run"
1414      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1415      IGCM_debug_Verif_Exit
1416    fi
1417
1418  fi
1419
1420# Check of the number of inodes
1421
1422  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1423  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1424
1425  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1426
1427    unit_avail=${inode_avail: -1}
1428    unit_quota=${inode_quota: -1}
1429
1430    if [ "${unit_quota}" = "*" ] ; then
1431      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1432      IGCM_debug_Print 1 "More than 100% of your quota is used"
1433      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1434      IGCM_debug_Print 1 "You must have more than 10% available to run"
1435      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1436      IGCM_debug_Verif_Exit
1437    fi
1438
1439    temp_avail=${inode_avail%%${inode_avail: -1}*}
1440    temp_quota=${inode_quota%%${inode_quota: -1}*}
1441
1442    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1443
1444    # Convertion
1445      if [ ${inode_avail: -1} = "T" ] ; then
1446        (( temp_avail = temp_avail * 1000000000000 ))
1447      elif [ ${inode_avail: -1} = "G" ] ; then
1448        (( temp_avail = temp_avail * 1000000000 ))
1449      elif [ ${inode_avail: -1} = "M" ] ; then
1450        (( temp_avail = temp_avail * 1000000 ))
1451      elif [ ${inode_avail: -1} = "k" ] ; then
1452        (( temp_avail = temp_avail * 1000 ))
1453      else
1454        (( temp_avail = inode_avail ))
1455      fi
1456
1457      if [ ${inode_quota: -1} = "T" ] ; then
1458        (( temp_quota = temp_quota * 1000000000000 ))
1459      elif [ ${inode_quota: -1} = "G" ] ; then
1460        (( temp_quota = temp_quota * 1000000000 ))
1461      elif [ ${inode_quota: -1} = "M" ] ; then
1462        (( temp_quota = temp_quota * 1000000 ))
1463      elif [ ${inode_quota: -1} = "k" ] ; then
1464        (( temp_quota = temp_quota * 1000 ))
1465      else
1466        (( temp_quota = inode_quota ))
1467      fi
1468    fi
1469    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1470#    echo "inode ratio is " $quota_inode
1471
1472    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1473      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1474      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1475      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1476      IGCM_debug_Print 1 "You must have more than 10% available to run"
1477      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1478      IGCM_debug_Verif_Exit
1479    fi
1480  fi
1481  IGCM_debug_PopStack "IGCM_sys_check_quota"
1482}
1483
1484#D-#==================================================
1485#D-function IGCM_sys_GetJobID
1486#D-* Purpose: Check if job_name is currently
1487#D-  running or in queue
1488#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1489#D-
1490function IGCM_sys_GetJobID {
1491  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1492  if ( $DEBUG_sys ) ; then
1493    echo "IGCM_sys_GetJobID"
1494  fi
1495
1496  # With -f option, the full job name is given in the last column
1497  ID=$( ccc_mstat -f -u $2 | \
1498        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1499
1500  eval ${3}=${ID}
1501  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1502}
1503
1504#D-#==================================================
1505#D-function IGCM_sys_CountJobInQueue
1506#D-* Purpose: Check if job_name is currently
1507#D-  running or in queue
1508#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1509#D-
1510function IGCM_sys_CountJobInQueue {
1511  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1512  if ( $DEBUG_sys ) ; then
1513    echo "IGCM_sys_CountJobInQueue"
1514  fi
1515
1516  # With -f option, the full job name is given in the last column
1517  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1518
1519  eval ${2}=${NbRun}
1520
1521  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1522}
1523
1524#D-#==================================================
1525#D-function IGCM_sys_ListJobInQueue
1526#D-* Purpose: Check if job_name is currently
1527#D-  running or in queue
1528#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1529#D-
1530function IGCM_sys_ListJobInQueue {
1531  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1532  if ( $DEBUG_sys ) ; then
1533    echo "IGCM_sys_ListJobInQueue"
1534  fi
1535
1536  # With -f option, the full job name is given in the last column
1537  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1538                                        '( $2  == User      && \
1539                                           $NF != /TS/      && \
1540                                           $NF !~ /PACK/    && \
1541                                           $NF !~ /REBUILD/ && \
1542                                           $NF !~ /pack/ )     \
1543                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1544
1545  eval set -A ${2} ${JobList[*]}
1546
1547  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1548}
Note: See TracBrowser for help on using the repository browser.