source: CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh @ 5899

Last change on this file since 5899 was 5899, checked in by aclsce, 3 years ago

Modified libIGCM and configuration to run XIOS ensembles.
File size: 59.5 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1534                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-06-17 13:32:52 +0200 (Wed, 17 Jun 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Jean-Zay
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=jeanzay
71# add default project set by ins_job
72typeset PROJECT=$( echo ${SLURM_JOB_ACCOUNT:=::default_project::} | cut -c "1-3" ) 
73### jobWarningDelay in seconds
74##typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75 
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=sbatch}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Access to module command
93#====================================================
94# See ticket #348
95# Following path should be used according to assist IDRIS
96. /gpfslocalsup/spack_soft/environment-modules/current/init/ksh
97# Following old path gives error messages while loading modules but it still works
98#. /usr/share/Modules/init/ksh
99
100#====================================================
101# Set environment tools (ferret, nco, cdo, rebuild, ...)
102#====================================================
103if [ X${TaskType} = Xcomputing ] ; then
104  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
105else
106  module purge > /dev/null 2>&1
107  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1
108
109  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1
110
111#  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
112  export UVCDAT_ANONYMOUS_LOG=FALSE
113fi
114
115# FYI
116[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
117[ ! X${TaskType} = Xchecking ] && module list
118
119# For AddNoise (not yet AddPertu3DOCE)
120export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/AddNoise/bin
121
122# For rebuild
123export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
124
125#====================================================
126# Host specific DIRECTORIES
127#====================================================
128
129#====================================================
130#- MirrorlibIGCM for frontend
131typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132
133#====================================================
134#- libIGCM_POST for frontend
135typeset -r libIGCM_POST=${libIGCM}
136
137#====================================================
138#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
139typeset -r R_EXE="${MODIPSL}/bin"
140
141#====================================================
142#- SUBMIT_DIR : submission dir
143typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
144
145#====================================================
146#- IN
147typeset -r R_IN=${R_IN:=/gpfswork/rech/psl/commun/IGCM}
148
149#====================================================
150#- RUN_DIR_PATH : Temporary working directory (=> TMP)
151typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}}
152
153#====================================================
154#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
155typeset -r OUTCOMMAND_PATH=/tmp
156
157#====================================================
158#- HOST_MPIRUN_COMMAND
159typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time srun "}
160
161#====================================================
162#- Max number of arguments passed to nco operator or demigration command
163UNIX_MAX_LIMIT=360
164
165#====================================================
166#- set PackDefault to true
167PackDefault=true
168
169#====================================================
170#- Default number of MPI task for IPSL coupled model
171#- required for backward compatibility
172#-
173DEFAULT_NUM_PROC_OCE=5
174DEFAULT_NUM_PROC_CPL=1
175(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
176
177#====================================================
178#- Number of computing cores per node
179#-
180NB_CORE_PER_NODE=40
181
182#D-#==================================================
183#D-function IGCM_sys_defineArchives
184#D-* Purpose:
185#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
186#D-* Define ARCHIVE : Dedicated to large files
187#D-* Define STORAGE : Dedicated to small/medium files
188#D-* Define R_OUT   : Output tree located on ARCHIVE
189#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monsitoring and atlas, and/or small files)
190#D-* Define R_BUF   : Output tree located on SCRATCH hosting files waiting for rebuild or pack processes
191#D-* if SpaceName=TEST everything is stored on SCRATCH
192#D-* Examples:
193#D-
194function IGCM_sys_defineArchives {
195  IGCM_debug_PushStack "IGCM_sys_defineArchives"
196
197  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
198  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
199    # The variable DataProject is not set in in section UserChoices in config.card or it is set to DEFAULT
200    # Default option: Set DataProject according to the project used for submission
201    DataProject=${PROJECT}
202  else
203    # Use the project set by the variable DataProject in section UserChoices in config.card
204    IGCM_debug_Print 1 "Use DataProject as set in config.card : ${config_UserChoices_DataProject}"
205    DataProject=${config_UserChoices_DataProject}
206  fi
207  IGCM_debug_Print 1 "Data project for output to be used: ${DataProject}"
208
209  #====================================================
210  #- RUN_DIR_PATH : Temporary working directory (=> TMP)
211  #- DataProject = for is the dataproject for Training account at Idris
212  #- they are not store in rech but in another partition
213  if [ X${DataProject} = Xfor  ]; then
214      typeset -r RUN_DIR_PATH=$SCRATCH/RUN_DIR/${SLURM_JOBID}_${$}
215  else
216      typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/gpfsscratch/rech/${DataProject}/${LOGIN}/RUN_DIR/${SLURM_JOBID}_${$}}
217  fi
218
219  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
220    #====================================================
221    #- ARCHIVE (dedicated to large files)
222    ARCHIVE=${config_UserChoices_ARCHIVE}
223  else
224      if [ X${DataProject} = Xfor  ]; then
225          ARCHIVE=$STORE
226      else
227          #====================================================
228          #- ARCHIVE (dedicated to large files)
229          ARCHIVE=/gpfsstore/rech/${DataProject}/${LOGIN}
230      fi
231  fi
232
233  if [ ! X${config_UserChoices_STORAGE} = X ]; then
234    #====================================================
235    #- STORAGE (dedicated to small/medium files)
236    STORAGE=${config_UserChoices_STORAGE}
237  else
238    #====================================================
239    #- STORAGE (dedicated to small/medium files)
240      if [ X${DataProject} = Xfor  ]; then
241          STORAGE=$WORK
242      else
243          STORAGE=/gpfswork/rech/${DataProject}/${LOGIN}
244      fi
245  fi
246
247  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
248    #====================================================
249    #- R_OUT
250    if [ X${DataProject} = Xfor  ]; then
251        R_OUT=$SCRATCH/IGCM_OUT
252    else
253        R_OUT=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
254    fi
255    #====================================================
256    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
257    R_FIG=${R_OUT}
258
259    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
260
261  else
262    #====================================================
263    #- R_OUT
264    R_OUT=${ARCHIVE}/IGCM_OUT
265
266    #====================================================
267    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
268    R_FIG=${STORAGE}/IGCM_OUT
269  fi
270
271  #====================================================
272  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
273  CMIP6_BUF=${STORAGE}/IGCM_OUT
274
275  #====================================================
276  #- R_BUF (ONLY FOR double copy an scratch)
277 
278  if [ X${DataProject} = Xfor  ]; then
279      R_BUF=$SCRATCH/IGCM_OUT
280  else
281      R_BUF=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
282  fi
283
284  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
285  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
286  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
287
288  IGCM_debug_PopStack "IGCM_sys_defineArchives"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_RshArchive
293#D-* Purpose: Archive rsh command
294#D-* Examples:
295#D-
296function IGCM_sys_RshArchive {
297  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
298  /bin/ksh <<-EOF
299    ${@}
300EOF
301  status=$?
302  if [ ${status} -gt 0 ] ; then
303    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
304    IGCM_debug_Exit "IGCM_sys_RshArchive"
305  fi
306  IGCM_debug_PopStack "IGCM_sys_RshArchive"
307}
308
309#D-#==================================================
310#D-function IGCM_sys_RshArchive_NoError
311#D-* Purpose: Archive rsh command, without error
312#D-*          used only in monitoring.job
313#D-* Examples:
314#D-
315function IGCM_sys_RshArchive_NoError {
316  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
317  /bin/ksh <<-EOF
318    ${@} 2> /dev/null
319EOF
320  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
321}
322
323#D-#==================================================
324#D-function IGCM_sys_MkdirArchive
325#D-* Purpose: Mkdir on Archive
326#D-* Examples:
327#D-
328function IGCM_sys_MkdirArchive {
329  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
330  if ( $DEBUG_sys ) ; then
331    echo "IGCM_sys_MkdirArchive :" $@
332  fi
333  #- creation de repertoire sur le serveur fichier
334  if [ ! -d ${1} ]; then
335    \mkdir -p $1
336    status=$?
337
338    if [ ${status} -gt 0 ] ; then
339      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
340      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
341    fi
342  fi
343  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
344}
345
346#D-#==================================================
347#D-function IGCM_sys_TestDirArchive
348#D-* Purpose: Test Directory that must exists on Archive
349#D-* Examples:
350#D-
351function IGCM_sys_TestDirArchive {
352  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
353  if ( $DEBUG_sys ) ; then
354    echo "IGCM_sys_TestDirArchive :" $@
355  fi
356  typeset ExistFlag
357  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
358  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
359  return ${ExistFlag}
360}
361
362#D-#==================================================
363#D-function IGCM_sys_IsFileArchived
364#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
365#D-* Examples:
366#D-
367function IGCM_sys_IsFileArchived {
368  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
369  if ( $DEBUG_sys ) ; then
370    echo "IGCM_sys_IsFileArchived :" $@
371  fi
372  typeset IsArchivedFlag
373# To be implemented on Jean Zay
374#  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
375  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
376
377  return ${IsArchivedFlag}
378}
379
380#D-#==================================================
381#D-function IGCM_sys_TestFileArchive
382#D-* Purpose: Test file that must NOT EXISTS on Archive
383#D-* Examples:
384#D-
385function IGCM_sys_TestFileArchive {
386  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
387  typeset ExistFlag
388  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
389  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
390
391  return ${ExistFlag}
392}
393
394#D-#==================================================
395#D-function IGCM_sys_CountFileArchive
396#D-* Purpose: Count files on Archive filesystem
397#D-* Examples:
398#D-
399function IGCM_sys_CountFileArchive {
400  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
401  ls ${@} 2>/dev/null | wc -l
402  if [ $? -gt 0 ] ; then
403    echo "IGCM_sys_CountFileArchive : erreur."
404  fi
405  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
406}
407
408#D-#==================================================
409#D-function IGCM_sys_Tree
410#D-* Purpose: Tree directories with files on ${ARCHIVE}
411#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
412#D-
413function IGCM_sys_Tree {
414  IGCM_debug_PushStack "IGCM_sys_Tree" $@
415  if ( $DEBUG_sys ) ; then
416    echo "IGCM_sys_Tree :" $@
417  fi
418
419  \ls -lR ${@}
420
421  IGCM_debug_PopStack "IGCM_sys_Tree"
422}
423
424#D-#==================================================
425#D-function IGCM_sys_Qsub
426#D-* Purpose: Qsub new job
427#D-* Examples:
428#D-
429function IGCM_sys_Qsub {
430  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
431  if ( $DEBUG_sys ) ; then
432    echo "IGCM_sys_Qsub :" $@
433  fi
434  typeset options status
435  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
436
437  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
438  status=$?
439
440  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
441  if [ ${status} -gt 0 ] ; then
442    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
443    IGCM_debug_Exit "IGCM_sys_Qsub"
444  else
445    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
446  fi
447  IGCM_debug_PopStack "IGCM_sys_Qsub"
448}
449
450#D-#==================================================
451#D-function IGCM_sys_QsubPost
452#D-* Purpose: Qsub new job on scalaire
453#D-* Examples:
454#D-
455function IGCM_sys_QsubPost {
456  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
457  if ( $DEBUG_sys ) ; then
458    echo "IGCM_sys_QsubPost :" $@
459  fi
460  typeset options status
461  options="-o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
462
463  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
464  status=$?
465
466  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
467  if [ ${status} -gt 0 ] ; then
468    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
469    IGCM_debug_Exit "IGCM_sys_QsubPost"
470  else
471    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
472  fi
473  IGCM_debug_PopStack "IGCM_sys_QsubPost"
474}
475
476#D-*************************
477#D- File transfer functions
478#D-*************************
479#D-
480
481#D-#==================================================
482#D-function IGCM_sys_RmRunDir
483#D-* Purpose: rm tmpdir (dummy function most of the time batch
484#D-                      scheduler will do the job)
485#D-* Examples:
486#D-
487function IGCM_sys_RmRunDir {
488  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
489  if ( $DEBUG_sys ) ; then
490    echo "IGCM_sys_RmRunDir :" $@
491  fi
492
493  typeset status
494
495  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
496  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
497  status=$?
498
499  if [ ${status} -gt 0 ] ; then
500    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
501    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
502    IGCM_debug_Exit "IGCM_sys_RmRunDir"
503  else
504    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
505  fi
506  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
507}
508
509#D-#==================================================
510#D-function IGCM_sys_Put_Dir
511#D-* Purpose: Copy a complete directory on $(ARCHIVE)
512#D-* Examples:
513#D-
514function IGCM_sys_Put_Dir {
515  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
516  if ( $DEBUG_sys ) ; then
517    echo "IGCM_sys_Put_Dir :" $@
518  fi
519  if [ $DRYRUN = 0 ]; then
520    if [ ! -d ${1} ] ; then
521      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
522      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
523      return
524    fi
525
526    typeset status
527
528    # Only if we use rsync
529    #IGCM_sys_TestDirArchive $( dirname $2 )
530    #
531    #USUAL WAY
532    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
533    status=$?
534
535    if [ ${status} -gt 0 ] ; then
536      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
537      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
538      IGCM_debug_Exit "IGCM_sys_Put_Dir"
539    else
540      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
541    fi
542  fi
543  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
544}
545
546#D-#==================================================
547#D-function IGCM_sys_Get_Dir
548#D-* Purpose: Copy a complete directory from ${ARCHIVE}
549#D-* Examples:
550#D-
551function IGCM_sys_Get_Dir {
552  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
553  if ( $DEBUG_sys ) ; then
554    echo "IGCM_sys_Get_Dir :" $@
555  fi
556  if [ $DRYRUN = 0 ]; then
557    typeset NB_ESSAI DELAI status i
558    # number of tentative
559    NB_ESSAI=3
560    # time delay between tentative
561    DELAI=2
562
563    #
564    # USUAL WAY
565    # add here command to demigrate all offline files to reduce time of this command
566    # actually on Jean Zay files are not archive on bands
567
568    i=0
569    while [ $i -lt $NB_ESSAI ] ; do
570      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
571      status=$?
572      if [ ${status} -gt 0 ] ; then
573        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
574        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
575        sleep $DELAI
576      else
577        break
578      fi
579      (( i = i + 1 ))
580    done
581
582    if [ ${status} -gt 0 ] ; then
583      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
584      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
585      IGCM_debug_Exit "IGCM_sys_Get_Dir"
586    else
587      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
588    fi
589  fi
590  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
591}
592
593#D-#==================================================
594#D-function IGCM_sys_Put_Rest
595#D-* Purpose: Put computied restarts on ${ARCHIVE}.
596#D-           File and target directory must exist.
597#D-* Examples:
598#D-
599function IGCM_sys_Put_Rest {
600  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
601  if ( $DEBUG_sys ) ; then
602    echo "IGCM_sys_Put_Rest :" $@
603  fi
604  if [ $DRYRUN = 0 ]; then
605    if [ ! -f ${1} ] ; then
606      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
607      IGCM_debug_Exit "IGCM_sys_Put_Rest"
608    fi
609
610    typeset status
611    #
612    # USUAL WAY
613    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
614    status=$?
615
616#       #RSYNC WITH NETWORK SSH CALL
617#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
618#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
619
620#       #RSYNC WITH NFS USE
621#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
622#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
623
624#       status=$?
625#       IGCM_sys_Rsync_out $status
626
627#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
628#       (( status=status+$? ))
629
630    if [ ${status} -gt 0 ] ; then
631      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
632      [ -f ${1} ] && ls -l ${1}
633      [ -f ${2} ] && ls -l ${2}
634      [ -f ${2}/${1} ] && ls -l ${2}/${1}
635      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636      IGCM_debug_Exit "IGCM_sys_Put_Rest"
637    else
638
639      if [ X${JobType} = XRUN ] ; then
640        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
641        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
642      fi
643
644      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645    fi
646  fi
647  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
648}
649
650#D-#==================================================
651#D-function IGCM_sys_Put_Out
652#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
653#D-* Examples:
654#D-
655function IGCM_sys_Put_Out {
656  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
657  if ( $DEBUG_sys ) ; then
658    echo "IGCM_sys_Put_Out :" $@
659  fi
660
661  typeset NB_ESSAI DELAI status i exist skip
662  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
663
664  # number of tentative
665  NB_ESSAI=3
666  # time delay between tentative
667  DELAI=2
668
669  if [ $DRYRUN = 0 ]; then
670    if [ ! -f ${1} ] ; then
671      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
672      IGCM_debug_PopStack "IGCM_sys_Put_Out"
673      return 1
674    fi
675    #
676    IGCM_sys_MkdirArchive $( dirname $2 )
677    #
678    exist=false
679    skip=false
680    if [ -f $2 ] ; then
681      IGCM_debug_Print 1 "$2 already exist"
682    # add here command to demigrate all offline files to reduce time of this command
683    # actually on Jean Zay files are not archive on bands
684
685      exist=true
686      if [ "X$( diff $1 $2 )" = X ] ; then
687        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
688        skip=true
689      else
690        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
691        skip=false
692      fi
693    fi
694    #
695    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
696      IGCM_sys_Chmod u+w $2
697    fi
698
699    if [ X${skip} = Xfalse ] ; then
700      i=0
701      while [ $i -lt $NB_ESSAI ] ; do
702        # Identify file system
703        fileDeviceNumberInHex=$( stat -c %d $1 )
704        status=$?
705        if [ ${status} -gt 0 ] ; then
706          IGCM_debug_Exit "IGCM_sys_Put_Out"
707        fi
708        # Identify file system
709        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
710        status=$?
711        if [ ${status} -gt 0 ] ; then
712          IGCM_debug_Exit "IGCM_sys_Put_Out"
713        fi
714
715        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
716          # They are not on the same device. USUAL WAY
717          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718          status=$?
719        else
720          # They are on the same device. NOT SO USUAL WAY
721          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
722          status=$?
723        fi
724        if [ ${status} -gt 0 ]; then
725          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
726          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
727          [ -f ${1} ] && ls -l ${1}
728          [ -f ${2} ] && ls -l ${2}
729          [ -f ${2}/${1} ] && ls -l ${2}/${1}
730          sleep $DELAI
731        else
732          break
733        fi
734        (( i = i + 1 ))
735      done
736    fi
737
738#       #RSYNC WITH NETWORK SSH CALL
739#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
740#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741
742#       #RSYNC WITH NFS USE
743#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
744#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
745
746#       status=$?
747#       IGCM_sys_Rsync_out $status
748
749#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
750#       (( status=status+$? ))
751
752    if [ ${status} -gt 0 ] ; then
753      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
754      [ -f ${1} ] && ls -l ${1}
755      [ -f ${2} ] && ls -l ${2}
756      [ -f ${2}/${1} ] && ls -l ${2}/${1}
757      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
758      IGCM_debug_Exit "IGCM_sys_Put_Out"
759    else
760
761      if [ X${JobType} = XRUN ] ; then
762        if [ X${3} = X ] ; then
763          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
764          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
765        fi
766      fi
767
768      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
769    fi
770  fi
771  IGCM_debug_PopStack "IGCM_sys_Put_Out"
772  return 0
773}
774
775#D-#==================================================
776#D-function IGCM_sys_Get
777#D-* Purpose: Get a file from ${ARCHIVE}
778#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
779#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
780function IGCM_sys_Get {
781  IGCM_debug_PushStack "IGCM_sys_Get" $@
782
783  typeset DEST dm_liste target file_work
784  typeset NB_ESSAI DELAI status i
785
786  if ( $DEBUG_sys ) ; then
787    echo "IGCM_sys_Get :" $@
788  fi
789
790  # number of tentative
791  NB_ESSAI=3
792  # time delay between tentative
793  DELAI=2
794
795  if [ $DRYRUN -le 2 ]; then
796    if [ X${1} = X'/l' ] ; then
797      eval set +A dm_liste \${${2}}
798    else
799      eval set +A dm_liste ${1}
800    fi
801    eval DEST=\${${#}}
802    # add here command to demigrate all offline files to reduce time of this command
803    # actually on Jean Zay files are not archive on bands
804
805    status=$?
806    if [ ${status} -gt 0 ] ; then
807      echo "WARNING IGCM_sys_Get : error code ${status}"
808      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
810    fi
811
812    #if [ ${status} -gt 0 ] ; then
813    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
814    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
815    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
816    #    sleep 30
817    #    echo "We try another time"
818    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
820    #    status=$?
821    #    if [ ${status} -gt 0 ] ; then
822    #      echo "ERROR IGCM_sys_Get : again demigration error :"
823    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
824    #      IGCM_debug_Exit "IGCM_sys_Get"
825    #    fi
826    #  else
827    #    echo "ERROR IGCM_sys_Get : demigration error :"
828    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829    #    IGCM_debug_Exit "IGCM_sys_Get"
830    #  fi
831    #fi
832
833    #   #RSYNC WITH NETWORK SSH CALL
834    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
835    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
836
837    #   #RSYNC WITH NFS USE
838    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840
841    #   status=$?
842    #   IGCM_sys_Rsync_out $status
843
844    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
845    #   (( status=status+$? ))
846
847    #USUAL WAY
848    if [ X${1} = X'/l' ] ; then
849      for target in ${dm_liste[*]} ; do
850        local_file=$( basename ${target} )
851        # test if the target file is present before the loop
852        IGCM_sys_TestFileArchive ${target}
853        status=$?
854        if [ ${status} -gt 0 ] ; then
855          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
856          IGCM_debug_Exit "IGCM_sys_Get"
857        else
858          i=0
859          while [ $i -lt $NB_ESSAI ] ; do
860            #if [ X${DoLink} = Xtrue ] ; then
861            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862            #  status=$?
863            #  else
864            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
865            #  status=$?
866            #fi
867            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
868            status=$?
869            if [ ${status} -gt 0 ]; then
870              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
871              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
872              sleep $DELAI
873            else
874              break
875            fi
876            (( i = i + 1 ))
877          done
878          if [ ${status} -gt 0 ] ; then
879            echo "IGCM_sys_Get : error"
880            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
881            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
882            IGCM_debug_Exit "IGCM_sys_Get"
883          else
884            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
885          fi
886        fi
887      done
888    else
889      i=0
890      while [ $i -lt $NB_ESSAI ] ; do
891        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
892        status=$?
893        if [ ${status} -gt 0 ]; then
894          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
895          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
896          sleep $DELAI
897        else
898          break
899        fi
900        (( i = i + 1 ))
901      done
902      if [ ${status} -gt 0 ] ; then
903        echo "IGCM_sys_Get : error"
904        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
905        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
906        IGCM_debug_Exit "IGCM_sys_Get"
907      else
908        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
909      fi
910    fi
911  fi
912  IGCM_debug_PopStack "IGCM_sys_Get"
913}
914
915#D-#==================================================
916#D-function IGCM_sys_GetDate_Monitoring
917#D-* Purpose: get the last year for which the monitoring has been computed
918#D-* Examples:
919#D-
920function IGCM_sys_GetDate_Monitoring {
921  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
922  if ( $DEBUG_sys ) ; then
923    echo "IGCM_sys_GetDate_Monitoring :" $@
924  fi
925
926  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
927
928  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
929}
930
931#D-#==================================================
932#D-function IGCM_sys_Dods_Rm
933#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
934#D-* Examples:
935#D-
936function IGCM_sys_Dods_Rm {
937  if ( $DEBUG_sys ) ; then
938    echo "IGCM_sys_Dods_Rm :" $@
939  fi
940  typeset status
941  if [ $DRYRUN = 0 ]; then
942
943#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
944#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
945#      echo "Nothing has been done."
946#      return
947#    fi
948
949    if [ "$#" -eq 1 ]; then
950      thredds_rm ${R_DODS}/${1}
951      status=$?
952    else
953        thredds_rm ${R_DODS}
954      status=$?
955    fi
956
957#    if [ ${status} -gt 0 ] ; then
958#      echo "IGCM_sys_Dods_Rm : error."
959#      cat out_dods_rm
960#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
961#    else
962#      rm out_dods_rm
963#    fi
964
965  fi
966  return $status
967}
968
969#D-#==================================================
970#D-function IGCM_sys_Dods_Cp
971#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
972#D-* Examples:
973#D-
974function IGCM_sys_Dods_Cp {
975  if ( $DEBUG_sys ) ; then
976    echo "IGCM_sys_Dods_Cp :" $@
977  fi
978  typeset status
979  if [ $DRYRUN = 0 ]; then
980
981#    if [ ! -d ${R_SAVE}/${1} ] ; then
982#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
983#      echo "Nothing has been done."
984#      return
985#    fi
986
987
988      thredds_cp ${1} ./${R_DODS}/.
989
990    status=$?
991
992#       if [ ${status} -gt 0 ] ; then
993#           echo "IGCM_sys_Dods_Cp : error."
994#           cat out_dods_cp
995#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
996#       else
997#           rm out_dods_cp
998#       fi
999
1000  fi
1001  return $status
1002}
1003
1004#D-#==================================================
1005#D-function IGCM_sys_Put_Dods
1006#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1007#D-* Examples:
1008#D-
1009function IGCM_sys_Put_Dods {
1010  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1011  if ( $DEBUG_sys ) ; then
1012    echo "IGCM_sys_Put_Dods :" $@
1013  fi
1014  typeset status
1015  if [ $DRYRUN = 0 ]; then
1016    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1017      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1018      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1019      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1020      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1021      return
1022    fi
1023
1024    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1025      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1026      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1027      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1028      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1029      return
1030    fi
1031    #
1032    if [ -d ${R_SAVE}/${1} ] ; then
1033      cd ${R_SAVE}
1034    elif [ -d ${R_FIGR}/${1} ] ; then
1035      cd ${R_FIGR}
1036    fi
1037
1038    IGCM_sys_Dods_Rm ${1}
1039    IGCM_sys_Dods_Cp ${1}
1040    status=0
1041
1042    if [ ${status} -gt 0 ] ; then
1043      echo "IGCM_sys_Put_Dods : error."
1044      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1045    fi
1046  fi
1047  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1048}
1049
1050##############################################################
1051# REBUILD OPERATOR
1052
1053#D-#==================================================
1054#D-function IGCM_sys_sync
1055#D-* Purpose: flush buffer on disk
1056#D-* Examples:
1057#D-
1058function IGCM_sys_sync {
1059  IGCM_debug_PushStack "IGCM_sys_sync" $@
1060  if ( $DEBUG_sys ) ; then
1061    echo "IGCM_sys_sync :" $@
1062  fi
1063
1064  /bin/sync
1065
1066  IGCM_debug_PopStack "IGCM_sys_sync"
1067}
1068
1069############################################################
1070# Activate Running Environnment Variables
1071
1072#D-#==================================================
1073#D-function IGCM_sys_activ_variables
1074#D-* Purpose: set environement variables prior to execution
1075#D-* Examples:
1076#D-
1077function IGCM_sys_activ_variables {
1078  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1079  if ( $DEBUG_sys ) ; then
1080    echo "IGCM_sys_activ_variables"
1081  fi
1082
1083# --------------------------------------------------------------------
1084#D- MODULE specifications
1085# --------------------------------------------------------------------
1086  # Source the file EnvFile containing all module load needed to run the model.
1087  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1088  # it will be used. Otherwise a default file will be used.
1089  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1090      # EnvFile is set in config.card and the file exists
1091      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1092      EnvFile=${config_UserChoices_EnvFile}
1093  else
1094      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1095      module purge > /dev/null 2>&1
1096      EnvFile=/gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay
1097  fi
1098  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1099  IGCM_debug_Print 1 ${EnvFile}
1100  . ${EnvFile}
1101
1102  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Jean-Zay. "
1103  module list
1104
1105# --------------------------------------------------------------------
1106#D- MPI specifications
1107# --------------------------------------------------------------------
1108
1109# --------------------------------------------------------------------
1110#D- Other specifications
1111# --------------------------------------------------------------------
1112
1113  ulimit -s unlimited
1114  if [ ${executionType} -eq 2 ] ; then
1115    export SLURM_HOSTFILE=./hostlist
1116  fi
1117
1118
1119  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1120}
1121
1122############################################################
1123# Desactivate Running Environnment Variables
1124
1125#D-#==================================================
1126#D-function IGCM_sys_desactiv_variables
1127#D-* Purpose: unset environement variables after execution
1128#D-* Examples:
1129#D-
1130function IGCM_sys_desactiv_variables {
1131  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1132  if ( $DEBUG_sys ) ; then
1133    echo "IGCM_sys_desactiv_variables"
1134  fi
1135# --------------------------------------------------------------------
1136#D- MPI specifications
1137# --------------------------------------------------------------------
1138  if [ ${executionType} -eq 2 ] ; then
1139    unset SLURM_HOSTFILE
1140  fi
1141# --------------------------------------------------------------------
1142#D- Other specifications
1143# --------------------------------------------------------------------
1144
1145  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1146}
1147
1148############################################################
1149# Update job headers to be used by the scheduler
1150
1151#D-#==================================================
1152#D-function IGCM_sys_updateHeaders
1153#D-* Purpose: Update job headers to be used by the scheduler
1154#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1155#D-
1156function IGCM_sys_updateHeaders {
1157  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1158  if ( $DEBUG_sys ) ; then
1159    echo "IGCM_sys_updateHeaders"
1160  fi
1161  typeset file
1162  file=$1
1163
1164  if [ ${executionType} -eq 1 ] ; then
1165    # MPMD + MPI
1166    sed -e "/::openMPthreads::/d"                  \
1167        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1168      ${file} > ${file}.tmp
1169
1170  elif [ ${executionType} -eq 2 ] ; then
1171    # MPMD + MPI + OMP
1172      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1173      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1174      sed -e "/::openMPthreads::/d"                 \
1175        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
1176        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
1177        -e "/ntasks/i\#SBATCH --exclusive"          \
1178         ${file} > ${file}.tmp
1179  elif [ ${executionType} -eq 3 ] ; then
1180    # SPMD + MPI/OMP
1181    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1182        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1183      ${file} > ${file}.tmp
1184
1185  elif [ ${executionType} -eq 4 ] ; then
1186    # SPMD + MPI only
1187    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1188        -e "/::openMPthreads::/d"                  \
1189      ${file} > ${file}.tmp
1190
1191  elif [ ${executionType} -eq 5 ] ; then
1192    # SPMD + OMP only
1193    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1194        -e "/::JobNumProcTot::/d"                  \
1195      ${file} > ${file}.tmp
1196
1197  elif [ ${executionType} -eq 6 ] ; then
1198    # SEQUENTIAL THEN
1199    sed -e "s/::JobNumProcTot::/1/"                \
1200        -e "/::openMPthreads::/d"                  \
1201      ${file} > ${file}.tmp
1202
1203  fi
1204
1205  IGCM_sys_Mv ${file}.tmp ${file}
1206
1207  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1208}
1209
1210############################################################
1211# Build MPI/OMP scripts run file (dummy function)
1212
1213#D-#==================================================
1214#D-function IGCM_sys_build_run_file
1215#D-* Purpose: build run file (deprecated)
1216#D-* Examples:
1217#D-
1218function IGCM_sys_build_run_file {
1219  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1220}
1221
1222############################################################
1223# Build MPI/OMP scripts
1224
1225#D-#==================================================
1226#D-function IGCM_sys_build_execution_scripts
1227#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1228#D-* Examples:
1229#D-
1230function IGCM_sys_build_execution_scripts
1231{
1232  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1233  if ( $DEBUG_sys ) ; then
1234    echo "IGCM_sys_build_execution_scripts " $@
1235  fi
1236  IGCM_debug_Print 1 "executionType= ${executionType}"
1237  EXECUTION=${HOST_MPIRUN_COMMAND}
1238
1239  if ( ${OK_PARA_MPMD} ) ; then
1240
1241    # MPMD mode
1242    # 1 MPI only : executionType=1
1243    # 2 MPI/OpenMP : executionType=2
1244
1245    if [ -f run_file ] ; then
1246      IGCM_sys_Rm -f run_file
1247    fi
1248    touch run_file
1249
1250    # case 1 : Only MPI (MPMD)
1251    if  ( ! ${OK_PARA_OMP} ) ; then
1252      # Build run_file
1253      current_core=0
1254      # First loop on the components for the coupler ie oasis (only if oasis3)
1255      # the coupler ie oasis3 must be the first one
1256      for comp in ${config_ListOfComponents[*]} ; do
1257
1258        eval ExeNameIn=\${config_Executable_${comp}[0]}
1259        eval ExeNameOut=\${config_Executable_${comp}[1]}
1260
1261        # Only if the component has an executable
1262        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1263
1264          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1265          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
1266          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
1267          (( current_core = ${end_core} + 1 ))
1268        fi
1269      done
1270
1271      EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file"
1272
1273      IGCM_sys_Chmod u+x run_file
1274      if ( $DEBUG_sys ) ; then
1275        echo "run_file contains : "
1276        cat run_file
1277      fi
1278
1279    else 
1280
1281    # 2 MPI/OpenMP : executionType=2
1282
1283    # MPI-OpenMP (MPMD)
1284    # export SLURM_HOSTFILE=./hostlist
1285    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1286    # example of  hostlist file :
1287    # r3i3n33
1288    # r3i3n33
1289    # ...
1290    # example of run_file :
1291    # 0-70 ./prog_lmdz.x.sh %o %t
1292    # 71-430 ./prog_opa.xx.sh %o %t
1293    # 431-431 ./prog_xios.x.sh %o %t
1294    # examples of prog_file :
1295    # prog_lmdz.x.sh :
1296    # (( init = 0 + $1 ))
1297    # (( index = init * 10 ))
1298    # (( slot = index % 40 ))
1299    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1300    # that will become
1301    # taskset -c 0-9 ./script_lmdz.x.ksh
1302    # ...
1303    # with script_lmdz.x.ksh
1304    # export OMP_STACKSIZE=3g
1305    # export OMP_PLACES=cores
1306    # export OMP_NUM_THREADS=10
1307    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1308
1309    #  Hosts treatment
1310    _bkIFS=$IFS;
1311        IFS=$'\n'; set -f
1312        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1313        IFS=$_bkIFS; set +f
1314        rm -f hostlist
1315
1316     # Loop on the components to build run_file and script_exec files
1317        rank=0
1318        current_core=0
1319        current_core_mpi=0
1320
1321      for comp in ${config_ListOfComponents[*]} ; do
1322
1323        eval ExeNameIn=\${config_Executable_${comp}[0]}
1324        eval ExeNameOut=\${config_Executable_${comp}[1]}
1325
1326        # Not possible if oasis has an executable (i.e old version of oasis3)
1327        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1328          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1329          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1330          IGCM_debug_Verif_Exit
1331        fi
1332
1333        # Only if we really have an executable for the component :
1334        if [ "X${ExeNameOut}" != X\"\" ] ; then
1335
1336          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1337          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1338          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1339     
1340
1341          # Build script files
1342
1343          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1344          echo ""  >> script_${ExeNameOut}.ksh
1345          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1346          # Check if the number of threads is correct
1347              case ${comp_proc_omp_loc} in
1348                  2|4|5|10|20)
1349                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1350                      ;;
1351                  *)
1352                      IGCM_debug_Exit "ERROR with OMP parameters !"
1353                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1354                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1355                      IGCM_debug_Verif_Exit
1356                      ;;
1357              esac
1358            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1359            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1360            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1361
1362          fi
1363
1364          # to have out/err per process on different files
1365          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1366
1367          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1368
1369        # Build run_file
1370        # Only if the component has an executable
1371        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1372
1373          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1374          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1375          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1376          (( current_core_mpi = ${end_core} + 1 ))
1377        fi
1378
1379          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1380              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1381          else
1382              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1383          fi
1384
1385          # Build configuration file
1386 
1387              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1388          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1389          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1390          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1391              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1392          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1393
1394          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1395
1396        # Build hostlist file
1397
1398          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1399             (( index_host = current_core / NB_CORE_PER_NODE ))
1400             host_value=${listnodes[${index_host}]}
1401               echo "$host_value" >> hostlist
1402             (( current_core = current_core + offset_comp_proc_loc ))
1403          done
1404        fi
1405      done
1406
1407      ## variable added to stop after 60s instead of 600s by default.
1408      ## This is used when no error comes from executables and when something stopped an executable without notice.
1409      export SLURM_WAIT=60
1410
1411      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1412
1413      IGCM_sys_Chmod u+x run_file
1414      if ( $DEBUG_sys ) ; then
1415        echo "run_file contains : "
1416        cat run_file
1417      fi
1418
1419    fi # if ${OK_PARA_MPMD}
1420
1421  else
1422  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1423
1424    for comp in ${config_ListOfComponents[*]} ; do
1425
1426      # Only if we really have an executable for the component :
1427      eval ExeNameOut=\${config_Executable_${comp}[1]}
1428      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1429
1430        # Build script files
1431
1432        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1433        echo ""  >> script_${ExeNameOut}.ksh
1434        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1435
1436        if ( ${OK_PARA_OMP} ) ; then
1437            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1438            # Check if the number of threads is correct
1439            case ${comp_proc_omp_loc} in
1440                2|4|5|10|20)
1441                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1442                    ;;
1443                *)
1444                    IGCM_debug_Exit "ERROR with OMP parameters !"
1445                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1446                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1447                    IGCM_debug_Verif_Exit
1448                    ;;
1449            esac
1450            echo ""  >> script_${ExeNameOut}.ksh
1451            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1452            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1453            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1454        fi
1455
1456        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1457       
1458        # To have out/err per process on different files
1459        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1460        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1461
1462        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1463        cat script_${ExeNameOut}.ksh
1464
1465      fi
1466
1467    done
1468
1469  fi # ${OK_PARA_MPMD}
1470
1471  IGCM_debug_Print 1 "sys Irene : execution command is "
1472  IGCM_debug_Print 1 "$EXECUTION"
1473
1474  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1475}
1476
1477#D-#==================================================
1478#D-function IGCM_sys_check_path
1479#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1480#D-* do not point to an important use directory. Stop immediately in that case.
1481#D-* Examples:
1482#D-
1483function IGCM_sys_check_path {
1484  IGCM_debug_PushStack "IGCM_sys_check_path"
1485  if ( $DEBUG_sys ) ; then
1486    echo "IGCM_sys_check_path"
1487  fi
1488
1489  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORK} ] || [ X${RUN_DIR_PATH} = X${SCRATCH} ] || [ X${RUN_DIR_PATH} = X${STORE} ] ) ; then
1490    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1491    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1492    IGCM_debug_Exit "This will stop the job"
1493  elif ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X/gpfswork/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsscratch/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsstore/rech/${DataProject}/${LOGIN} ] ) ; then
1494    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1495    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1496    IGCM_debug_Exit "This will stop the job"
1497  fi
1498  IGCM_debug_PopStack "IGCM_sys_check_path"
1499}
1500
1501#D-#==================================================
1502#D-function IGCM_sys_check_quota
1503#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1504#D-* Examples:
1505#D-
1506function IGCM_sys_check_quota {
1507  IGCM_debug_PushStack "IGCM_sys_check_quota"
1508  echo "IGCM_sys_check_quota is desactivated on JeanZay because there is actually no way to check scratch quota on this computing center (2019.11.19)"
1509#  if ( $DEBUG_sys ) ; then
1510#    echo "IGCM_sys_check_quota"
1511#  fi
1512#  # Limit of quota (in %)
1513#  limit_quota=90
1514#
1515#  # Check of the volume
1516#  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1517#  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1518#
1519#  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1520#
1521#    unit_avail=${volume_avail: -1}
1522#    unit_quota=${volume_quota: -1}
1523#
1524#    if [ "${unit_quota}" = "*" ] ; then
1525#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1526#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1527#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1528#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1529#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1530#      IGCM_debug_Verif_Exit
1531#    fi
1532#
1533#    temp_avail=${volume_avail%%${volume_avail: -1}*}
1534#    temp_quota=${volume_quota%%${volume_quota: -1}*}
1535#
1536#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1537#
1538#    # Convertion
1539#      if [ ${volume_avail: -1} = "T" ] ; then
1540#        (( temp_avail = temp_avail * 1000000000000 ))
1541#      elif [ ${volume_avail: -1} = "G" ] ; then
1542#        (( temp_avail = temp_avail * 1000000000 ))
1543#      elif [ ${volume_avail: -1} = "M" ] ; then
1544#        (( temp_avail = temp_avail * 1000000 ))
1545#      elif [ ${volume_avail: -1} = "k" ] ; then
1546#        (( temp_avail = temp_avail * 1000 ))
1547#      else
1548#        (( temp_avail = volume_avail ))
1549#      fi
1550#      if [ ${volume_quota: -1} = "T" ] ; then
1551#        (( temp_quota = temp_quota * 1000000000000 ))
1552#      elif [ ${volume_quota: -1} = "G" ] ; then
1553#        (( temp_quota = temp_quota * 1000000000 ))
1554#      elif [ ${volume_quota: -1} = "M" ] ; then
1555#        (( temp_quota = temp_quota * 1000000 ))
1556#      elif [ ${volume_quota: -1} = "k" ] ; then
1557#        (( temp_quota = temp_quota * 1000 ))
1558#      else
1559#        (( temp_quota = volume_quota ))
1560#      fi
1561#    fi
1562#
1563#    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1564##    echo "volume ratio is " $quota_volume
1565#
1566#    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1567#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1568#      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1569#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1570#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1571#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1572#      IGCM_debug_Verif_Exit
1573#    fi
1574#
1575#  fi
1576#
1577## Check of the number of inodes
1578#
1579#  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1580#  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1581#
1582#  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1583#
1584#    unit_avail=${inode_avail: -1}
1585#    unit_quota=${inode_quota: -1}
1586#
1587#    if [ "${unit_quota}" = "*" ] ; then
1588#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1589#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1590#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1591#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1592#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1593#      IGCM_debug_Verif_Exit
1594#    fi
1595#
1596#    temp_avail=${inode_avail%%${inode_avail: -1}*}
1597#    temp_quota=${inode_quota%%${inode_quota: -1}*}
1598#
1599#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1600#
1601#    # Convertion
1602#      if [ ${inode_avail: -1} = "T" ] ; then
1603#        (( temp_avail = temp_avail * 1000000000000 ))
1604#      elif [ ${inode_avail: -1} = "G" ] ; then
1605#        (( temp_avail = temp_avail * 1000000000 ))
1606#      elif [ ${inode_avail: -1} = "M" ] ; then
1607#        (( temp_avail = temp_avail * 1000000 ))
1608#      elif [ ${inode_avail: -1} = "k" ] ; then
1609#        (( temp_avail = temp_avail * 1000 ))
1610#      else
1611#        (( temp_avail = inode_avail ))
1612#      fi
1613#
1614#      if [ ${inode_quota: -1} = "T" ] ; then
1615#        (( temp_quota = temp_quota * 1000000000000 ))
1616#      elif [ ${inode_quota: -1} = "G" ] ; then
1617#        (( temp_quota = temp_quota * 1000000000 ))
1618#      elif [ ${inode_quota: -1} = "M" ] ; then
1619#        (( temp_quota = temp_quota * 1000000 ))
1620#      elif [ ${inode_quota: -1} = "k" ] ; then
1621#        (( temp_quota = temp_quota * 1000 ))
1622#      else
1623#        (( temp_quota = inode_quota ))
1624#      fi
1625#    fi
1626#    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1627##    echo "inode ratio is " $quota_inode
1628#
1629#    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1630#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1631#      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1632#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1633#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1634#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1635#      IGCM_debug_Verif_Exit
1636#    fi
1637#  fi
1638  IGCM_debug_PopStack "IGCM_sys_check_quota"
1639}
1640
1641#D-#==================================================
1642#D-function IGCM_sys_projectAccounting
1643#D-* Purpose: store project accounting information in a file
1644#D-* Examples:
1645#D-
1646function IGCM_sys_projectAccounting {
1647  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1648  if ( $DEBUG_sys ) ; then
1649    echo "IGCM_sys_check_quota"
1650  fi
1651#need to implemented on Jean Zay
1652  touch $1
1653#  ssh irene191 /usr/bin/ccc_myproject > $1
1654
1655  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1656}
1657
1658#D-#==================================================
1659#D-function IGCM_sys_getJobSchedulerID
1660#D-* Purpose: Get the job ID during execution
1661#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1662#D-
1663function IGCM_sys_getJobSchedulerID {
1664  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1665  if ( $DEBUG_sys ) ; then
1666    echo "IGCM_sys_getJobSchedulerID"
1667  fi
1668
1669  eval ${1}=${SLURM_JOBID}
1670
1671  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1672}
1673
1674#D-#==================================================
1675#D-function IGCM_sys_GetJobID
1676#D-* Purpose: Get the job ID from the JobName
1677#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1678#D-
1679function IGCM_sys_GetJobID {
1680  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1681  if ( $DEBUG_sys ) ; then
1682    echo "IGCM_sys_GetJobID"
1683  fi
1684
1685  ID=$( squeue -u $user | grep Job_${config_UserChoices_JobName} | gawk ' { print $1}' )
1686
1687
1688  eval ${3}=${ID}
1689  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1690}
1691
1692#D-#==================================================
1693#D-function IGCM_sys_CountJobInQueue
1694#D-* Purpose: Count number of users job
1695#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1696#D-
1697function IGCM_sys_CountJobInQueue {
1698  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1699  if ( $DEBUG_sys ) ; then
1700    echo "IGCM_sys_CountJobInQueue"
1701  fi
1702
1703  # With -f option, the full job name is given in the last column
1704  NbRun=$(squeue -u $user | \
1705      grep -v JOBID | gawk 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1706
1707  eval ${2}=${NbRun}
1708
1709  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1710}
1711
1712#D-#==================================================
1713#D-function IGCM_sys_ListJobInQueue
1714#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1715#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1716#D-
1717function IGCM_sys_ListJobInQueue {
1718  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1719  if ( $DEBUG_sys ) ; then
1720    echo "IGCM_sys_ListJobInQueue"
1721  fi
1722
1723# to be implemented on Jean Zay
1724 set -A JobList $( squeue -u $user | gawk '{print $3}' | grep -v NAME)
1725#  # With -f option, the full job name is given in the last column
1726#  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1727#                                        '( $2  == User      && \
1728#                                           $NF != /TS/      && \
1729#                                           $NF !~ /PACK/    && \
1730#                                           $NF !~ /REBUILD/ && \
1731#                                           $NF !~ /pack/ )     \
1732#                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1733#
1734  eval set -A ${2} ${JobList[*]}
1735
1736  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1737}
1738
1739#D-#==================================================
1740#D-function IGCM_sys_atlas
1741#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1742#D-* Examples:
1743#D-
1744function IGCM_sys_atlas {
1745  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1746  if ( $DEBUG_sys ) ; then
1747    echo "IGCM_sys_atlas :" $@
1748  fi
1749
1750  typeset status
1751# To be implemented on Jean Zay
1752#
1753#  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1754#  status=$?
1755#  if [ ${status} -gt 0 ] ; then
1756#    echo "IGCM_sys_atlas : error code ${status}"
1757#    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1758#    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1759#    IGCM_debug_PopStack "IGCM_sys_atlas"
1760#    return 1
1761#  else
1762#    IGCM_debug_PopStack "IGCM_sys_atlas"
1763#    return 0
1764#  fi
1765#
1766  IGCM_debug_PopStack "IGCM_sys_atlas"
1767}
1768
1769#D-#==================================================
1770#D-function IGCM_sys_rebuild_nemo
1771#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1772#D-* Examples:
1773#D-
1774
1775function IGCM_sys_rebuild_nemo {
1776  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1777  if ( $DEBUG_sys ) ; then
1778    echo "IGCM_sys_rebuild_nemo :" $@
1779  fi
1780
1781  /gpfswork/rech/psl/commun/Tools/rebuild_nemo/bin/rebuild_nemo ${1} ${2}
1782
1783  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1784}
1785
Note: See TracBrowser for help on using the repository browser.