source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_mesoipsl.ksh @ 1558

Last change on this file since 1558 was 1558, checked in by jgipsl, 22 months ago

First version possible to use at spirit and spiritx on IPSL ESPRI MESO cluster.

The execution part in libIGCM_sys_mesoipsl.ksh comes from libIGCM_sys_jeanzay.ksh. It has only been tested for the case MPMD MPI only and SPMD MPI only.
  • Property svn:keywords set to Revision Date Author
File size: 42.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for IPSL ESPRI MESO cluster: spirit and spiritx
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=${MASTER:=$( hostname )}
71# project name
72typeset PROJECT=NONE
73# jobWarningDelay in seconds
74#typeset jobWarningDelay=${PBS_WALLTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=sbatch}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89
90#====================================================
91# Access to module command
92#====================================================
93. /etc/profile.d/modules.sh
94
95#====================================================
96# Set environment tools (ferret, nco, cdo, rebuild, ...)
97#====================================================
98if [ X${TaskType} = Xcomputing ] ; then
99    IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
100else
101    if [ $CENTER == spirit ] ; then
102        # At spirit
103        . /home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
104    elif [ $CENTER == spiritx ] ; then
105        # At spiritx
106        . /ciclad-home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
107    fi
108fi
109
110[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
111[ ! X${TaskType} = Xchecking ] && module list
112
113#====================================================
114# Host specific DIRECTORIES
115#====================================================
116# For rebuild
117if [ $CENTER == spirit ] ; then
118    export PATH=${PATH}:/home/igcmg/rebuild/spirit/modipsl/modeles/IOIPSL/bin/
119else
120    export PATH=${PATH}:/ciclad-home/igcmg/rebuild/spirit/modipsl/modeles/IOIPSL/bin/
121fi
122#====================================================
123#- MirrorlibIGCM for frontend
124typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
125
126#====================================================
127#- libIGCM_POST for frontend
128typeset -r libIGCM_POST=${libIGCM}
129
130#====================================================
131#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
132typeset -r R_EXE="${MODIPSL}/bin"
133
134#====================================================
135#- SUBMIT_DIR : submission dir
136if [ X${SLURM_SUMBIT_DIR} != X ] ; then
137  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
138else
139  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
140fi
141
142#====================================================
143#- IN
144typeset -r R_IN=${R_IN:=/projsu/igcmg/IGCM}
145
146#====================================================
147#- RUN_DIR_PATH : Temporary working directory
148
149if [ $CENTER == spirit ] ; then
150    RUN_DIR_PATH=${RUN_DIR_PATH:=/scratchu/${LOGIN}/RUN_DIR}
151else
152    RUN_DIR_PATH=${RUN_DIR_PATH:=/scratchx/${LOGIN}/RUN_DIR}
153fi
154if [ X${SLURM_JOBID} != X ] ; then
155    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH}/${SLURM_JOBID}_${$}
156else
157    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH}/${$}
158fi
159
160#====================================================
161#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
162# Note that typeset -r can not be set at spirit because it sets the path as read-only
163typeset OUTCOMMAND_PATH=/tmp
164
165#====================================================
166#- HOST_MPIRUN_COMMAND
167typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
168
169#====================================================
170#- Max number of arguments passed to nco operator or demigration command
171UNIX_MAX_LIMIT=360
172
173#====================================================
174#- set PackDefault to false on meso-ipsl
175PackDefault=false
176
177#====================================================
178#- Default number of MPI task for IPSL coupled model
179#- required for backward compatibility
180#-
181DEFAULT_NUM_PROC_OCE=5
182DEFAULT_NUM_PROC_CPL=1
183(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
184
185#D-#==================================================
186#D-function IGCM_sys_defineArchives
187#D-* Purpose:
188#D-* Define ARCHIVE : Dedicated to large files
189#D-* Define STORAGE : Dedicated to small/medium files
190#D-* Define R_OUT   : Output tree located on ARCHIVE
191#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
192#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
193#D-* if SpaceName=TEST nothing special will hapen
194#D-* Examples:
195#D-
196function IGCM_sys_defineArchives {
197  IGCM_debug_PushStack "IGCM_sys_defineArchives"
198
199  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
200    #====================================================
201    #- ARCHIVE (dedicated to large files)
202    ARCHIVE=${config_UserChoices_ARCHIVE}
203  else
204    #====================================================
205    #- ARCHIVE (dedicated to large files)
206    if [ $CENTER == spirit ] ; then
207        # At spirit
208        ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
209    else
210        # At spiritx
211        ARCHIVE=${ARCHIVE:=/homedata/${LOGIN}}
212    fi
213  fi
214
215  if [ ! X${config_UserChoices_STORAGE} = X ]; then
216    #====================================================
217    #- STORAGE (dedicated to small/medium files)
218    STORAGE=${config_UserChoices_STORAGE}
219  else
220    #====================================================
221    #- STORAGE (dedicated to small/medium files)
222    STORAGE=${ARCHIVE}
223  fi
224
225  # ON MESO-IPSL NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
226
227  #====================================================
228  #- R_OUT
229  R_OUT=${ARCHIVE}/IGCM_OUT
230
231  #====================================================
232  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
233  R_FIG=${STORAGE}/IGCM_OUT
234
235  #====================================================
236  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
237  CMIP6_BUF=${STORAGE}/IGCM_OUT
238
239  #====================================================
240  #- R_BUF (ONLY FOR double copy an scratch)
241  R_BUF=${STORAGE}/IGCM_OUT
242
243  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
244  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
245  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
246
247  IGCM_debug_PopStack "IGCM_sys_defineArchives"
248}
249
250#D-#==================================================
251#D-function IGCM_sys_RshArchive
252#D-* Purpose: Archive rsh command
253#D-* Examples:
254#D-
255function IGCM_sys_RshArchive {
256  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
257  /bin/ksh <<-EOF
258    ${@}
259EOF
260  status=$?
261  if [ ${status} -gt 0 ] ; then
262    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
263    IGCM_debug_Exit "IGCM_sys_RshArchive"
264  fi
265  IGCM_debug_PopStack "IGCM_sys_RshArchive"
266}
267
268#D-#==================================================
269#D-function IGCM_sys_RshArchive_NoError
270#D-* Purpose: Archive rsh command, without error
271#D-*          used only in monitoring.job
272#D-* Examples:
273#D-
274function IGCM_sys_RshArchive_NoError {
275  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
276  /bin/ksh <<-EOF
277    ${@} 2> /dev/null
278EOF
279  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
280}
281
282#D-#==================================================
283#D-function IGCM_sys_MkdirArchive
284#D-* Purpose: Mkdir on Archive
285#D-* Examples:
286#D-
287function IGCM_sys_MkdirArchive {
288  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
289  if ( $DEBUG_sys ) ; then
290    echo "IGCM_sys_MkdirArchive :" $@
291  fi
292  #- creation de repertoire sur le serveur fichier
293  if [ ! -d ${1} ]; then
294    \mkdir -p $1
295    status=$?
296
297    if [ ${status} -gt 0 ] ; then
298      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
299      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
300    fi
301  fi
302  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
303}
304
305#D-#==================================================
306#D-function IGCM_sys_TestDirArchive
307#D-* Purpose: Test Directory that must exists on Archive
308#D-* Examples:
309#D-
310function IGCM_sys_TestDirArchive {
311  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
312  if ( $DEBUG_sys ) ; then
313    echo "IGCM_sys_TestDirArchive :" $@
314  fi
315  typeset ExistFlag
316  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
317  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
318  return ${ExistFlag}
319}
320
321#D-#==================================================
322#D-function IGCM_sys_IsFileArchived
323#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
324#D-* Examples:
325#D-
326function IGCM_sys_IsFileArchived {
327  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
328  if ( $DEBUG_sys ) ; then
329    echo "IGCM_sys_IsFileArchived :" $@
330  fi
331  typeset IsArchivedFlag
332  # Never archived for this system
333  IsArchivedFlag=1
334  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
335
336  return ${IsArchivedFlag}
337}
338
339#D-#==================================================
340#D-function IGCM_sys_TestFileArchive
341#D-* Purpose: Test file that must NOT EXISTS on Archive
342#D-* Examples:
343#D-
344function IGCM_sys_TestFileArchive {
345  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
346  typeset ExistFlag
347  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
348  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
349
350  return ${ExistFlag}
351}
352
353#D-#==================================================
354#D-function IGCM_sys_CountFileArchive
355#D-* Purpose: Count files on Archive filesystem
356#D-* Examples:
357#D-
358function IGCM_sys_CountFileArchive {
359  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
360  ls ${@} 2>/dev/null | wc -l
361  if [ $? -gt 0 ] ; then
362    echo "IGCM_sys_CountFileArchive : erreur."
363  fi
364  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
365}
366
367#D-#==================================================
368#D-function IGCM_sys_Tree
369#D-* Purpose: Tree directories with files on ${ARCHIVE}
370#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
371#D-
372function IGCM_sys_Tree {
373  IGCM_debug_PushStack "IGCM_sys_Tree" $@
374  if ( $DEBUG_sys ) ; then
375    echo "IGCM_sys_Tree :" $@
376  fi
377
378  \ls -lR ${@}
379
380  IGCM_debug_PopStack "IGCM_sys_Tree"
381}
382
383#D-#==================================================
384#D-function IGCM_sys_Qsub
385#D-* Purpose: Qsub new job
386#D-* Examples:
387#D-
388function IGCM_sys_Qsub {
389  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
390  if ( $DEBUG_sys ) ; then
391    echo "IGCM_sys_Qsub :" $@
392  fi
393  typeset options status
394  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
395  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
396  status=$?
397
398  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
399  if [ ${status} -gt 0 ] ; then
400    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
401    IGCM_debug_Exit "IGCM_sys_Qsub"
402  else
403    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
404  fi
405  IGCM_debug_PopStack "IGCM_sys_Qsub"
406}
407
408#D-#==================================================
409#D-function IGCM_sys_QsubPost
410#D-* Purpose: Qsub new job on scalaire
411#D-* Examples:
412#D-
413function IGCM_sys_QsubPost {
414  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
415  if ( $DEBUG_sys ) ; then
416    echo "IGCM_sys_QsubPost :" $@
417  fi
418  typeset options status
419  options="-o ${POST_DIR}/${Script_Post_Output}.out  -e ${POST_DIR}/${Script_Post_Output}.out"
420  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
421  status=$?
422
423  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
424  if [ ${status} -gt 0 ] ; then
425    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
426    IGCM_debug_Exit "IGCM_sys_QsubPost"
427  else
428    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
429  fi
430  IGCM_debug_PopStack "IGCM_sys_QsubPost"
431}
432
433#D-*************************
434#D- File transfer functions
435#D-*************************
436#D-
437
438#D-#==================================================
439#D-function IGCM_sys_RmRunDir
440#D-* Purpose: rm tmpdir (dummy function most of the time batch
441#D-                      scheduler will do the job)
442#D-* Examples:
443#D-
444function IGCM_sys_RmRunDir {
445  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
446  if ( $DEBUG_sys ) ; then
447    echo "IGCM_sys_RmRunDir :" $@
448  fi
449
450  typeset status
451
452  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
453  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
454  status=$?
455
456  if [ ${status} -gt 0 ] ; then
457    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
458    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
459    IGCM_debug_Exit "IGCM_sys_RmRunDir"
460  else
461    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
462  fi
463  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
464}
465
466#D-#==================================================
467#D-function IGCM_sys_Put_Dir
468#D-* Purpose: Copy a complete directory on $(ARCHIVE)
469#D-* Examples:
470#D-
471function IGCM_sys_Put_Dir {
472  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
473  if ( $DEBUG_sys ) ; then
474    echo "IGCM_sys_Put_Dir :" $@
475  fi
476  if [ $DRYRUN = 0 ]; then
477    if [ ! -d ${1} ] ; then
478      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
479      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
480      return
481    fi
482
483    typeset status
484
485    # Only if we use rsync
486    #IGCM_sys_TestDirArchive $( dirname $2 )
487    #
488    #USUAL WAY
489    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
490    status=$?
491
492    if [ ${status} -gt 0 ] ; then
493      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
494      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
495      IGCM_debug_Exit "IGCM_sys_Put_Dir"
496    else
497      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
498    fi
499  fi
500  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
501}
502
503#D-#==================================================
504#D-function IGCM_sys_Get_Dir
505#D-* Purpose: Copy a complete directory from ${ARCHIVE}
506#D-* Examples:
507#D-
508function IGCM_sys_Get_Dir {
509  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
510  if ( $DEBUG_sys ) ; then
511    echo "IGCM_sys_Get_Dir :" $@
512  fi
513  if [ $DRYRUN = 0 ]; then
514    typeset NB_ESSAI DELAI status i
515    # number of tentative
516    NB_ESSAI=3
517    # time delay between tentative
518    DELAI=2
519
520    #
521    # USUAL WAY
522    i=0
523    while [ $i -lt $NB_ESSAI ] ; do
524      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
525      status=$?
526      if [ ${status} -gt 0 ] ; then
527        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
528        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
529        sleep $DELAI
530      else
531        break
532      fi
533      (( i = i + 1 ))
534    done
535
536    if [ ${status} -gt 0 ] ; then
537      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
538      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
539      IGCM_debug_Exit "IGCM_sys_Get_Dir"
540    else
541      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
542    fi
543  fi
544  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
545}
546
547#D-#==================================================
548#D-function IGCM_sys_Put_Rest
549#D-* Purpose: Put computied restarts on ${ARCHIVE}.
550#D-           File and target directory must exist.
551#D-* Examples:
552#D-
553function IGCM_sys_Put_Rest {
554  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
555  if ( $DEBUG_sys ) ; then
556    echo "IGCM_sys_Put_Rest :" $@
557  fi
558  if [ $DRYRUN = 0 ]; then
559    if [ ! -f ${1} ] ; then
560      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
561      IGCM_debug_Exit "IGCM_sys_Put_Rest"
562    fi
563
564    typeset status
565    #
566    # USUAL WAY
567    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
568    status=$?
569
570#       #RSYNC WITH NETWORK SSH CALL
571#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
572#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
573
574#       #RSYNC WITH NFS USE
575#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
576#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
577
578#       status=$?
579#       IGCM_sys_Rsync_out $status
580
581#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
582#       (( status=status+$? ))
583
584    if [ ${status} -gt 0 ] ; then
585      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
586      [ -f ${1} ] && ls -l ${1}
587      [ -f ${2} ] && ls -l ${2}
588      [ -f ${2}/${1} ] && ls -l ${2}/${1}
589      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
590      IGCM_debug_Exit "IGCM_sys_Put_Rest"
591    else
592
593      if [ X${JobType} = XRUN ] ; then
594        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
595        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
596      fi
597
598      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
599    fi
600  fi
601  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
602}
603
604#D-#==================================================
605#D-function IGCM_sys_Put_Out
606#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
607#D-* Examples:
608#D-
609function IGCM_sys_Put_Out {
610  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
611  if ( $DEBUG_sys ) ; then
612    echo "IGCM_sys_Put_Out :" $@
613  fi
614
615  typeset status
616
617  if [ $DRYRUN = 0 ]; then
618    if [ ! -f ${1} ] ; then
619      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
620      IGCM_debug_PopStack "IGCM_sys_Put_Out"
621      return 1
622    fi
623    #
624    IGCM_sys_MkdirArchive $( dirname $2 )
625    #
626    if [ X${JobType} = XRUN ] ; then
627      if [ X${3} = X ] ; then
628        IGCM_sys_Chmod 444 ${1}
629      fi
630    fi
631    #
632
633    echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
634    ${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
635    status=$?
636    IGCM_sys_Rsync_out $status
637
638    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
639    (( status=status+$? ))
640
641    if [ ${status} -gt 0 ] ; then
642      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
643      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
644      IGCM_debug_Exit "IGCM_sys_Put_Out"
645    else
646      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
647    fi
648  fi
649  IGCM_debug_PopStack "IGCM_sys_Put_Out"
650  return 0
651}
652
653#D-#==================================================
654#D-function IGCM_sys_Get
655#D-* Purpose: Get a file from ${ARCHIVE}
656#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
657#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
658function IGCM_sys_Get {
659  IGCM_debug_PushStack "IGCM_sys_Get" $@
660
661  typeset DEST status dm_liste
662
663  if ( $DEBUG_sys ) ; then
664    echo "IGCM_sys_Get :" $@
665  fi
666  if [ $DRYRUN -le 2 ]; then
667    if [ X${1} = X'/l' ] ; then
668      eval set +A dm_liste \${${2}}
669    else
670      dm_liste=${1}
671    fi
672    eval DEST=\${${#}}
673
674    # test if the (first) file is present in the old computation :
675    IGCM_sys_TestFileArchive ${dm_liste[0]}
676    status=$?
677    if [ ${status} -gt 0 ] ; then
678      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
679      IGCM_debug_Exit "IGCM_sys_Get"
680      return
681    fi
682
683    #USUAL WAY
684    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
685    status=$?
686
687    if [ ${status} -gt 0 ] ; then
688      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
689      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
690      IGCM_debug_Exit "IGCM_sys_Get"
691    else
692      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
693    fi
694  fi
695  IGCM_debug_PopStack "IGCM_sys_Get"
696}
697
698#D-#==================================================
699#D-function IGCM_sys_GetDate_Monitoring
700#D-* Purpose: get the last year for which the monitoring has been computed
701#D-* Examples:
702#D-
703function IGCM_sys_GetDate_Monitoring {
704  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
705  if ( $DEBUG_sys ) ; then
706    echo "IGCM_sys_GetDate_Monitoring :" $@
707  fi
708
709  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
710
711  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
712}
713
714#D-#==================================================
715#D-function IGCM_sys_Dods_Rm
716#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
717#D-* Examples:
718#D-
719function IGCM_sys_Dods_Rm {
720  if ( $DEBUG_sys ) ; then
721    echo "IGCM_sys_Dods_Rm :" $@
722  fi
723  return 0
724}
725
726#D-#==================================================
727#D-function IGCM_sys_Dods_Cp
728#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
729#D-* Examples:
730#D-
731function IGCM_sys_Dods_Cp {
732  if ( $DEBUG_sys ) ; then
733    echo "IGCM_sys_Dods_Cp :" $@
734  fi
735  return 0
736}
737
738#D-#==================================================
739#D-function IGCM_sys_Put_Dods
740#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
741#D-* Examples:
742#D-
743function IGCM_sys_Put_Dods {
744  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
745  if ( $DEBUG_sys ) ; then
746    echo "IGCM_sys_Put_Dods :" $@
747  fi
748  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
749}
750
751##############################################################
752# REBUILD OPERATOR
753
754#D-#==================================================
755#D-function IGCM_sys_sync
756#D-* Purpose: flush buffer on disk (dummy function on Ciclad)
757#D-* Examples:
758#D-
759function IGCM_sys_sync {
760  IGCM_debug_PushStack "IGCM_sys_sync" $@
761  if ( $DEBUG_sys ) ; then
762    echo "IGCM_sys_sync :" $@
763    echo "Dummy call, let the system do that."
764  fi
765  IGCM_debug_PopStack "IGCM_sys_sync"
766}
767
768############################################################
769# Activate Running Environnment Variables
770
771#D-#==================================================
772#D-function IGCM_sys_activ_variables
773#D-* Purpose: set environement variables prior to execution
774#D-* Examples:
775#D-
776function IGCM_sys_activ_variables {
777  IGCM_debug_PushStack "IGCM_sys_activ_variables"
778  if ( $DEBUG_sys ) ; then
779    echo "IGCM_sys_activ_variables"
780  fi
781
782# --------------------------------------------------------------------
783#D- MODULE specifications
784# --------------------------------------------------------------------
785  # Source the file EnvFile containing all module load needed to run the model.
786  # EnvFile can be specified in config.card. If this is the case and if the file exists,
787  # it will be used. Otherwise a default file will be used.
788  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
789      # EnvFile is set in config.card and the file exists
790      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
791      EnvFile=${config_UserChoices_EnvFile}
792  else
793      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
794      if [ $CENTER == spirit ] ; then
795          # At spirit
796          EnvFile=/home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
797      else
798          # At spirit
799          EnvFile=/ciclad-home/igcmg/MachineEnvironment/mesoipsl/atlas_env_mesoipsl
800      fi
801  fi
802  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
803  IGCM_debug_Print 1 ${EnvFile}
804  . ${EnvFile}
805
806
807  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for meso-ipsl . "
808  module list
809 
810
811# --------------------------------------------------------------------
812#D- MPI specifications
813# --------------------------------------------------------------------
814
815# --------------------------------------------------------------------
816#D- Other specifications
817# --------------------------------------------------------------------
818
819  ulimit -s unlimited
820
821  IGCM_debug_PopStack "IGCM_sys_activ_variables"
822}
823
824############################################################
825# Desactivate Running Environnment Variables
826
827#D-#==================================================
828#D-function IGCM_sys_desactiv_variables
829#D-* Purpose: unset environement variables after execution
830#D-* Examples:
831#D-
832function IGCM_sys_desactiv_variables {
833  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
834  if ( $DEBUG_sys ) ; then
835    echo "IGCM_sys_desactiv_variables"
836  fi
837# --------------------------------------------------------------------
838#D- MPI specifications
839# --------------------------------------------------------------------
840
841# --------------------------------------------------------------------
842#D- Other specifications
843# --------------------------------------------------------------------
844
845  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
846}
847
848############################################################
849# Update job headers to be used by the scheduler
850
851#D-#==================================================
852#D-function IGCM_sys_updateHeaders
853#D-* Purpose: Update job headers to be used by the scheduler
854#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
855#D-
856function IGCM_sys_updateHeaders {
857  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
858  if ( $DEBUG_sys ) ; then
859    echo "IGCM_sys_updateHeaders"
860  fi
861  typeset file
862  file=$1
863
864  if [ ${executionType} -eq 1 ] ; then
865    # MPMD + MPI
866    sed -e "/::openMPthreads::/d"                  \
867        -e "s/::JobNumProcTot::/${coreNumber}/"    \
868      ${file} > ${file}.tmp
869
870  elif [ ${executionType} -eq 2 ] ; then
871    # MPMD + MPI + OMP
872      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
873      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
874      sed -e "/::openMPthreads::/d"                 \
875        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
876        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
877        -e "/ntasks/i\#SBATCH --exclusive"          \
878         ${file} > ${file}.tmp
879  elif [ ${executionType} -eq 3 ] ; then
880    # SPMD + MPI/OMP
881    sed -e "s/::openMPthreads::/${openMPthreads}/" \
882        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
883      ${file} > ${file}.tmp
884
885  elif [ ${executionType} -eq 4 ] ; then
886    # SPMD + MPI only
887    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
888        -e "/::openMPthreads::/d"                  \
889      ${file} > ${file}.tmp
890
891  elif [ ${executionType} -eq 5 ] ; then
892    # SPMD + OMP only
893    sed -e "s/::openMPthreads::/${openMPthreads}/" \
894        -e "/::JobNumProcTot::/d"                  \
895      ${file} > ${file}.tmp
896
897  elif [ ${executionType} -eq 6 ] ; then
898    # SEQUENTIAL THEN
899    sed -e "s/::JobNumProcTot::/1/"                \
900        -e "/::openMPthreads::/d"                  \
901      ${file} > ${file}.tmp
902
903  fi
904
905  IGCM_sys_Mv ${file}.tmp ${file}
906
907  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
908}
909
910############################################################
911# Build MPI/OMP scripts run file (dummy function)
912
913#D-#==================================================
914#D-function IGCM_sys_build_run_file
915#D-* Purpose: build run file (deprecated)
916#D-* Examples:
917#D-
918function IGCM_sys_build_run_file {
919  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
920}
921
922############################################################
923# Build MPI/OMP scripts
924
925#D-#==================================================
926#D-function IGCM_sys_build_execution_scripts
927#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
928#D-* Examples:
929#D-
930function IGCM_sys_build_execution_scripts
931{
932  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
933  if ( $DEBUG_sys ) ; then
934    echo "IGCM_sys_build_execution_scripts " $@
935  fi
936  IGCM_debug_Print 1 "executionType= ${executionType}"
937  EXECUTION=${HOST_MPIRUN_COMMAND}
938
939  if ( ${OK_PARA_MPMD} ) ; then
940
941    # MPMD mode
942    # 1 MPI only : executionType=1
943    # 2 MPI/OpenMP : executionType=2
944
945    if [ -f run_file ] ; then
946      IGCM_sys_Rm -f run_file
947    fi
948    touch run_file
949
950    # case 1 : Only MPI (MPMD)
951    if  ( ! ${OK_PARA_OMP} ) ; then
952      # Build run_file
953      current_core=0
954      # First loop on the components for the coupler ie oasis (only if oasis3)
955      # the coupler ie oasis3 must be the first one
956      for comp in ${config_ListOfComponents[*]} ; do
957
958        eval ExeNameIn=\${config_Executable_${comp}[0]}
959        eval ExeNameOut=\${config_Executable_${comp}[1]}
960
961        # Only if the component has an executable
962        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
963
964#          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
965#          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
966#          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
967#          (( current_core = ${end_core} + 1 ))
968          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
969          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
970          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
971
972        fi
973      done
974
975      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
976
977      IGCM_sys_Chmod u+x run_file
978      if ( $DEBUG_sys ) ; then
979        echo "run_file contains : "
980        cat run_file
981      fi
982
983    else 
984
985    # 2 MPI/OpenMP : executionType=2
986
987    # MPI-OpenMP (MPMD)
988    # export SLURM_HOSTFILE=./hostlist
989    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
990    # example of  hostlist file :
991    # r3i3n33
992    # r3i3n33
993    # ...
994    # example of run_file :
995    # 0-70 ./prog_lmdz.x.sh %o %t
996    # 71-430 ./prog_opa.xx.sh %o %t
997    # 431-431 ./prog_xios.x.sh %o %t
998    # examples of prog_file :
999    # prog_lmdz.x.sh :
1000    # (( init = 0 + $1 ))
1001    # (( index = init * 10 ))
1002    # (( slot = index % 40 ))
1003    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1004    # that will become
1005    # taskset -c 0-9 ./script_lmdz.x.ksh
1006    # ...
1007    # with script_lmdz.x.ksh
1008    # export OMP_STACKSIZE=3g
1009    # export OMP_PLACES=cores
1010    # export OMP_NUM_THREADS=10
1011    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1012
1013    #  Hosts treatment
1014    _bkIFS=$IFS;
1015        IFS=$'\n'; set -f
1016        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1017        IFS=$_bkIFS; set +f
1018        rm -f hostlist
1019
1020     # Loop on the components to build run_file and script_exec files
1021        rank=0
1022        current_core=0
1023        current_core_mpi=0
1024
1025      for comp in ${config_ListOfComponents[*]} ; do
1026
1027        eval ExeNameIn=\${config_Executable_${comp}[0]}
1028        eval ExeNameOut=\${config_Executable_${comp}[1]}
1029
1030        # Not possible if oasis has an executable (i.e old version of oasis3)
1031        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1032          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1033          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1034          IGCM_debug_Verif_Exit
1035        fi
1036
1037        # Only if we really have an executable for the component :
1038        if [ "X${ExeNameOut}" != X\"\" ] ; then
1039
1040          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1041          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1042          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1043     
1044
1045          # Build script files
1046
1047          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1048          echo ""  >> script_${ExeNameOut}.ksh
1049          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1050          # Check if the number of threads is correct
1051              case ${comp_proc_omp_loc} in
1052                  2|4|5|10|20)
1053                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1054                      ;;
1055                  *)
1056                      IGCM_debug_Exit "ERROR with OMP parameters !"
1057                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1058                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1059                      IGCM_debug_Verif_Exit
1060                      ;;
1061              esac
1062            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1063            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1064            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1065
1066          fi
1067
1068          # to have out/err per process on different files
1069          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1070
1071          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1072
1073        # Build run_file
1074        # Only if the component has an executable
1075        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1076
1077          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1078          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1079          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1080          (( current_core_mpi = ${end_core} + 1 ))
1081        fi
1082
1083          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1084              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1085          else
1086              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1087          fi
1088
1089          # Build configuration file
1090 
1091              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1092          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1093          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1094          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1095              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1096          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1097
1098          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1099
1100        # Build hostlist file
1101
1102          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1103             (( index_host = current_core / NB_CORE_PER_NODE ))
1104             host_value=${listnodes[${index_host}]}
1105               echo "$host_value" >> hostlist
1106             (( current_core = current_core + offset_comp_proc_loc ))
1107          done
1108        fi
1109      done
1110
1111      ## variable added to stop after 60s instead of 600s by default.
1112      ## This is used when no error comes from executables and when something stopped an executable without notice.
1113      export SLURM_WAIT=60
1114
1115      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1116
1117      IGCM_sys_Chmod u+x run_file
1118      if ( $DEBUG_sys ) ; then
1119        echo "run_file contains : "
1120        cat run_file
1121      fi
1122
1123    fi # if ${OK_PARA_MPMD}
1124
1125  else
1126  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1127
1128    for comp in ${config_ListOfComponents[*]} ; do
1129
1130      # Only if we really have an executable for the component :
1131      eval ExeNameOut=\${config_Executable_${comp}[1]}
1132      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1133
1134        # Build script files
1135
1136        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1137        echo ""  >> script_${ExeNameOut}.ksh
1138        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1139
1140        if ( ${OK_PARA_OMP} ) ; then
1141            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1142            # Check if the number of threads is correct
1143            case ${comp_proc_omp_loc} in
1144                2|4|5|10|20)
1145                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1146                    ;;
1147                *)
1148                    IGCM_debug_Exit "ERROR with OMP parameters !"
1149                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1150                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1151                    IGCM_debug_Verif_Exit
1152                    ;;
1153            esac
1154            echo ""  >> script_${ExeNameOut}.ksh
1155            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1156            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1157            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1158        fi
1159
1160        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1161       
1162        # To have out/err per process on different files
1163        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1164        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1165
1166        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
1167        cat script_${ExeNameOut}.ksh
1168
1169      fi
1170
1171    done
1172
1173  fi # ${OK_PARA_MPMD}
1174
1175  IGCM_debug_Print 1 "sys meso-ipsl : execution command is "
1176  IGCM_debug_Print 1 "$EXECUTION"
1177
1178  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1179}
1180
1181#D-#==================================================
1182#D-function IGCM_sys_check_path
1183#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1184#D-* do not point to an important use directory. Stop immediately in that case.
1185#D-* Examples:
1186#D-
1187function IGCM_sys_check_path {
1188  IGCM_debug_PushStack "IGCM_sys_check_path"
1189  if ( $DEBUG_sys ) ; then
1190    echo "IGCM_sys_check_path"
1191  fi
1192
1193  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1194    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1195    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1196    IGCM_debug_Exit "This will stop the job"
1197  fi
1198  IGCM_debug_PopStack "IGCM_sys_check_path"
1199}
1200
1201#D-#==================================================
1202#D-function IGCM_sys_check_quota. Dummy call here
1203#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1204#D-* Examples:
1205#D-
1206function IGCM_sys_check_quota {
1207  IGCM_debug_PushStack "IGCM_sys_check_quota"
1208  if ( $DEBUG_sys ) ; then
1209    echo "IGCM_sys_check_quota"
1210  fi
1211
1212  # TO BE IMPLEMENTED
1213
1214  IGCM_debug_PopStack "IGCM_sys_check_quota"
1215}
1216
1217#D-#==================================================
1218#D-function IGCM_sys_projectAccounting
1219#D-* Purpose: store project accounting information in a file
1220#D-* Examples:
1221#D-
1222function IGCM_sys_projectAccounting {
1223  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1224  if ( $DEBUG_sys ) ; then
1225    echo "IGCM_sys_check_quota"
1226  fi
1227
1228  touch $1
1229
1230  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1231}
1232
1233#D-#==================================================
1234#D-function IGCM_sys_getJobSchedulerID
1235#D-* Purpose: Get the job ID during execution
1236#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1237#D-
1238function IGCM_sys_getJobSchedulerID {
1239  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1240  if ( $DEBUG_sys ) ; then
1241    echo "IGCM_sys_getJobSchedulerID"
1242  fi
1243
1244  eval ${1}=$( echo ${PBS_JOBID} | awk -F. '{print $1}' )
1245
1246  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1247}
1248
1249#D-#==================================================
1250#D-function IGCM_sys_GetJobID
1251#D-* Purpose: Get the job ID from the JobName
1252#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1253#D-
1254function IGCM_sys_GetJobID {
1255  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1256  if ( $DEBUG_sys ) ; then
1257    echo "IGCM_sys_GetJobID"
1258  fi
1259
1260  # With -f option, the full job name is given in the last column
1261  ID="$( qstat -u $2 | grep -w $1 | gawk '-F ' '{print $10}' )"
1262
1263  eval ${3}=${ID}
1264  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1265}
1266
1267#D-#==================================================
1268#D-function IGCM_sys_CountJobInQueue
1269#D-* Purpose: Count number of users job
1270#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1271#D-
1272function IGCM_sys_CountJobInQueue {
1273  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1274  if ( $DEBUG_sys ) ; then
1275    echo "IGCM_sys_CountJobInQueue"
1276  fi
1277  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1278}
1279
1280#D-#==================================================
1281#D-function IGCM_sys_atlas
1282#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1283#D-* Examples:
1284#D-
1285function IGCM_sys_atlas {
1286  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1287  if ( $DEBUG_sys ) ; then
1288    echo "IGCM_sys_atlas :" $@
1289  fi
1290
1291  typeset status
1292
1293  \atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1294  status=$?
1295  if [ ${status} -gt 0 ] ; then
1296    echo "IGCM_sys_atlas : error code ${status}"
1297    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1298    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1299    IGCM_debug_PopStack "IGCM_sys_atlas"
1300    return 1
1301  else
1302    IGCM_debug_PopStack "IGCM_sys_atlas"
1303    return 0
1304  fi
1305
1306  IGCM_debug_PopStack "IGCM_sys_atlas"
1307}
1308
1309#D-#==================================================
1310#D-function IGCM_sys_rebuild_nemo
1311#D-* Purpose: rebuild nemo parallel files with general rebuild
1312#D-* Examples:
1313#D-
1314
1315function IGCM_sys_rebuild_nemo {
1316  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1317  if ( $DEBUG_sys ) ; then
1318    echo "IGCM_sys_rebuild_nemo :" $@
1319  fi
1320
1321  nemo_generic_restart_file_name_out=${1}
1322  nemo_extension_out=${3}
1323  shift ; shift ; shift
1324
1325  IGCM_sys_rebuild -o ${nemo_generic_restart_file_name_out}.${nemo_extension_out} $@
1326
1327  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1328}
1329
1330
1331
Note: See TracBrowser for help on using the repository browser.