source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_obelix.ksh @ 907

Last change on this file since 907 was 900, checked in by labetoulle, 11 years ago

Remove suffix ${CumulPeriod?} from ${JobName?} when resubmitting (#144).
  • Property svn:keywords set to Revision Author Date
File size: 69.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Christian Laguerre
5# Contact: Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for obelix
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=${MASTER:=$( hostname )}
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# Submit command
78typeset SUBMIT=${SUBMIT:=qsub}
79# rsync with path
80typeset -r RSYNC=/usr/bin/rsync
81# RSYNC_opt args to rsync
82typeset -r RSYNC_opt="-va"
83# ie storage filesystem
84typeset -r STOREHOST=${MASTER}
85
86#====================================================
87# Set environment tools (ferret, nco, cdo)
88#====================================================
89. /home/users/brock/.atlas_env_asterix_ksh
90
91#====================================================
92# Set lf95 environment
93#====================================================
94. /usr/local/install/lf6481/bash_laheyfort_setup
95
96#====================================================
97# Host specific DIRECTORIES
98#====================================================
99
100#====================================================
101#- MirrorlibIGCM for frontend
102typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
103
104#====================================================
105#- libIGCM_POST for frontend
106typeset -r libIGCM_POST=${libIGCM}
107
108#====================================================
109#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
110typeset -r R_EXE="${MODIPSL}/bin"
111
112#====================================================
113#- SUBMIT_DIR : submission dir
114if [ X${PBS_O_WORKDIR} != X ] ; then
115  typeset -r SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
116else
117  typeset -r SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
118fi
119
120#====================================================
121#- IN
122typeset -r R_IN=${R_IN:=/home/orchidee01/mmancip/IGCM}
123
124#====================================================
125#- ARCHIVE
126typeset ARCHIVE=${ARCHIVE:=/home/scratch01/${LOGIN}}
127
128#====================================================
129#- STORAGE (dedicated to small/medium files)
130typeset STORAGE=${ARCHIVE}
131
132#====================================================
133#- R_OUT
134typeset R_OUT=${ARCHIVE}/IGCM_OUT
135
136#====================================================
137#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
138typeset R_FIG=${STORAGE}/IGCM_OUT
139
140#====================================================
141#- R_BUF (ONLY FOR double copy an scratch)
142typeset -r R_BUF=${STORAGE}/IGCM_OUT
143
144#====================================================
145#- RUN_DIR_PATH : Temporary working directory (=> TMP)
146if [ X${PBS_JOBID} != X ] ; then
147  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/scratch/$PBS_O_LOGNAME.$PBS_JOBID}
148else
149  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
150fi
151
152#====================================================
153#- HOST_MPIRUN_COMMAND
154typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
155
156#====================================================
157#- Max number of arguments passed to nco operator or demigration command
158UNIX_MAX_LIMIT=360
159
160#====================================================
161#- Number of core per node (max number of OpenMP task)
162NUM_COREPERNODE=1
163
164#====================================================
165#- Default number of MPI task for IPSL coupled model
166#- required for backward compatibility
167#-
168DEFAULT_NUM_PROC_OCE=1
169DEFAULT_NUM_PROC_CPL=1
170(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
171DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
172
173#D-#==================================================
174#D-function IGCM_sys_ChangeArchive
175#D-* Purpose: Just a dummy call on this machine
176#D-* Examples:
177#D-
178function IGCM_sys_ChangeArchive {
179  IGCM_debug_Print 1 " dummy function : IGCM_sys_ChangeArchive "
180}
181
182#D-#==================================================
183#D-function IGCM_sys_RshMaster
184#D-* Purpose: Connection to frontend machine.
185#D-* Examples:
186#D-
187function IGCM_sys_RshMaster {
188  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
189  /bin/ksh  <<-EOF
190    export libIGCM=${libIGCM}
191    export DEBUG_debug=${DEBUG_debug}
192    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
193    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
194    ${@}
195EOF
196  if [ $? -gt 0 ] ; then
197    echo "IGCM_sys_RshMaster : erreur."
198    IGCM_debug_Exit "IGCM_sys_RshMaster"
199  fi
200  IGCM_debug_PopStack "IGCM_sys_RshMaster"
201}
202
203#D-#==================================================
204#D-function IGCM_sys_RshArchive
205#D-* Purpose: Archive rsh command
206#D-* Examples:
207#D-
208function IGCM_sys_RshArchive {
209  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
210  /bin/ksh <<-EOF
211    ${@}
212EOF
213  status=$?
214  if [ ${status} -gt 0 ] ; then
215    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
216    IGCM_debug_Exit "IGCM_sys_RshArchive"
217  fi
218  IGCM_debug_PopStack "IGCM_sys_RshArchive"
219}
220
221#D-#==================================================
222#D-function IGCM_sys_RshPost
223#D-* Purpose: Post-process rsh command
224#D-* Examples:
225#D-
226function IGCM_sys_RshPost {
227  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
228  if ( $DEBUG_sys ) ; then
229    echo "IGCM_sys_RshPost :" $@
230  fi
231
232  # keep standard input (stdin) for the loop onto temporary file
233  cat >tmp_IGCM_sys_RshPost_$$
234
235# ============ FRONTEND START ============ #
236
237  /bin/ksh <tmp_IGCM_sys_RshPost_$$
238  if [ $? -gt 0 ] ; then
239    echo "IGCM_sys_RshPost : erreur."
240    IGCM_debug_Exit "IGCM_sys_RshPost"
241  fi
242  # delete temporary file
243  \rm tmp_IGCM_sys_RshPost_$$
244  IGCM_debug_PopStack "IGCM_sys_RshPost"
245}
246
247#D-#==================================================
248#D-function IGCM_sys_SendMail
249#D-* Purpose: Send mail when simulation is over
250#D-* Examples:
251#D-
252function IGCM_sys_SendMail {
253  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
254  if ( $DEBUG_sys ) ; then
255    echo "IGCM_sys_SendMail :" $@
256  fi
257
258  if ( ${ExitFlag} ) ; then
259    status=failed
260  else
261    status=completed
262  fi
263
264  cat  << END_MAIL > job_end.mail
265Dear ${LOGIN},
266
267  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
268  Job started : ${DateBegin}
269  Job ended   : ${DateEnd}
270  Output files are available in ${R_SAVE}
271  Files to be rebuild are temporarily available in ${REBUILD_DIR}
272  Pre-packed files are temporarily available in ${R_BUFR}
273  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
274END_MAIL
275
276  if [ ! -z ${config_UserChoices_MailName} ] ; then
277    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
278  elif [ -f ~/.forward ] ; then
279    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
280  fi
281
282  sleep 10
283  rm -f job_end.mail
284
285  if [ $? -gt 0 ] ; then
286    echo "IGCM_sys_SendMail : erreur."
287    IGCM_debug_Exit "IGCM_sys_SendMail"
288  fi
289  IGCM_debug_PopStack "IGCM_sys_SendMail"
290}
291
292#D-#==================================================
293#D-function IGCM_sys_Mkdir
294#D-* Purpose: Master locale mkdir command
295#D-* Examples:
296#D-
297function IGCM_sys_Mkdir {
298  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
299  if ( $DEBUG_sys ) ; then
300    echo "IGCM_sys_Mkdir :" $@
301  fi
302  if [ ! -d ${1} ]; then
303    \mkdir -p $1
304    if [ $? -gt 0 ] ; then
305      echo "IGCM_sys_Mkdir : erreur."
306      IGCM_debug_Exit "IGCM_sys_Mkdir"
307    fi
308  fi
309  # vérification :
310  if [ ! -d ${1} ] ; then
311    echo "IGCM_sys_Mkdir : erreur."
312    IGCM_debug_Exit "IGCM_sys_Mkdir"
313  fi
314  IGCM_debug_PopStack "IGCM_sys_Mkdir"
315}
316
317#D-#==================================================
318#D-function IGCM_sys_MkdirArchive
319#D-* Purpose: Mkdir on Archive
320#D-* Examples:
321#D-
322function IGCM_sys_MkdirArchive {
323  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
324  if ( $DEBUG_sys ) ; then
325    echo "IGCM_sys_MkdirArchive :" $@
326  fi
327  #- creation de repertoire sur le serveur fichier
328  if [ ! -d ${1} ]; then
329    mkdir -p $1
330    status=$?
331
332    if [ ${status} -gt 0 ] ; then
333      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
334      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
335    fi
336  fi
337  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
338}
339
340#D-#==================================================
341#D-function IGCM_sys_MkdirWork
342#D-* Purpose: Mkdir on Work
343#D-* Examples:
344#D-
345function IGCM_sys_MkdirWork {
346  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
347  if ( $DEBUG_sys ) ; then
348    echo "IGCM_sys_MkdirWork :" $@
349  fi
350  #- creation de repertoire sur le serveur fichier
351  if [ ! -d ${1} ]; then
352    \mkdir -p $1
353    if [ $? -gt 0 ] ; then
354      echo "IGCM_sys_MkdirWork : erreur."
355      IGCM_debug_Exit "IGCM_sys_MkdirWork"
356    fi
357  fi
358  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
359}
360
361#D-#==================================================
362#D-function IGCM_sys_Cd
363#D-* Purpose: master cd command
364#D-* Examples:
365#D-
366function IGCM_sys_Cd {
367  IGCM_debug_PushStack "IGCM_sys_Cd" $@
368  if ( $DEBUG_sys ) ; then
369    echo "IGCM_sys_Cd :" $@
370  fi
371  \cd $1
372  if [ $? -gt 0 ] ; then
373    echo "IGCM_sys_Cd : erreur."
374    IGCM_debug_Exit "IGCM_sys_Cd"
375  fi
376  IGCM_debug_PopStack "IGCM_sys_Cd"
377}
378
379#D-#==================================================
380#D-function IGCM_sys_Chmod
381#D-* Purpose: Chmod
382#D-* Examples:
383#D-
384function IGCM_sys_Chmod {
385  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
386  if ( $DEBUG_sys ) ; then
387    echo "IGCM_sys_Chmod :" $@
388  fi
389  \chmod $@
390  if [ $? -gt 0 ] ; then
391    echo "IGCM_sys_Chmod : erreur."
392    IGCM_debug_Exit "IGCM_sys_Chmod"
393  fi
394  IGCM_debug_PopStack "IGCM_sys_Chmod"
395}
396
397#D-#==================================================
398#D-function IGCM_sys_FileSize
399#D-* Purpose: Filesize
400#D-* Examples:
401#D-
402function IGCM_sys_FileSize {
403  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
404
405  typeset sizeF
406  set +A sizeF -- $( ls -la ${1} )
407  if [ $? -gt 0 ] ; then
408    IGCM_debug_Exit "IGCM_sys_FileSize"
409  fi
410  eval ${2}=${sizeF[4]}
411
412  IGCM_debug_PopStack "IGCM_sys_FileSize"
413}
414
415#D-#==================================================
416#D-function IGCM_sys_TestDir
417#D-* Purpose: Test Directory that must exists
418#D-* Examples:
419#D-
420function IGCM_sys_TestDir {
421  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
422  if ( $DEBUG_sys ) ; then
423    echo "IGCM_sys_TestDir :" $@
424  fi
425  typeset ExistFlag
426  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
427  IGCM_debug_PopStack "IGCM_sys_TestDir"
428
429  return ${ExistFlag}
430}
431
432#D-#==================================================
433#D-function IGCM_sys_TestDirArchive
434#D-* Purpose: Test Directory that must exists on Archive
435#D-* Examples:
436#D-
437function IGCM_sys_TestDirArchive {
438  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
439  if ( $DEBUG_sys ) ; then
440    echo "IGCM_sys_TestDirArchive :" $@
441  fi
442  typeset ExistFlag
443  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
444  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
445  return ${ExistFlag}
446}
447
448#D-#==================================================
449#D-function IGCM_sys_IsFileArchived
450#D-* Purpose: Test file that must NOT EXISTS on Archive
451#D-* Examples:
452#D-
453function IGCM_sys_IsFileArchived {
454  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
455  if ( $DEBUG_sys ) ; then
456    echo "IGCM_sys_IsFileArchived :" $@
457  fi
458  typeset IsArchivedFlag
459  # Never archived for this system
460  IsArchivedFlag=1
461  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
462
463  return ${IsArchivedFlag}
464}
465
466#D-#==================================================
467#D-function IGCM_sys_TestFileArchive
468#D-* Purpose: Test file that must NOT EXISTS on Archive
469#D-* Examples:
470#D-
471function IGCM_sys_TestFileArchive {
472  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
473  typeset ExistFlag
474  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
475  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
476
477  return ${ExistFlag}
478}
479
480#D-#==================================================
481#D-function IGCM_sys_TestFileBuffer
482#D-* Purpose: Test file that must NOT EXISTS on Buffer
483#D-* Examples:
484#D-
485function IGCM_sys_TestFileBuffer {
486  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
487  typeset ExistFlag
488  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
489  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
490
491  return ${ExistFlag}
492}
493
494#D-#==================================================
495#D-function IGCM_sys_CountFileArchive
496#D-* Purpose: Count files on Archive filesystem
497#D-* Examples:
498#D-
499function IGCM_sys_CountFileArchive {
500  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
501  ls ${@} 2>/dev/null | wc -l
502  if [ $? -gt 0 ] ; then
503    echo "IGCM_sys_CountFileArchive : erreur."
504  fi
505  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
506}
507
508#D-#==================================================
509#D-function IGCM_sys_CountFileBuffer
510#D-* Purpose: Count files on Scratch filesystem
511#D-* Examples:
512#D-
513function IGCM_sys_CountFileBuffer {
514  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
515  ls ${@} 2>/dev/null | wc -l
516  if [ $? -gt 0 ] ; then
517    echo "IGCM_sys_CountFileBuffer : erreur."
518  fi
519  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
520}
521
522#D-#==================================================
523#D-function IGCM_sys_Tree
524#D-* Purpose: Tree directories with files on ${ARCHIVE}
525#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
526#D-
527function IGCM_sys_Tree {
528  IGCM_debug_PushStack "IGCM_sys_Tree" $@
529  if ( $DEBUG_sys ) ; then
530    echo "IGCM_sys_Tree :" $@
531  fi
532
533  \tree -f $@
534
535  IGCM_debug_PopStack "IGCM_sys_Tree"
536}
537
538#D-#==================================================
539#D-function IGCM_sys_Tar
540#D-* Purpose: master tar command
541#D-* Examples:
542#D-
543function IGCM_sys_Tar {
544  IGCM_debug_PushStack "IGCM_sys_Tar" $@
545  if ( $DEBUG_sys ) ; then
546    echo "IGCM_sys_Tar :" $@
547  fi
548  \tar cf $@
549  if [ $? -gt 0 ] ; then
550    echo "IGCM_sys_Tar : erreur."
551    IGCM_debug_Exit "IGCM_sys_Tar"
552  fi
553  IGCM_debug_PopStack "IGCM_sys_Tar"
554}
555
556#D-#==================================================
557#D-function IGCM_sys_UnTar
558#D-* Purpose: master un-tar command
559#D-* Examples:
560#D-
561function IGCM_sys_UnTar {
562  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
563  if ( $DEBUG_sys ) ; then
564    echo "IGCM_sys_UnTar :" $@
565  fi
566  \tar xvf $1
567  if [ $? -gt 0 ] ; then
568    echo "IGCM_sys_UnTar : erreur."
569    IGCM_debug_Exit "IGCM_sys_UnTar"
570  fi
571  IGCM_debug_PopStack "IGCM_sys_UnTar"
572}
573
574#D-#==================================================
575#D-function IGCM_sys_Qsub
576#D-* Purpose: Qsub new job
577#D-* Examples:
578#D-
579function IGCM_sys_Qsub {
580  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
581  if ( $DEBUG_sys ) ; then
582    echo "IGCM_sys_Qsub :" $@
583  fi
584  typeset status
585  #/usr/local/bin/qsub -q medium -o ${SUBMIT_DIR}/${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} < $1
586  /usr/local/bin/qsub -q medium -o ${SUBMIT_DIR}/${Script_Output} -N ${config_UserChoices_JobName} < $1
587  status=$?
588  if [ ${status} -gt 0 ] ; then
589    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
590    IGCM_debug_Exit "IGCM_sys_Qsub"
591  fi
592  IGCM_debug_PopStack "IGCM_sys_Qsub"
593}
594
595#D-#==================================================
596#D-function IGCM_sys_QsubPost
597#D-* Purpose: Qsub new job on scalaire
598#D-* Examples:
599#D-
600function IGCM_sys_QsubPost {
601  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
602  if ( $DEBUG_sys ) ; then
603    echo "IGCM_sys_QsubPost :" $@
604  fi
605  typeset status
606  /usr/local/bin/qsub -q medium -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job -v ${listVarEnv}
607  status=$?
608  if [ ${status} -gt 0 ] ; then
609    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
610    IGCM_debug_Exit "IGCM_sys_QsubPost"
611  fi
612  IGCM_debug_PopStack "IGCM_sys_QsubPost"
613}
614
615#D-*************************
616#D- File transfer functions
617#D-*************************
618#D-
619
620#D-#==================================================
621#D-function IGCM_sys_Rsync_out
622#D-* Purpose: treat return val of rsync
623#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
624#D-  Error values and explanations can depend on your system version.
625function IGCM_sys_Rsync_out {
626  status=$1
627  if [ ! $status ] ; then
628    echo "rsync error !"
629  fi
630
631  if [ $MYLANG = "fr" ]; then
632    case $status in
633    0)  return ;;
634    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
635      echo "Erreur de syntaxe ou d'utilisation."
636      return;;
637    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
638      echo "Incompatibilité de protocole."
639      return;;
640    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
641      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
642      echo "répertoires"
643      return;;
644    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
645      echo "Action demandée non supportée : une tentative de manipulation de"
646      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
647      echo "été faite ; ou une option qui est supportée par le  client  mais"
648      echo "pas par le serveur a été spécifiée."
649      return;;
650    10) echo "Erreur de rsync ; RERR_SOCKETIO"
651      echo "Erreur dans le socket d'entrée sortie"
652      return;;
653    11) echo "Erreur de rsync ; RERR_FILEIO"
654      echo "Erreur d'entrée sortie fichier"
655      return;;
656    12) echo "Erreur de rsync ; RERR_STREAMIO"
657      echo "Erreur dans flux de donnée du protocole rsync"
658      return;;
659    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
660      echo "Erreur avec les diagnostics du programme"
661      return;;
662    14) echo "Erreur de rsync ; RERR_IPC"
663      echo "Erreur dans le code IPC"
664      return;;
665    20) echo "Erreur de rsync ; RERR_SIGNAL"
666      echo "SIGUSR1 ou SIGINT reçu"
667      return;;
668    21) echo "Erreur de rsync ; RERR_WAITCHILD"
669      echo "Une erreur retournée par waitpid()"
670      return;;
671    22) echo "Erreur de rsync ; RERR_MALLOC"
672      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
673      return;;
674    23) echo ""
675      echo "Erreur fichier inexistant"
676      return;;
677    30) echo "Erreur de rsync ; RERR_TIMEOUT"
678      echo "Temps d'attente écoulé dans l'envoi/réception de données"
679      return;;
680    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
681      return;;
682    esac
683  elif [ $MYLANG = "en" ] ; then
684    case $status in
685    0)  return;;
686    1)  echo "rsync error : Syntax or usage error "
687      return;;
688    2)  echo "rsync error : Protocol incompatibility "
689      return;;
690    3)  echo "rsync error : Errors selecting input/output files, dirs"
691      return;;
692    4)  echo "rsync error : Requested action not supported: an attempt"
693      echo "was made to manipulate 64-bit files on a platform that cannot support"
694      echo "them; or an option was specified that is supported by the client and"
695      echo "not by the server."
696      return;;
697    5)  echo "rsync error : Error starting client-server protocol"
698      return;;
699    10) echo "rsync error : Error in socket I/O "
700      return;;
701    11) echo "rsync error : Error in file I/O "
702      return;;
703    12) echo "rsync error : Error in rsync protocol data stream "
704      return;;
705    13) echo "rsync error : Errors with program diagnostics "
706      return;;
707    14) echo "rsync error : Error in IPC code "
708      return;;
709    20) echo "rsync error : Received SIGUSR1 or SIGINT "
710      return;;
711    21) echo "rsync error : Some error returned by waitpid() "
712      return;;
713    22) echo "rsync error : Error allocating core memory buffers "
714      return;;
715    23) echo "rsync error : Partial transfer due to error"
716      return;;
717    24) echo "rsync error : Partial transfer due to vanished source files"
718      return;;
719    30) echo "rsync error : Timeout in data send/receive "
720      return;;
721    *)  echo "rsync error : return code of rsync unknown :" $status
722      return;;
723    esac
724  else
725    echo "unknown language $MYLANG."
726    return
727  fi
728}
729
730#D-#==================================================
731#D-function IGCM_sys_Miror_libIGCM
732#D-* Purpose: Mirror libIGCM PATH and lib to frontend
733#D-* Examples:
734#D-
735function IGCM_sys_Mirror_libIGCM {
736  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
737  if ( $DEBUG_sys ) ; then
738    echo "IGCM_sys_Mirror_libIGCM"
739  fi
740
741  typeset status
742
743  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
744
745  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
746  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
747  status=$?
748
749  if [ ${status} -gt 0 ] ; then
750    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
751    cat out_rsync
752  fi
753  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
754}
755
756#D-#==================================================
757#D-function IGCM_sys_Cp
758#D-* Purpose: generic cp
759#D-* Examples:
760#D-
761function IGCM_sys_Cp {
762  IGCM_debug_PushStack "IGCM_sys_Cp" $@
763  if ( $DEBUG_sys ) ; then
764    echo "IGCM_sys_Cp :" $@
765  fi
766
767  typeset status
768
769  echo cp $@ > out_rsync 2>&1
770  \cp $@ >> out_rsync 2>&1
771  status=$?
772
773  if [ ${status} -gt 0 ] ; then
774    echo "IGCM_sys_Cp : error code ${status}"
775    cat out_rsync
776    IGCM_debug_Exit "IGCM_sys_Cp"
777  else
778    \rm out_rsync
779  fi
780  IGCM_debug_PopStack "IGCM_sys_Cp"
781}
782
783#D-#==================================================
784#D-function IGCM_sys_Rm
785#D-* Purpose: generic rm
786#D-* Examples:
787#D-
788function IGCM_sys_Rm {
789  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
790  if ( $DEBUG_sys ) ; then
791    echo "IGCM_sys_Rm :" $@
792  fi
793
794  typeset status
795
796  echo rm $@ > out_rsync 2>&1
797  \rm $@ >> out_rsync 2>&1
798  status=$?
799
800  if [ ${status} -gt 0 ] ; then
801    echo "IGCM_sys_Rm : error code ${status}"
802    cat out_rsync
803    IGCM_debug_Exit "IGCM_sys_Rm"
804  else
805    \rm out_rsync
806  fi
807  IGCM_debug_PopStack "IGCM_sys_Rm"
808}
809
810#D-#==================================================
811#D-function IGCM_sys_RmRunDir
812#D-* Purpose: rm tmpdir (dummy function most of the time batch
813#D-                      scheduler will do the job)
814#D-* Examples:
815#D-
816function IGCM_sys_RmRunDir {
817  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
818  if ( $DEBUG_sys ) ; then
819    echo "IGCM_sys_RmRunDir :" $@
820  fi
821
822  typeset status
823
824  echo rm $@ > out_rsync 2>&1
825  \rm $@ >> out_rsync 2>&1
826  status=$?
827
828  if [ ${status} -gt 0 ] ; then
829    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
830    cat out_rsync
831    IGCM_debug_Exit "IGCM_sys_RmRunDir"
832  else
833    \rm out_rsync
834  fi
835  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
836}
837
838#D-#==================================================
839#D-function IGCM_sys_Mv
840#D-* Purpose: generic move
841#D-* Examples:
842#D-
843function IGCM_sys_Mv {
844  IGCM_debug_PushStack "IGCM_sys_Mv" $@
845  if ( $DEBUG_sys ) ; then
846    echo "IGCM_sys_Mv :" $@
847  fi
848
849  if [ $DRYRUN = 0 ]; then
850
851    typeset status
852
853    echo mv $@ > out_rsync 2>&1
854    \mv $@ >> out_rsync 2>&1
855    status=$?
856
857    if [ ${status} -gt 0 ] ; then
858      echo "IGCM_sys_Mv : error code ${status}"
859      cat out_rsync
860      IGCM_debug_Exit "IGCM_sys_Mv"
861    else
862      \rm out_rsync
863    fi
864  fi
865  IGCM_debug_PopStack "IGCM_sys_Mv"
866}
867
868#D-#==================================================
869#D-function IGCM_sys_Put_Dir
870#D-* Purpose: Copy a complete directory on $(ARCHIVE)
871#D-* Examples:
872#D-
873function IGCM_sys_Put_Dir {
874  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
875  if ( $DEBUG_sys ) ; then
876    echo "IGCM_sys_Put_Dir :" $@
877  fi
878  if [ $DRYRUN = 0 ]; then
879    if [ ! -d ${1} ] ; then
880      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
881      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
882      return
883    fi
884
885    typeset status
886
887    # Only if we use rsync
888    #IGCM_sys_TestDirArchive $( dirname $2 )
889    #
890    #USUAL WAY
891    \cp -r $1 $2 > out_rsync 2>&1
892    status=$?
893
894    if [ ${status} -gt 0 ] ; then
895      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
896      cat out_rsync
897      IGCM_debug_Exit "IGCM_sys_Put_Dir"
898    else
899      \rm out_rsync
900    fi
901  fi
902  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
903}
904
905#D-#==================================================
906#D-function IGCM_sys_Get_Dir
907#D-* Purpose: Copy a complete directory from ${ARCHIVE}
908#D-* Examples:
909#D-
910function IGCM_sys_Get_Dir {
911  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
912  if ( $DEBUG_sys ) ; then
913    echo "IGCM_sys_Get_Dir :" $@
914  fi
915  if [ $DRYRUN = 0 ]; then
916    if [ ! -d ${1} ] ; then
917      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
918      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
919      return
920    fi
921
922    typeset status
923
924    #USUAL WAY
925    \cp -ur $1 $2 > out_rsync 2>&1
926    status=$?
927
928    if [ ${status} -gt 0 ] ; then
929      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
930      cat out_rsync
931      IGCM_debug_Exit "IGCM_sys_Get_Dir"
932    else
933      \rm out_rsync
934    fi
935  fi
936  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
937}
938
939#D-#==================================================
940#D-function IGCM_sys_Get_Master
941#D-* Purpose: Copy a complete directory from MASTER filesystem
942#D-* Examples:
943#D-
944function IGCM_sys_Get_Master {
945  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
946  if ( $DEBUG_sys ) ; then
947    echo "IGCM_sys_Get_Master :" $@
948  fi
949  if [ $DRYRUN = 0 ]; then
950    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
951      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
952      IGCM_debug_PopStack "IGCM_sys_Get_Master"
953      return
954    fi
955
956    typeset NB_ESSAI DELAI status i
957    # number of tentative
958    NB_ESSAI=3
959    # time delay between tentative
960    DELAI=2
961
962    i=0
963    while [ $i -lt $NB_ESSAI ] ; do
964      \cp -urL $1 $2 > out_rsync 2>&1
965      status=$?
966      if [ ${status} -gt 0 ]; then
967        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
968        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
969        sleep $DELAI
970      else
971        break
972      fi
973      (( i = i + 1 ))
974    done
975
976    if [ ${status} -gt 0 ] ; then
977      echo "IGCM_sys_Get_Master : error."
978      cat out_rsync
979      IGCM_debug_Exit "IGCM_sys_Get_Master"
980    else
981      \rm out_rsync
982    fi
983  fi
984  IGCM_debug_PopStack "IGCM_sys_Get_Master"
985}
986
987#====================================================
988#- Call IGCM_sys_Mirror_libIGCM now !
989if ( $MirrorlibIGCM ) ; then
990  IGCM_sys_Mirror_libIGCM
991fi
992
993#D-#==================================================
994#D-function IGCM_sys_Put_Rest
995#D-* Purpose: Put computied restarts on ${ARCHIVE}.
996#D-           File and target directory must exist.
997#D-* Examples:
998#D-
999function IGCM_sys_Put_Rest {
1000  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1001  if ( $DEBUG_sys ) ; then
1002    echo "IGCM_sys_Put_Rest :" $@
1003  fi
1004  if [ $DRYRUN = 0 ]; then
1005    if [ ! -f ${1} ] ; then
1006      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1007      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1008    fi
1009
1010    typeset status
1011    #
1012    if [ X${JobType} = XRUN ] ; then
1013      IGCM_sys_Chmod 444 ${1}
1014    fi
1015    #
1016    # Only if we use rsync
1017    #IGCM_sys_MkdirArchive $( dirname $2 )
1018    #
1019    #USUAL WAY
1020    \cp $1 $2 > out_rsync 2>&1
1021    status=$?
1022
1023#       #RSYNC WITH NETWORK SSH CALL
1024#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1025#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1026
1027#       #RSYNC WITH NFS USE
1028#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1029#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1030
1031#       status=$?
1032#       IGCM_sys_Rsync_out $status
1033
1034#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1035#       (( status=status+$? ))
1036
1037    if [ ${status} -gt 0 ] ; then
1038      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1039      cat out_rsync
1040      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1041    else
1042      \rm out_rsync
1043    fi
1044  fi
1045  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1046}
1047
1048#D-#==================================================
1049#D-function IGCM_sys_PutBuffer_Rest
1050#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1051#D-           File and target directory must exist.
1052#D-* Examples:
1053#D-
1054function IGCM_sys_PutBuffer_Rest {
1055  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1056  if ( $DEBUG_sys ) ; then
1057    echo "IGCM_sys_PutBuffer_Rest :" $@
1058  fi
1059  if [ $DRYRUN = 0 ]; then
1060    if [ ! -f ${1} ] ; then
1061      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1062      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1063    fi
1064
1065    typeset status
1066    #
1067    if [ X${JobType} = XRUN ] ; then
1068      IGCM_sys_Chmod 444 ${1}
1069    fi
1070
1071    #
1072    # USUAL WAY
1073    \cp $1 $2 > out_rsync 2>&1
1074    status=$?
1075
1076    if [ ${status} -gt 0 ] ; then
1077      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1078      [ -f ${2} ] && ls -l ${2}
1079      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1080      cat out_rsync
1081      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1082    else
1083      \rm out_rsync
1084    fi
1085  fi
1086  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1087}
1088
1089#D-#==================================================
1090#D-function IGCM_sys_PrepareTaredRestart
1091#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1092#D-* Examples:
1093#D-
1094function IGCM_sys_PrepareTaredRestart {
1095  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1096  echo $1
1097  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1098}
1099
1100#D-#==================================================
1101#D-function IGCM_sys_Put_Out
1102#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1103#D-* Examples:
1104#D-
1105function IGCM_sys_Put_Out {
1106  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1107  if ( $DEBUG_sys ) ; then
1108    echo "IGCM_sys_Put_Out :" $@
1109  fi
1110
1111  typeset status
1112
1113  if [ $DRYRUN = 0 ]; then
1114    if [ ! -f ${1} ] ; then
1115      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1116      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1117      return 1
1118    fi
1119    #
1120    IGCM_sys_MkdirArchive $( dirname $2 )
1121    #
1122    if [ X${JobType} = XRUN ] ; then
1123      if [ X${3} = X ] ; then
1124        IGCM_sys_Chmod 444 ${1}
1125      fi
1126    fi
1127    #
1128
1129    echo ${RSYNC} ${RSYNC_opt} $1 $2 > out_rsync 2>&1
1130    ${RSYNC} ${RSYNC_opt} $1 $2 >> out_rsync 2>&1
1131    status=$?
1132    IGCM_sys_Rsync_out $status
1133
1134    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1135    (( status=status+$? ))
1136
1137    if [ ${status} -gt 0 ] ; then
1138      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
1139      cat out_rsync
1140      IGCM_debug_Exit "IGCM_sys_Put_Out"
1141    else
1142      \rm out_rsync
1143    fi
1144  fi
1145  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1146  return 0
1147}
1148
1149#D-#==================================================
1150#D-function IGCM_sys_PutBuffer_Out
1151#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1152#D-* Examples:
1153#D-
1154function IGCM_sys_PutBuffer_Out {
1155  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1156  if ( $DEBUG_sys ) ; then
1157    echo "IGCM_sys_PutBuffer_Out :" $@
1158  fi
1159
1160  typeset NB_ESSAI DELAI status i exist skip
1161
1162  # number of tentative
1163  NB_ESSAI=3
1164  # time delay between tentative
1165  DELAI=2
1166
1167  if [ $DRYRUN = 0 ]; then
1168    if [ ! -f ${1} ] ; then
1169      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1170      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1171      return 1
1172    fi
1173    #
1174    IGCM_sys_Mkdir $( dirname $2 )
1175    #
1176
1177    exist=false
1178    skip=false
1179    if [ -f $2 ] ; then
1180      IGCM_debug_Print 1 "$2 already exist"
1181      exist=true
1182      if [ "X$( diff $1 $2 )" = X ] ; then
1183        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1184        status=0
1185        skip=true
1186      else
1187        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1188        skip=false
1189      fi
1190    fi
1191    #
1192    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1193      IGCM_sys_Chmod u+w $2
1194    fi
1195
1196    if [ X${skip} = Xfalse ] ; then
1197      i=0
1198      while [ $i -lt $NB_ESSAI ] ; do
1199        # USUAL WAY
1200        \cp $1 $2 > out_rsync 2>&1
1201        status=$?
1202        if [ ${status} -gt 0 ]; then
1203          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1204          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1205          [ -f ${2} ] && ls -l ${2}
1206          [ -f ${2}/${1} ] && ls -l ${2}/${1}
1207          sleep $DELAI
1208        else
1209          break
1210        fi
1211        (( i = i + 1 ))
1212      done
1213    fi
1214
1215    if [ ${status} -gt 0 ] ; then
1216      echo "IGCM_sys_PutBuffer_Out : error."
1217      [ -f ${2} ] && ls -l ${2}
1218      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1219      cat out_rsync
1220      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1221    else
1222
1223      if [ X${JobType} = XRUN ] ; then
1224        if [ X${3} = X ] ; then
1225          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1226          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1227        fi
1228      fi
1229
1230      \rm out_rsync
1231    fi
1232  fi
1233  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1234  return 0
1235}
1236
1237#D-#==================================================
1238#D-function IGCM_sys_Get
1239#D-* Purpose: Get a file from ${ARCHIVE}
1240#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1241#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1242function IGCM_sys_Get {
1243  IGCM_debug_PushStack "IGCM_sys_Get" $@
1244
1245  typeset DEST status dm_liste
1246
1247  if ( $DEBUG_sys ) ; then
1248    echo "IGCM_sys_Get :" $@
1249  fi
1250  if [ $DRYRUN -le 2 ]; then
1251    if [ X${1} = X'/l' ] ; then
1252      # test if the first file is present in the old computation :
1253      eval set +A dm_liste \${${2}}
1254    else
1255      dm_liste=${1}
1256    fi
1257    eval DEST=\${${#}}
1258
1259    # test if the (first) file is present in the old computation :
1260    IGCM_sys_TestFileArchive ${dm_liste[0]}
1261    status=$?
1262    if [ ${status} -gt 0 ] ; then
1263      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1264      IGCM_debug_Exit "IGCM_sys_Get"
1265      return
1266    fi
1267
1268    #USUAL WAY
1269    \cp ${dm_liste[*]} ${DEST} > out_rsync 2>&1
1270    status=$?
1271
1272    if [ ${status} -gt 0 ] ; then
1273      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
1274      cat out_rsync
1275      IGCM_debug_Exit "IGCM_sys_Get"
1276    else
1277      \rm out_rsync
1278    fi
1279  fi
1280  IGCM_debug_PopStack "IGCM_sys_Get"
1281}
1282
1283#D-#==================================================
1284#D-function IGCM_sys_GetBuffer
1285#D-* Purpose: Get a file from ${SCRATCHDIR}
1286#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1287#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1288function IGCM_sys_GetBuffer {
1289  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1290
1291  typeset DEST buf_liste target file_work
1292  typeset NB_ESSAI DELAI status i
1293
1294  if ( $DEBUG_sys ) ; then
1295    echo "IGCM_sys_GetBuffer :" $@
1296  fi
1297
1298  # number of tentative
1299  NB_ESSAI=3
1300  # time delay between tentative
1301  DELAI=2
1302
1303  if [ $DRYRUN -le 2 ]; then
1304    if [ X${1} = X'/l' ] ; then
1305      # test if the first file is present in the old computation :
1306      eval set +A buf_liste \${${2}}
1307    else
1308      eval set +A buf_liste ${1}
1309    fi
1310    eval DEST=\${${#}}
1311
1312    #USUAL WAY
1313    if [ X${1} = X'/l' ] ; then
1314      for target in ${buf_liste[*]} ; do
1315        local_file=$( basename ${target} )
1316        i=0
1317        while [ $i -lt $NB_ESSAI ] ; do
1318          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1319          status=$?
1320          if [ ${status} -gt 0 ]; then
1321            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1322            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1323            sleep $DELAI
1324          else
1325            break
1326          fi
1327          (( i = i + 1 ))
1328        done
1329        if [ ${status} -gt 0 ] ; then
1330          echo "IGCM_sys_Get : error"
1331          cat out_rsync
1332          \rm out_rsync
1333          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1334        else
1335          \rm out_rsync
1336        fi
1337      done
1338    else
1339      i=0
1340      while [ $i -lt $NB_ESSAI ] ; do
1341        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1342        status=$?
1343        if [ ${status} -gt 0 ]; then
1344          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1345          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1346          sleep $DELAI
1347        else
1348          break
1349        fi
1350        (( i = i + 1 ))
1351      done
1352      if [ ${status} -gt 0 ] ; then
1353        echo "IGCM_sys_Get : error"
1354        cat out_rsync
1355        \rm out_rsync
1356        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1357      else
1358        \rm out_rsync
1359      fi
1360    fi
1361  fi
1362  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1363}
1364
1365#D-#==================================================
1366#D-function IGCM_sys_GetDate_FichWork
1367#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1368#D-* Examples:
1369#D-
1370function IGCM_sys_GetDate_FichWork {
1371  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1372  if ( $DEBUG_sys ) ; then
1373    echo "IGCM_sys_GetDate_FichWork :" $@
1374  fi
1375  typeset dateF
1376  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1377  eval ${2}=${dateF[5]}
1378
1379  # donne la date filesys d'un fichier sur la machine work
1380  IGCM_debug_PopStack "IGCM_sys_FichWork"
1381}
1382
1383#D-#==================================================
1384#D-function IGCM_sys_GetDate_FichArchive
1385#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1386#D-* Examples:
1387#D-
1388function IGCM_sys_GetDate_FichArchive {
1389  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1390  if ( $DEBUG_sys ) ; then
1391    echo "IGCM_sys_GetDate_FichArchive :" $@
1392  fi
1393  typeset dateF
1394  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1395  eval ${2}=${dateF[5]}
1396
1397  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1398}
1399
1400#D-#==================================================
1401#D-function IGCM_sys_Dods_Rm
1402#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1403#D-* Examples:
1404#D-
1405function IGCM_sys_Dods_Rm {
1406  if ( $DEBUG_sys ) ; then
1407    echo "IGCM_sys_Dods_Rm :" $@
1408  fi
1409  return 0
1410}
1411
1412#D-#==================================================
1413#D-function IGCM_sys_Dods_Cp
1414#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1415#D-* Examples:
1416#D-
1417function IGCM_sys_Dods_Cp {
1418  if ( $DEBUG_sys ) ; then
1419    echo "IGCM_sys_Dods_Cp :" $@
1420  fi
1421  return 0
1422}
1423
1424#D-#==================================================
1425#D-function IGCM_sys_Put_Dods
1426#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
1427#D-* Examples:
1428#D-
1429function IGCM_sys_Put_Dods {
1430  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1431  if ( $DEBUG_sys ) ; then
1432    echo "IGCM_sys_Put_Dods :" $@
1433  fi
1434  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1435}
1436
1437##############################################################
1438# REBUILD OPERATOR
1439
1440#D-#==================================================
1441#D-function IGCM_sys_rebuild
1442#D-* Purpose: rebuild parallel files
1443#D-* Examples:
1444#D-
1445function IGCM_sys_rebuild {
1446  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1447  if ( $DEBUG_sys ) ; then
1448    echo "IGCM_sys_rebuild :" $@
1449  fi
1450
1451  typeset NB_ESSAI DELAI status i firstArg
1452  # number of tentative
1453  NB_ESSAI=3
1454  # time delay between tentative
1455  DELAI=2
1456
1457  i=0
1458  while [ $i -lt $NB_ESSAI ] ; do
1459    /home/users/igcmg/rebuild/bin/rebuild -f -o $@ > out_rsync 2>&1
1460    status=$?
1461    if [ ${status} -gt 0 ] ; then
1462      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1463      cat out_rsync
1464      \rm out_rsync
1465      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1466      firstArg=${1}
1467      \rm ${firstArg}
1468      sleep $DELAI
1469    else
1470      \rm out_rsync
1471      break
1472    fi
1473    (( i = i + 1 ))
1474  done
1475
1476  if [ ${status} -gt 0 ] ; then
1477    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1478    IGCM_debug_Exit "rebuild"
1479  fi
1480
1481  IGCM_debug_PopStack "IGCM_sys_rebuild"
1482}
1483
1484#D-#==================================================
1485#D-function IGCM_sys_rebuild_station
1486#D-* Purpose: rebuild parallel files describing station
1487#D-* Examples:
1488#D-
1489function IGCM_sys_rebuild_station {
1490  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1491  typeset i list_opt file_in file_out prefix_invert list_invert
1492  if ( $DEBUG_sys ) ; then
1493    echo "IGCM_sys_rebuild_station :" $@
1494  fi
1495  list_opt=$@
1496
1497  # Invert Axis : t,x -> x,t
1498  #               t,pres,x -> x,t,pres
1499  # So that we can concatenate along x
1500  i=0
1501  for file_in in ${list_opt} ; do
1502    (( i = i + 1))
1503    [ ${i} = 1 ] && file_out=${file_in} && continue
1504    prefix_invert=$( basename ${file_in} .nc )
1505    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1506    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1507  done
1508
1509  # Concatenate
1510  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1511
1512  # Re-ivert file
1513  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1514
1515  # Station re-ordering is too expansive to be run within libICGM
1516  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1517  # This re-ordering must be done "in memory" by the cmorization process
1518  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1519  # BEGIN reordering
1520
1521  # Only LMDZ text output contains the exact ordering of the station.
1522  # We isolate this in the code below:
1523  #  0  38  -157.5000000000000  70.98591549295774
1524  #  0  54  27.49999999999999   67.18309859154928
1525  #  0  56  -62.50000000000001  82.39436619718309
1526  #  0  79  12.49999999999999   78.59154929577466
1527  #  0  116 -165.0000000000000  76.05633802816901
1528  #  0  117 130.0000000000000   70.98591549295774
1529  #  0  118 110.0000000000000   87.46478873239437
1530  #  1  40  4.999999999999995   51.97183098591550
1531#  typeset iStation iProc list_opt file_in file_out prefix_invert
1532#  typeset -Z4 j4
1533#  typeset -Z3 j3
1534
1535#  unset list_opt
1536#  set +A list_opt $@
1537
1538  # Filename after rebuild
1539#  file_out=${list_opt[0]}
1540  # Prefix of output files
1541#  prefix_invert=$( basename ${file_out} .nc )
1542  # Number of procs
1543#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1544
1545#  iProc=0
1546#  while [ ${iProc} -lt ${num_proc} ] ; do
1547    # Array containing Station as a number
1548#    unset proc_stn
1549#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | awk ' {print $2}' )
1550    # Number of stations produced by processor proc
1551#    stationLast=${#proc_stn[*]}
1552    # Proc number on 4 digits
1553#    j4=${iProc}
1554    # Init
1555#    iStation=0
1556#    while [ ${iStation} -lt ${stationLast} ] ; do
1557      # Station number on 3 digits
1558#      j3=${proc_stn[${iStation}]}
1559      # Extract station
1560      # Invert Axis : t,x -> x,t
1561      #               t,pres,x -> x,t,pres
1562      # So that we can concatenate along x
1563#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1564#      (( iStation = iStation + 1 ))
1565#    done
1566#    (( iProc = iProc + 1 ))
1567#  done
1568
1569  # Concatenate all station along x
1570#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1571
1572  # Re-invert file
1573#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1574
1575  # END reordering
1576
1577  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1578}
1579
1580############################################################
1581# Activate Running Environnment Variables
1582
1583#D-#==================================================
1584#D-function IGCM_sys_desactiv_variables
1585#D-* Purpose: set environement variables prior to execution
1586#D-* Examples:
1587#D-
1588function IGCM_sys_activ_variables {
1589  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1590  if ( $DEBUG_sys ) ; then
1591    echo "IGCM_sys_activ_variables"
1592  fi
1593
1594# --------------------------------------------------------------------
1595#D- MPI specifications
1596# --------------------------------------------------------------------
1597
1598# --------------------------------------------------------------------
1599#D- Other specifications
1600# --------------------------------------------------------------------
1601
1602  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1603}
1604
1605############################################################
1606# Desactivate Running Environnment Variables
1607
1608#D-#==================================================
1609#D-function IGCM_sys_desactiv_variables
1610#D-* Purpose: unset environement variables after execution
1611#D-* Examples:
1612#D-
1613function IGCM_sys_desactiv_variables {
1614  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1615  if ( $DEBUG_sys ) ; then
1616    echo "IGCM_sys_desactiv_variables"
1617  fi
1618# --------------------------------------------------------------------
1619#D- MPI specifications
1620# --------------------------------------------------------------------
1621
1622# --------------------------------------------------------------------
1623#D- Other specifications
1624# --------------------------------------------------------------------
1625
1626  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1627}
1628
1629############################################################
1630# Build MPI/OMP scripts run file (dummy function)
1631
1632#D-#==================================================
1633#D-function IGCM_sys_build_run_file
1634#D-* Purpose: build run file (deprecated)
1635#D-* Examples:
1636#D-
1637function IGCM_sys_build_run_file {
1638
1639  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1640
1641}
1642
1643############################################################
1644# Build MPI/OMP scripts
1645
1646#D-#==================================================
1647#D-function IGCM_sys_build_execution_scripts
1648#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1649#D-* Examples:
1650#D-
1651function IGCM_sys_build_execution_scripts
1652{
1653  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1654  if ( $DEBUG_sys ) ; then
1655    echo "IGCM_sys_build_execution_scripts " $@
1656  fi
1657  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
1658  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1659  typeset num_corempi nombre_restant_node nombre_restant_comp
1660
1661  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1662    IGCM_debug_Exit "IGCM_sys_obelix build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1663  fi
1664
1665  if ( ${OK_PARA_MPMD} ) ; then
1666
1667    if [ -f run_file ] ; then
1668      IGCM_sys_Rm -f run_file
1669    fi
1670    touch run_file
1671
1672    if ( ${OK_PARA_OMP} ) ; then
1673
1674      #  Hosts treatment
1675
1676      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
1677
1678      i=0
1679      rm -f hosts
1680      IGCM_debug_Print 1 "sys Obelix, Hosts avaible :"
1681      for nodes in `cat hosts.tmp` ; do
1682        host[$i]=$nodes
1683        echo "${host[$i]} slots=1 max_slots=1" >> hosts
1684        IGCM_debug_Print 1 ${host[$i]}
1685        i=$((i+1))
1686      done
1687      rm -f hosts.tmp
1688
1689      listnodes=${host[*]}
1690
1691      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
1692
1693      # Initialisation
1694
1695      init_node=y
1696      node_num_current=0
1697      start_num=0
1698      init_exec=n
1699
1700      # Test : if oasis is there, we put it at the first position
1701
1702      for comp in ${config_ListOfComponents[*]} ; do
1703
1704        if [ "X${comp}" = "XCPL" ]  ; then
1705
1706          eval ExeNameIn=\${config_Executable_${comp}[0]}
1707          eval ExeNameOut=\${config_Executable_${comp}[1]}
1708
1709          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1710          echo ""  >> script_${ExeNameOut}.ksh
1711          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1712          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1713          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1714          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
1715          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1716
1717          init_node=n
1718
1719          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
1720          node_num_current=0
1721          node_current=${host[${node_num_current}]}
1722
1723          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1724
1725          init_exec=y
1726          start_num=1
1727
1728        fi
1729
1730      done
1731
1732      # Then loop on the components (except for oasis)
1733
1734      for comp in ${config_ListOfComponents[*]} ; do
1735
1736        eval ExeNameIn=\${config_Executable_${comp}[0]}
1737        eval ExeNameOut=\${config_Executable_${comp}[1]}
1738
1739        # Only if we really have an executable for the component :
1740        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
1741
1742          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1743          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1744
1745          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1746          # echo "set -vx" >> script_${ExeNameOut}.ksh
1747          echo ""  >> script_${ExeNameOut}.ksh
1748          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1749          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1750          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1751          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1752          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1753          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1754          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1755
1756          node_num=0
1757
1758          # We define the number of MPI process to be assigned for the component
1759
1760          nombre_restant_comp=${comp_proc_mpi_loc}
1761
1762          # Loop on the allocated nodes
1763
1764          for node in ${listnodes} ; do
1765
1766            # We go to the current node
1767            if [ ${node_num} = ${node_num_current} ] ; then
1768
1769              node_current=${host[${node_num_current}]}
1770
1771              # If first time on the node : initialisation
1772
1773              if [ ${init_node} = y ] ; then
1774                nombre_restant_node=${NUM_COREPERNODE}
1775              fi
1776
1777              # Test on the number of OMP threads
1778
1779              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1780                (( node_num = node_num + 1 ))
1781                node_num_current=${node_num}
1782                init_node=y
1783                continue
1784              fi
1785
1786              # Number of MPI process to assign
1787
1788              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1789
1790              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1791                num_corempi=${nombre_restant_comp}
1792              fi
1793
1794              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1795              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1796
1797              if [ ${init_exec} = y ] ; then
1798                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1799              else
1800                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1801                init_exec=y
1802              fi
1803
1804              ((  start_num = num_corempi + start_num ))
1805
1806            else
1807
1808              (( node_num = node_num + 1 ))
1809              continue
1810            fi
1811
1812            # Test on the number of core/process remaining on the node/component
1813
1814            if [ ${nombre_restant_node} = 0 ] ; then
1815              (( node_num = node_num + 1 ))
1816              node_num_current=${node_num}
1817              init_node=y
1818
1819              if [ ${nombre_restant_comp} = 0 ] ; then
1820                break 1
1821              fi
1822            else
1823
1824              node_num_current=${node_num}
1825              init_node=n
1826
1827              if [ ${nombre_restant_comp} = 0 ] ; then
1828                break 1
1829              fi
1830            fi
1831          done
1832        fi
1833      done
1834
1835    else
1836
1837      # Then first loop on the components for the coupler ie oasis
1838
1839      ## the coupler ie oasis must be the first one
1840      for comp in ${config_ListOfComponents[*]} ; do
1841
1842        eval ExeNameOut=\${config_Executable_${comp}[1]}
1843
1844        # for CPL component only
1845        if [ "X${comp}" = "XCPL" ] ; then
1846          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1847          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1848        fi
1849      done
1850
1851      # Then second loop on the components
1852
1853      for comp in ${config_ListOfComponents[*]} ; do
1854
1855        eval ExeNameOut=\${config_Executable_${comp}[1]}
1856
1857        # Only if we really have an executable for the component and not the coupler ie oasis:
1858        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1859          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1860          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1861        fi
1862      done
1863      IGCM_sys_Chmod u+x run_file
1864
1865      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1866
1867    fi
1868
1869  else # Only one executable. launch it.
1870
1871    for comp in ${config_ListOfComponents[*]} ; do
1872
1873      # Only if we really have an executable for the component :
1874      eval ExeNameOut=\${config_Executable_${comp}[1]}
1875      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1876
1877        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1878        echo ""  >> script_${ExeNameOut}.ksh
1879        if ( ${OK_PARA_OMP} ) ; then
1880          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1881          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1882        fi
1883        if  ( ${OK_PARA_MPI} ) ; then
1884          # Default : mpirun used if nb_proc gt 1
1885          # pour sortie out/err par process
1886          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
1887          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1888          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1889          EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1890        else
1891          # Default : mpirun is NOT used if nb_proc eq 1
1892          # pour sortie out/err par process
1893          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1894          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1895          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1896          EXECUTION="time ./script_${ExeNameOut}.ksh"
1897        fi
1898      fi
1899    done
1900
1901  fi
1902
1903  IGCM_debug_Print 1 "sys Obelix : execution command is"
1904  IGCM_debug_Print 1 "$EXECUTION"
1905
1906  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1907}
1908
1909############################################################
1910# Check of space available on temporary filesytems. Dummy function here
1911
1912#D-#==================================================
1913#D-function IGCM_sys_check_quota. Dummy call here
1914#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1915#D-* Examples:
1916#D-
1917function IGCM_sys_check_quota {
1918  IGCM_debug_PushStack "IGCM_sys_check_quota"
1919  if ( $DEBUG_sys ) ; then
1920    echo "IGCM_sys_check_quota"
1921  fi
1922  IGCM_debug_PopStack "IGCM_sys_check_quota"
1923}
1924
1925#D-#==================================================
1926#D-function IGCM_sys_CountJobInQueue
1927#D-* Purpose: Check if job_name is currently
1928#D-  running or in queue
1929#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1930#D-
1931function IGCM_sys_CountJobInQueue {
1932  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1933  if ( $DEBUG_sys ) ; then
1934    echo "IGCM_sys_CountJobInQueue"
1935  fi
1936  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1937}
1938
1939##############################################################
1940# NCO OPERATOR
1941
1942#D-#==================================================
1943#D-function IGCM_sys_ncap2
1944#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
1945#D-* Examples:
1946#D-
1947function IGCM_sys_ncap2 {
1948  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
1949  if ( $DEBUG_sys ) ; then
1950    echo "IGCM_sys_ncap2 :" $@
1951  fi
1952
1953  typeset NB_ESSAI DELAI status i
1954  # number of tentative
1955  NB_ESSAI=3
1956  # time delay between tentative
1957  DELAI=2
1958
1959  i=0
1960  while [ $i -lt $NB_ESSAI ] ; do
1961    ncap2 "$@" > out_rsync 2>&1
1962    status=$?
1963    if [ ${status} -gt 0 ] ; then
1964      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
1965      cat out_rsync
1966      \rm out_rsync
1967      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1968      sleep $DELAI
1969    else
1970      \rm out_rsync
1971      break
1972    fi
1973    (( i = i + 1 ))
1974  done
1975
1976  if [ ${status} -gt 0 ] ; then
1977    echo "IGCM_sys_ncap2 : ncap2 error"
1978    IGCM_debug_Exit "ncap2"
1979  fi
1980
1981  IGCM_debug_PopStack "IGCM_sys_ncap2"
1982}
1983
1984#D-#==================================================
1985#D-function IGCM_sys_ncatted
1986#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
1987#D-* Examples:
1988#D-
1989function IGCM_sys_ncatted {
1990  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1991  if ( $DEBUG_sys ) ; then
1992    echo "IGCM_sys_ncatted :" $@
1993  fi
1994
1995  typeset NB_ESSAI DELAI status i
1996  # number of tentative
1997  NB_ESSAI=3
1998  # time delay between tentative
1999  DELAI=2
2000
2001  i=0
2002  while [ $i -lt $NB_ESSAI ] ; do
2003    ncatted "$@" > out_rsync 2>&1
2004    status=$?
2005    if [ ${status} -gt 0 ] ; then
2006      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2007      cat out_rsync
2008      \rm out_rsync
2009      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2010      sleep $DELAI
2011    else
2012      \rm out_rsync
2013      break
2014    fi
2015    (( i = i + 1 ))
2016  done
2017
2018  if [ ${status} -gt 0 ] ; then
2019    echo "IGCM_sys_ncatted : ncatted error"
2020    IGCM_debug_Exit "ncatted"
2021  fi
2022
2023  IGCM_debug_PopStack "IGCM_sys_ncatted"
2024}
2025
2026#D-#==================================================
2027#D-function IGCM_sys_ncbo
2028#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
2029#D-* Examples:
2030#D-
2031function IGCM_sys_ncbo {
2032  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2033  if ( $DEBUG_sys ) ; then
2034    echo "IGCM_sys_ncbo :" $@
2035  fi
2036
2037  typeset NB_ESSAI DELAI status i
2038  # number of tentative
2039  NB_ESSAI=3
2040  # time delay between tentative
2041  DELAI=2
2042
2043  i=0
2044  while [ $i -lt $NB_ESSAI ] ; do
2045    ncbo $@ > out_rsync 2>&1
2046    status=$?
2047    if [ ${status} -gt 0 ] ; then
2048      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2049      cat out_rsync
2050      \rm out_rsync
2051      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2052      sleep $DELAI
2053    else
2054      \rm out_rsync
2055      break
2056    fi
2057    (( i = i + 1 ))
2058  done
2059
2060  if [ ${status} -gt 0 ] ; then
2061    echo "IGCM_sys_ncbo : ncbo error"
2062    IGCM_debug_Exit "ncbo"
2063  fi
2064
2065  IGCM_debug_PopStack "IGCM_sys_ncbo"
2066}
2067
2068#D-#==================================================
2069#D-function IGCM_sys_ncdif
2070#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2071#D-* Examples:
2072#D-
2073function IGCM_sys_ncdiff {
2074  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2075  if ( $DEBUG_sys ) ; then
2076    echo "IGCM_sys_ncdiff :" $@
2077  fi
2078
2079  typeset NB_ESSAI DELAI status i
2080  # number of tentative
2081  NB_ESSAI=3
2082  # time delay between tentative
2083  DELAI=2
2084
2085  i=0
2086  while [ $i -lt $NB_ESSAI ] ; do
2087    ncdiff $@ > out_rsync 2>&1
2088    status=$?
2089    if [ ${status} -gt 0 ] ; then
2090      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2091      cat out_rsync
2092      \rm out_rsync
2093      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2094      sleep $DELAI
2095    else
2096      \rm out_rsync
2097      break
2098    fi
2099    (( i = i + 1 ))
2100  done
2101
2102  if [ ${status} -gt 0 ] ; then
2103    echo "IGCM_sys_ncdiff : ncdiff error"
2104    IGCM_debug_Exit "ncdiff"
2105  fi
2106
2107  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2108}
2109
2110#D-#==================================================
2111#D-function IGCM_sys_ncea
2112#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2113#D-* Examples:
2114#D-
2115function IGCM_sys_ncea {
2116  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2117  if ( $DEBUG_sys ) ; then
2118    echo "IGCM_sys_ncea :" $@
2119  fi
2120
2121  typeset NB_ESSAI DELAI status i
2122  # number of tentative
2123  NB_ESSAI=3
2124  # time delay between tentative
2125  DELAI=2
2126
2127  i=0
2128  while [ $i -lt $NB_ESSAI ] ; do
2129    ncea $@ > out_rsync 2>&1
2130    status=$?
2131    if [ ${status} -gt 0 ] ; then
2132      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2133      cat out_rsync
2134      \rm out_rsync
2135      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2136      sleep $DELAI
2137    else
2138      \rm out_rsync
2139      break
2140    fi
2141    (( i = i + 1 ))
2142  done
2143
2144  if [ ${status} -gt 0 ] ; then
2145    echo "IGCM_sys_ncea : ncea error"
2146    IGCM_debug_Exit "ncea"
2147  fi
2148
2149  IGCM_debug_PopStack "IGCM_sys_ncea"
2150}
2151
2152#D-#==================================================
2153#D-function IGCM_sys_ncecat
2154#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2155#D-* Examples:
2156#D-
2157function IGCM_sys_ncecat {
2158  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2159  if ( $DEBUG_sys ) ; then
2160    echo "IGCM_sys_ncecat :" $@
2161  fi
2162
2163  typeset NB_ESSAI DELAI status i
2164  # number of tentative
2165  NB_ESSAI=3
2166  # time delay between tentative
2167  DELAI=2
2168
2169  i=0
2170  while [ $i -lt $NB_ESSAI ] ; do
2171    ncecat $@ > out_rsync 2>&1
2172    status=$?
2173    if [ ${status} -gt 0 ] ; then
2174      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2175      cat out_rsync
2176      \rm out_rsync
2177      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2178      sleep $DELAI
2179    else
2180      \rm out_rsync
2181      break
2182    fi
2183    (( i = i + 1 ))
2184  done
2185
2186  if [ ${status} -gt 0 ] ; then
2187    echo "IGCM_sys_ncecat : ncecat error"
2188    IGCM_debug_Exit "ncecat"
2189  fi
2190
2191  IGCM_debug_PopStack "IGCM_sys_ncecat"
2192}
2193
2194#D-#==================================================
2195#D-function IGCM_sys_ncflint
2196#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2197#D-* Examples:
2198#D-
2199function IGCM_sys_ncflint {
2200  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2201  if ( $DEBUG_sys ) ; then
2202    echo "IGCM_sys_ncflint :" $@
2203  fi
2204
2205  typeset NB_ESSAI DELAI status i
2206  # number of tentative
2207  NB_ESSAI=3
2208  # time delay between tentative
2209  DELAI=2
2210
2211  i=0
2212  while [ $i -lt $NB_ESSAI ] ; do
2213    ncflint $@ > out_rsync 2>&1
2214    status=$?
2215    if [ ${status} -gt 0 ] ; then
2216      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2217      cat out_rsync
2218      \rm out_rsync
2219      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2220      sleep $DELAI
2221    else
2222      \rm out_rsync
2223      break
2224    fi
2225    (( i = i + 1 ))
2226  done
2227
2228  if [ ${status} -gt 0 ] ; then
2229    echo "IGCM_sys_ncflint : ncflint error"
2230    IGCM_debug_Exit "ncflint"
2231  fi
2232
2233  IGCM_debug_PopStack "IGCM_sys_ncflint"
2234}
2235
2236#D-#==================================================
2237#D-function IGCM_sys_ncks
2238#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2239#D-* Examples:
2240#D-
2241function IGCM_sys_ncks {
2242  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2243  if ( $DEBUG_sys ) ; then
2244    echo "IGCM_sys_ncks :" $@
2245  fi
2246
2247  typeset NB_ESSAI DELAI status i
2248  # number of tentative
2249  NB_ESSAI=3
2250  # time delay between tentative
2251  DELAI=2
2252
2253  i=0
2254  while [ $i -lt $NB_ESSAI ] ; do
2255    ncks $@ > out_rsync 2>&1
2256    status=$?
2257    if [ ${status} -gt 0 ] ; then
2258      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2259      cat out_rsync
2260      \rm out_rsync
2261      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2262      sleep $DELAI
2263    else
2264      \rm out_rsync
2265      break
2266    fi
2267    (( i = i + 1 ))
2268  done
2269
2270  if [ ${status} -gt 0 ] ; then
2271    echo "IGCM_sys_ncks : ncks error"
2272    IGCM_debug_Exit "ncks"
2273  fi
2274
2275  IGCM_debug_PopStack "IGCM_sys_ncks"
2276}
2277
2278#D-#==================================================
2279#D-function IGCM_sys_ncpdq
2280#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2281#D-* Examples:
2282#D-
2283function IGCM_sys_ncpdq {
2284  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2285  if ( $DEBUG_sys ) ; then
2286    echo "IGCM_sys_ncpdq :" $@
2287  fi
2288
2289  typeset NB_ESSAI DELAI status i
2290  # number of tentative
2291  NB_ESSAI=3
2292  # time delay between tentative
2293  DELAI=2
2294
2295  i=0
2296  while [ $i -lt $NB_ESSAI ] ; do
2297    ncpdq $@ > out_rsync 2>&1
2298    status=$?
2299    if [ ${status} -gt 0 ] ; then
2300      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2301      cat out_rsync
2302      \rm out_rsync
2303      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2304      sleep $DELAI
2305    else
2306      \rm out_rsync
2307      break
2308    fi
2309    (( i = i + 1 ))
2310  done
2311
2312  if [ ${status} -gt 0 ] ; then
2313    echo "IGCM_sys_ncpdq : ncpdq error"
2314    IGCM_debug_Exit "ncpdq"
2315  fi
2316
2317  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2318}
2319
2320#D-#==================================================
2321#D-function IGCM_sys_ncra
2322#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2323#D-* Examples:
2324#D-
2325function IGCM_sys_ncra {
2326  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2327  if ( $DEBUG_sys ) ; then
2328    echo "IGCM_sys_ncra :" $@
2329  fi
2330
2331  typeset NB_ESSAI DELAI status i
2332  # number of tentative
2333  NB_ESSAI=3
2334  # time delay between tentative
2335  DELAI=2
2336
2337  i=0
2338  while [ $i -lt $NB_ESSAI ] ; do
2339    ncra $@ > out_rsync 2>&1
2340    status=$?
2341    if [ ${status} -gt 0 ] ; then
2342      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2343      cat out_rsync
2344      \rm out_rsync
2345      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2346      sleep $DELAI
2347    else
2348      \rm out_rsync
2349      break
2350    fi
2351    (( i = i + 1 ))
2352  done
2353
2354  if [ ${status} -gt 0 ] ; then
2355    echo "IGCM_sys_ncra : ncra error"
2356    IGCM_debug_Exit "ncra"
2357  fi
2358
2359  IGCM_debug_PopStack "IGCM_sys_ncra"
2360}
2361
2362#D-#==================================================
2363#D-function IGCM_sys_ncrcat
2364#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
2365#D-* Examples:
2366#D-
2367function IGCM_sys_ncrcat {
2368  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2369  if ( $DEBUG_sys ) ; then
2370    echo "IGCM_sys_ncrcat :" $@
2371  fi
2372
2373  typeset NB_ESSAI DELAI status i lastArg
2374  # number of tentative
2375  NB_ESSAI=3
2376  # time delay between tentative
2377  DELAI=2
2378
2379  i=0
2380  while [ $i -lt $NB_ESSAI ] ; do
2381    ncrcat $@ > out_rsync 2>&1
2382    status=$?
2383    if [ ${status} -gt 0 ] ; then
2384      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2385      cat out_rsync
2386      \rm out_rsync
2387      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2388      sleep $DELAI
2389    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
2390      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2391      cat out_rsync
2392      # remove files having corrupted time axis
2393      eval lastArg=\${$#}
2394      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2395      \rm ${lastArg}
2396      \rm out_rsync
2397      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2398      sleep $DELAI
2399    else
2400      \rm out_rsync
2401      break
2402    fi
2403    (( i = i + 1 ))
2404  done
2405
2406  if [ ${status} -gt 0 ] ; then
2407    echo "IGCM_sys_ncrcat : ncrcat error"
2408    #IGCM_debug_Exit "ncrcat"
2409  fi
2410
2411  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2412}
2413
2414#D-#==================================================
2415#D-function IGCM_sys_ncrename
2416#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
2417#D-* Examples:
2418#D-
2419function IGCM_sys_ncrename {
2420  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2421  if ( $DEBUG_sys ) ; then
2422    echo "IGCM_sys_ncrename :" $@
2423  fi
2424
2425  typeset NB_ESSAI DELAI status i
2426  # number of tentative
2427  NB_ESSAI=3
2428  # time delay between tentative
2429  DELAI=2
2430
2431  i=0
2432  while [ $i -lt $NB_ESSAI ] ; do
2433    ncrename $@ > out_rsync 2>&1
2434    status=$?
2435    if [ ${status} -gt 0 ] ; then
2436      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2437      cat out_rsync
2438      \rm out_rsync
2439      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2440      sleep $DELAI
2441    else
2442      \rm out_rsync
2443      break
2444    fi
2445    (( i = i + 1 ))
2446  done
2447
2448  if [ ${status} -gt 0 ] ; then
2449    echo "IGCM_sys_ncrename : ncrename error"
2450    IGCM_debug_Exit "ncrename"
2451  fi
2452
2453  IGCM_debug_PopStack "IGCM_sys_ncrename"
2454}
2455
2456#D-#==================================================
2457#D-function IGCM_sys_ncwa
2458#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
2459#D-* Examples:
2460#D-
2461function IGCM_sys_ncwa {
2462  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2463  if ( $DEBUG_sys ) ; then
2464    echo "IGCM_sys_ncwa :" $@
2465  fi
2466
2467  typeset NB_ESSAI DELAI status i
2468  # number of tentative
2469  NB_ESSAI=3
2470  # time delay between tentative
2471  DELAI=2
2472
2473  i=0
2474  while [ $i -lt $NB_ESSAI ] ; do
2475    ncwa $@ > out_rsync 2>&1
2476    status=$?
2477    if [ ${status} -gt 0 ] ; then
2478      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2479      cat out_rsync
2480      \rm out_rsync
2481      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2482      sleep $DELAI
2483    else
2484      \rm out_rsync
2485      break
2486    fi
2487    (( i = i + 1 ))
2488  done
2489
2490  if [ ${status} -gt 0 ] ; then
2491    echo "IGCM_sys_ncwa : ncwa error"
2492    IGCM_debug_Exit "ncwa"
2493  fi
2494
2495  IGCM_debug_PopStack "IGCM_sys_ncwa"
2496}
2497
2498##############################################################
2499# CDO OPERATOR
2500
2501#D-#==================================================
2502#D-function IGCM_sys_cdo
2503#D-* Purpose: encapsulate cdo call so as to manage error code and retry
2504#D-* Examples:
2505#D-
2506function IGCM_sys_cdo {
2507  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2508  if ( $DEBUG_sys ) ; then
2509    echo "IGCM_sys_cdo :" $@
2510  fi
2511
2512  typeset status
2513
2514  \cdo $@ > out_rsync 2>&1
2515  status=$?
2516  if [ ${status} -gt 0 ] ; then
2517    echo "IGCM_sys_cdo : error code ${status}"
2518    cat out_rsync
2519    \rm out_rsync
2520    IGCM_debug_PopStack "IGCM_sys_cdo"
2521    return 1
2522  else
2523    IGCM_debug_PopStack "IGCM_sys_cdo"
2524    return 0
2525  fi
2526
2527  IGCM_debug_PopStack "IGCM_sys_cdo"
2528}
Note: See TracBrowser for help on using the repository browser.