source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_obelix.ksh @ 918

Last change on this file since 918 was 918, checked in by sdipsl, 11 years ago

Active instrumentation #140 disturbs this kind of call
TarFileLocation?=$( IGCM_sys_PrepareTaredRestart ${TarFileFound?} )
  • Property svn:keywords set to Revision Author Date
File size: 69.7 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Christian Laguerre
5# Contact: Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for obelix
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=${MASTER:=$( hostname )}
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# Submit command
78typeset SUBMIT=${SUBMIT:=qsub}
79# rsync with path
80typeset -r RSYNC=/usr/bin/rsync
81# RSYNC_opt args to rsync
82typeset -r RSYNC_opt="-va"
83# ie storage filesystem
84typeset -r STOREHOST=${MASTER}
85
86#====================================================
87# Set environment tools (ferret, nco, cdo)
88#====================================================
89. /home/users/brock/.atlas_env_asterix_ksh
90
91#====================================================
92# Set lf95 environment
93#====================================================
94. /usr/local/install/lf6481/bash_laheyfort_setup
95
96#====================================================
97# Host specific DIRECTORIES
98#====================================================
99
100#====================================================
101#- MirrorlibIGCM for frontend
102typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
103
104#====================================================
105#- libIGCM_POST for frontend
106typeset -r libIGCM_POST=${libIGCM}
107
108#====================================================
109#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
110typeset -r R_EXE="${MODIPSL}/bin"
111
112#====================================================
113#- SUBMIT_DIR : submission dir
114if [ X${PBS_O_WORKDIR} != X ] ; then
115  typeset -r SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
116else
117  typeset -r SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
118fi
119
120#====================================================
121#- IN
122typeset -r R_IN=${R_IN:=/home/orchidee01/mmancip/IGCM}
123
124#====================================================
125#- ARCHIVE
126typeset ARCHIVE=${ARCHIVE:=/home/scratch01/${LOGIN}}
127
128#====================================================
129#- STORAGE (dedicated to small/medium files)
130typeset STORAGE=${ARCHIVE}
131
132#====================================================
133#- R_OUT
134typeset R_OUT=${ARCHIVE}/IGCM_OUT
135
136#====================================================
137#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
138typeset R_FIG=${STORAGE}/IGCM_OUT
139
140#====================================================
141#- R_BUF (ONLY FOR double copy an scratch)
142typeset -r R_BUF=${STORAGE}/IGCM_OUT
143
144#====================================================
145#- RUN_DIR_PATH : Temporary working directory (=> TMP)
146if [ X${PBS_JOBID} != X ] ; then
147  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/scratch/$PBS_O_LOGNAME.$PBS_JOBID}
148else
149  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
150fi
151
152#====================================================
153#- HOST_MPIRUN_COMMAND
154typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
155
156#====================================================
157#- Max number of arguments passed to nco operator or demigration command
158UNIX_MAX_LIMIT=360
159
160#====================================================
161#- Number of core per node (max number of OpenMP task)
162NUM_COREPERNODE=1
163
164#====================================================
165#- Default number of MPI task for IPSL coupled model
166#- required for backward compatibility
167#-
168DEFAULT_NUM_PROC_OCE=1
169DEFAULT_NUM_PROC_CPL=1
170(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
171DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
172
173#D-#==================================================
174#D-function IGCM_sys_ChangeArchive
175#D-* Purpose: Just a dummy call on this machine
176#D-* Examples:
177#D-
178function IGCM_sys_ChangeArchive {
179  IGCM_debug_Print 1 " dummy function : IGCM_sys_ChangeArchive "
180}
181
182#D-#==================================================
183#D-function IGCM_sys_RshMaster
184#D-* Purpose: Connection to frontend machine.
185#D-* Examples:
186#D-
187function IGCM_sys_RshMaster {
188  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
189  /bin/ksh  <<-EOF
190    export libIGCM=${libIGCM}
191    export DEBUG_debug=${DEBUG_debug}
192    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
193    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
194    ${@}
195EOF
196  if [ $? -gt 0 ] ; then
197    echo "IGCM_sys_RshMaster : erreur."
198    IGCM_debug_Exit "IGCM_sys_RshMaster"
199  fi
200  IGCM_debug_PopStack "IGCM_sys_RshMaster"
201}
202
203#D-#==================================================
204#D-function IGCM_sys_RshArchive
205#D-* Purpose: Archive rsh command
206#D-* Examples:
207#D-
208function IGCM_sys_RshArchive {
209  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
210  /bin/ksh <<-EOF
211    ${@}
212EOF
213  status=$?
214  if [ ${status} -gt 0 ] ; then
215    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
216    IGCM_debug_Exit "IGCM_sys_RshArchive"
217  fi
218  IGCM_debug_PopStack "IGCM_sys_RshArchive"
219}
220
221#D-#==================================================
222#D-function IGCM_sys_RshPost
223#D-* Purpose: Post-process rsh command
224#D-* Examples:
225#D-
226function IGCM_sys_RshPost {
227  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
228  if ( $DEBUG_sys ) ; then
229    echo "IGCM_sys_RshPost :" $@
230  fi
231
232  # keep standard input (stdin) for the loop onto temporary file
233  cat >tmp_IGCM_sys_RshPost_$$
234
235# ============ FRONTEND START ============ #
236
237  /bin/ksh <tmp_IGCM_sys_RshPost_$$
238  if [ $? -gt 0 ] ; then
239    echo "IGCM_sys_RshPost : erreur."
240    IGCM_debug_Exit "IGCM_sys_RshPost"
241  fi
242  # delete temporary file
243  \rm tmp_IGCM_sys_RshPost_$$
244  IGCM_debug_PopStack "IGCM_sys_RshPost"
245}
246
247#D-#==================================================
248#D-function IGCM_sys_SendMail
249#D-* Purpose: Send mail when simulation is over
250#D-* Examples:
251#D-
252function IGCM_sys_SendMail {
253  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
254  if ( $DEBUG_sys ) ; then
255    echo "IGCM_sys_SendMail :" $@
256  fi
257
258  if ( ${ExitFlag} ) ; then
259    status=failed
260  else
261    status=completed
262  fi
263
264  cat  << END_MAIL > job_end.mail
265Dear ${LOGIN},
266
267  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
268  Job started : ${DateBegin}
269  Job ended   : ${DateEnd}
270  Output files are available in ${R_SAVE}
271  Files to be rebuild are temporarily available in ${REBUILD_DIR}
272  Pre-packed files are temporarily available in ${R_BUFR}
273  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
274END_MAIL
275
276  if [ ! -z ${config_UserChoices_MailName} ] ; then
277    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
278  elif [ -f ~/.forward ] ; then
279    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
280  fi
281
282  sleep 10
283  rm -f job_end.mail
284
285  if [ $? -gt 0 ] ; then
286    echo "IGCM_sys_SendMail : erreur."
287    IGCM_debug_Exit "IGCM_sys_SendMail"
288  fi
289  IGCM_debug_PopStack "IGCM_sys_SendMail"
290}
291
292#D-#==================================================
293#D-function IGCM_sys_Mkdir
294#D-* Purpose: Master locale mkdir command
295#D-* Examples:
296#D-
297function IGCM_sys_Mkdir {
298  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
299  if ( $DEBUG_sys ) ; then
300    echo "IGCM_sys_Mkdir :" $@
301  fi
302  if [ ! -d ${1} ]; then
303    \mkdir -p $1
304    if [ $? -gt 0 ] ; then
305      echo "IGCM_sys_Mkdir : erreur."
306      IGCM_debug_Exit "IGCM_sys_Mkdir"
307    fi
308  fi
309  # vérification :
310  if [ ! -d ${1} ] ; then
311    echo "IGCM_sys_Mkdir : erreur."
312    IGCM_debug_Exit "IGCM_sys_Mkdir"
313  fi
314  IGCM_debug_PopStack "IGCM_sys_Mkdir"
315}
316
317#D-#==================================================
318#D-function IGCM_sys_MkdirArchive
319#D-* Purpose: Mkdir on Archive
320#D-* Examples:
321#D-
322function IGCM_sys_MkdirArchive {
323  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
324  if ( $DEBUG_sys ) ; then
325    echo "IGCM_sys_MkdirArchive :" $@
326  fi
327  #- creation de repertoire sur le serveur fichier
328  if [ ! -d ${1} ]; then
329    mkdir -p $1
330    status=$?
331
332    if [ ${status} -gt 0 ] ; then
333      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
334      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
335    fi
336  fi
337  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
338}
339
340#D-#==================================================
341#D-function IGCM_sys_MkdirWork
342#D-* Purpose: Mkdir on Work
343#D-* Examples:
344#D-
345function IGCM_sys_MkdirWork {
346  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
347  if ( $DEBUG_sys ) ; then
348    echo "IGCM_sys_MkdirWork :" $@
349  fi
350  #- creation de repertoire sur le serveur fichier
351  if [ ! -d ${1} ]; then
352    \mkdir -p $1
353    if [ $? -gt 0 ] ; then
354      echo "IGCM_sys_MkdirWork : erreur."
355      IGCM_debug_Exit "IGCM_sys_MkdirWork"
356    fi
357  fi
358  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
359}
360
361#D-#==================================================
362#D-function IGCM_sys_Cd
363#D-* Purpose: master cd command
364#D-* Examples:
365#D-
366function IGCM_sys_Cd {
367  IGCM_debug_PushStack "IGCM_sys_Cd" $@
368  if ( $DEBUG_sys ) ; then
369    echo "IGCM_sys_Cd :" $@
370  fi
371  \cd $1
372  if [ $? -gt 0 ] ; then
373    echo "IGCM_sys_Cd : erreur."
374    IGCM_debug_Exit "IGCM_sys_Cd"
375  fi
376  IGCM_debug_PopStack "IGCM_sys_Cd"
377}
378
379#D-#==================================================
380#D-function IGCM_sys_Chmod
381#D-* Purpose: Chmod
382#D-* Examples:
383#D-
384function IGCM_sys_Chmod {
385  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
386  if ( $DEBUG_sys ) ; then
387    echo "IGCM_sys_Chmod :" $@
388  fi
389  \chmod $@
390  if [ $? -gt 0 ] ; then
391    echo "IGCM_sys_Chmod : erreur."
392    IGCM_debug_Exit "IGCM_sys_Chmod"
393  fi
394  IGCM_debug_PopStack "IGCM_sys_Chmod"
395}
396
397#D-#==================================================
398#D-function IGCM_sys_FileSize
399#D-* Purpose: Filesize
400#D-* Examples:
401#D-
402function IGCM_sys_FileSize {
403  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
404
405  typeset sizeF
406  set +A sizeF -- $( ls -la ${1} )
407  if [ $? -gt 0 ] ; then
408    IGCM_debug_Exit "IGCM_sys_FileSize"
409  fi
410  eval ${2}=${sizeF[4]}
411
412  IGCM_debug_PopStack "IGCM_sys_FileSize"
413}
414
415#D-#==================================================
416#D-function IGCM_sys_TestDir
417#D-* Purpose: Test Directory that must exists
418#D-* Examples:
419#D-
420function IGCM_sys_TestDir {
421  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
422  if ( $DEBUG_sys ) ; then
423    echo "IGCM_sys_TestDir :" $@
424  fi
425  typeset ExistFlag
426  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
427  IGCM_debug_PopStack "IGCM_sys_TestDir"
428
429  return ${ExistFlag}
430}
431
432#D-#==================================================
433#D-function IGCM_sys_TestDirArchive
434#D-* Purpose: Test Directory that must exists on Archive
435#D-* Examples:
436#D-
437function IGCM_sys_TestDirArchive {
438  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
439  if ( $DEBUG_sys ) ; then
440    echo "IGCM_sys_TestDirArchive :" $@
441  fi
442  typeset ExistFlag
443  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
444  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
445  return ${ExistFlag}
446}
447
448#D-#==================================================
449#D-function IGCM_sys_IsFileArchived
450#D-* Purpose: Test file that must NOT EXISTS on Archive
451#D-* Examples:
452#D-
453function IGCM_sys_IsFileArchived {
454  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
455  if ( $DEBUG_sys ) ; then
456    echo "IGCM_sys_IsFileArchived :" $@
457  fi
458  typeset IsArchivedFlag
459  # Never archived for this system
460  IsArchivedFlag=1
461  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
462
463  return ${IsArchivedFlag}
464}
465
466#D-#==================================================
467#D-function IGCM_sys_TestFileArchive
468#D-* Purpose: Test file that must NOT EXISTS on Archive
469#D-* Examples:
470#D-
471function IGCM_sys_TestFileArchive {
472  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
473  typeset ExistFlag
474  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
475  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
476
477  return ${ExistFlag}
478}
479
480#D-#==================================================
481#D-function IGCM_sys_TestFileBuffer
482#D-* Purpose: Test file that must NOT EXISTS on Buffer
483#D-* Examples:
484#D-
485function IGCM_sys_TestFileBuffer {
486  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
487  typeset ExistFlag
488  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
489  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
490
491  return ${ExistFlag}
492}
493
494#D-#==================================================
495#D-function IGCM_sys_CountFileArchive
496#D-* Purpose: Count files on Archive filesystem
497#D-* Examples:
498#D-
499function IGCM_sys_CountFileArchive {
500  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
501  ls ${@} 2>/dev/null | wc -l
502  if [ $? -gt 0 ] ; then
503    echo "IGCM_sys_CountFileArchive : erreur."
504  fi
505  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
506}
507
508#D-#==================================================
509#D-function IGCM_sys_CountFileBuffer
510#D-* Purpose: Count files on Scratch filesystem
511#D-* Examples:
512#D-
513function IGCM_sys_CountFileBuffer {
514  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
515  ls ${@} 2>/dev/null | wc -l
516  if [ $? -gt 0 ] ; then
517    echo "IGCM_sys_CountFileBuffer : erreur."
518  fi
519  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
520}
521
522#D-#==================================================
523#D-function IGCM_sys_Tree
524#D-* Purpose: Tree directories with files on ${ARCHIVE}
525#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
526#D-
527function IGCM_sys_Tree {
528  IGCM_debug_PushStack "IGCM_sys_Tree" $@
529  if ( $DEBUG_sys ) ; then
530    echo "IGCM_sys_Tree :" $@
531  fi
532
533  \tree -f $@
534
535  IGCM_debug_PopStack "IGCM_sys_Tree"
536}
537
538#D-#==================================================
539#D-function IGCM_sys_Tar
540#D-* Purpose: master tar command
541#D-* Examples:
542#D-
543function IGCM_sys_Tar {
544  IGCM_debug_PushStack "IGCM_sys_Tar" $@
545  if ( $DEBUG_sys ) ; then
546    echo "IGCM_sys_Tar :" $@
547  fi
548  \tar cf $@
549  if [ $? -gt 0 ] ; then
550    echo "IGCM_sys_Tar : erreur."
551    IGCM_debug_Exit "IGCM_sys_Tar"
552  fi
553  IGCM_debug_PopStack "IGCM_sys_Tar"
554}
555
556#D-#==================================================
557#D-function IGCM_sys_UnTar
558#D-* Purpose: master un-tar command
559#D-* Examples:
560#D-
561function IGCM_sys_UnTar {
562  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
563  if ( $DEBUG_sys ) ; then
564    echo "IGCM_sys_UnTar :" $@
565  fi
566  \tar xvf $1
567  if [ $? -gt 0 ] ; then
568    echo "IGCM_sys_UnTar : erreur."
569    IGCM_debug_Exit "IGCM_sys_UnTar"
570  fi
571  IGCM_debug_PopStack "IGCM_sys_UnTar"
572}
573
574#D-#==================================================
575#D-function IGCM_sys_Qsub
576#D-* Purpose: Qsub new job
577#D-* Examples:
578#D-
579function IGCM_sys_Qsub {
580  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
581  if ( $DEBUG_sys ) ; then
582    echo "IGCM_sys_Qsub :" $@
583  fi
584  typeset status
585  #/usr/local/bin/qsub -q medium -o ${SUBMIT_DIR}/${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} < $1
586  /usr/local/bin/qsub -q medium -o ${SUBMIT_DIR}/${Script_Output} -N ${config_UserChoices_JobName} < $1
587  status=$?
588  if [ ${status} -gt 0 ] ; then
589    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
590    IGCM_debug_Exit "IGCM_sys_Qsub"
591  fi
592  IGCM_debug_PopStack "IGCM_sys_Qsub"
593}
594
595#D-#==================================================
596#D-function IGCM_sys_QsubPost
597#D-* Purpose: Qsub new job on scalaire
598#D-* Examples:
599#D-
600function IGCM_sys_QsubPost {
601  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
602  if ( $DEBUG_sys ) ; then
603    echo "IGCM_sys_QsubPost :" $@
604  fi
605  typeset status
606  /usr/local/bin/qsub -q medium -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job -v ${listVarEnv}
607  status=$?
608  if [ ${status} -gt 0 ] ; then
609    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
610    IGCM_debug_Exit "IGCM_sys_QsubPost"
611  fi
612  IGCM_debug_PopStack "IGCM_sys_QsubPost"
613}
614
615#D-*************************
616#D- File transfer functions
617#D-*************************
618#D-
619
620#D-#==================================================
621#D-function IGCM_sys_Rsync_out
622#D-* Purpose: treat return val of rsync
623#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
624#D-  Error values and explanations can depend on your system version.
625function IGCM_sys_Rsync_out {
626  status=$1
627  if [ ! $status ] ; then
628    echo "rsync error !"
629  fi
630
631  if [ $MYLANG = "fr" ]; then
632    case $status in
633    0)  return ;;
634    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
635      echo "Erreur de syntaxe ou d'utilisation."
636      return;;
637    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
638      echo "Incompatibilité de protocole."
639      return;;
640    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
641      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
642      echo "répertoires"
643      return;;
644    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
645      echo "Action demandée non supportée : une tentative de manipulation de"
646      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
647      echo "été faite ; ou une option qui est supportée par le  client  mais"
648      echo "pas par le serveur a été spécifiée."
649      return;;
650    10) echo "Erreur de rsync ; RERR_SOCKETIO"
651      echo "Erreur dans le socket d'entrée sortie"
652      return;;
653    11) echo "Erreur de rsync ; RERR_FILEIO"
654      echo "Erreur d'entrée sortie fichier"
655      return;;
656    12) echo "Erreur de rsync ; RERR_STREAMIO"
657      echo "Erreur dans flux de donnée du protocole rsync"
658      return;;
659    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
660      echo "Erreur avec les diagnostics du programme"
661      return;;
662    14) echo "Erreur de rsync ; RERR_IPC"
663      echo "Erreur dans le code IPC"
664      return;;
665    20) echo "Erreur de rsync ; RERR_SIGNAL"
666      echo "SIGUSR1 ou SIGINT reçu"
667      return;;
668    21) echo "Erreur de rsync ; RERR_WAITCHILD"
669      echo "Une erreur retournée par waitpid()"
670      return;;
671    22) echo "Erreur de rsync ; RERR_MALLOC"
672      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
673      return;;
674    23) echo ""
675      echo "Erreur fichier inexistant"
676      return;;
677    30) echo "Erreur de rsync ; RERR_TIMEOUT"
678      echo "Temps d'attente écoulé dans l'envoi/réception de données"
679      return;;
680    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
681      return;;
682    esac
683  elif [ $MYLANG = "en" ] ; then
684    case $status in
685    0)  return;;
686    1)  echo "rsync error : Syntax or usage error "
687      return;;
688    2)  echo "rsync error : Protocol incompatibility "
689      return;;
690    3)  echo "rsync error : Errors selecting input/output files, dirs"
691      return;;
692    4)  echo "rsync error : Requested action not supported: an attempt"
693      echo "was made to manipulate 64-bit files on a platform that cannot support"
694      echo "them; or an option was specified that is supported by the client and"
695      echo "not by the server."
696      return;;
697    5)  echo "rsync error : Error starting client-server protocol"
698      return;;
699    10) echo "rsync error : Error in socket I/O "
700      return;;
701    11) echo "rsync error : Error in file I/O "
702      return;;
703    12) echo "rsync error : Error in rsync protocol data stream "
704      return;;
705    13) echo "rsync error : Errors with program diagnostics "
706      return;;
707    14) echo "rsync error : Error in IPC code "
708      return;;
709    20) echo "rsync error : Received SIGUSR1 or SIGINT "
710      return;;
711    21) echo "rsync error : Some error returned by waitpid() "
712      return;;
713    22) echo "rsync error : Error allocating core memory buffers "
714      return;;
715    23) echo "rsync error : Partial transfer due to error"
716      return;;
717    24) echo "rsync error : Partial transfer due to vanished source files"
718      return;;
719    30) echo "rsync error : Timeout in data send/receive "
720      return;;
721    *)  echo "rsync error : return code of rsync unknown :" $status
722      return;;
723    esac
724  else
725    echo "unknown language $MYLANG."
726    return
727  fi
728}
729
730#D-#==================================================
731#D-function IGCM_sys_Miror_libIGCM
732#D-* Purpose: Mirror libIGCM PATH and lib to frontend
733#D-* Examples:
734#D-
735function IGCM_sys_Mirror_libIGCM {
736  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
737  if ( $DEBUG_sys ) ; then
738    echo "IGCM_sys_Mirror_libIGCM"
739  fi
740
741  typeset status
742
743  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
744
745  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
746  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
747  status=$?
748
749  if [ ${status} -gt 0 ] ; then
750    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
751    cat out_rsync
752  fi
753  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
754}
755
756#D-#==================================================
757#D-function IGCM_sys_Cp
758#D-* Purpose: generic cp
759#D-* Examples:
760#D-
761function IGCM_sys_Cp {
762  IGCM_debug_PushStack "IGCM_sys_Cp" $@
763  if ( $DEBUG_sys ) ; then
764    echo "IGCM_sys_Cp :" $@
765  fi
766
767  typeset status
768
769  echo cp $@ > out_rsync 2>&1
770  \cp $@ >> out_rsync 2>&1
771  status=$?
772
773  if [ ${status} -gt 0 ] ; then
774    echo "IGCM_sys_Cp : error code ${status}"
775    cat out_rsync
776    IGCM_debug_Exit "IGCM_sys_Cp"
777  else
778    \rm out_rsync
779  fi
780  IGCM_debug_PopStack "IGCM_sys_Cp"
781}
782
783#D-#==================================================
784#D-function IGCM_sys_Rm
785#D-* Purpose: generic rm
786#D-* Examples:
787#D-
788function IGCM_sys_Rm {
789  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
790  if ( $DEBUG_sys ) ; then
791    echo "IGCM_sys_Rm :" $@
792  fi
793
794  typeset status
795
796  echo rm $@ > out_rsync 2>&1
797  \rm $@ >> out_rsync 2>&1
798  status=$?
799
800  if [ ${status} -gt 0 ] ; then
801    echo "IGCM_sys_Rm : error code ${status}"
802    cat out_rsync
803    IGCM_debug_Exit "IGCM_sys_Rm"
804  else
805    \rm out_rsync
806  fi
807  IGCM_debug_PopStack "IGCM_sys_Rm"
808}
809
810#D-#==================================================
811#D-function IGCM_sys_RmRunDir
812#D-* Purpose: rm tmpdir (dummy function most of the time batch
813#D-                      scheduler will do the job)
814#D-* Examples:
815#D-
816function IGCM_sys_RmRunDir {
817  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
818  if ( $DEBUG_sys ) ; then
819    echo "IGCM_sys_RmRunDir :" $@
820  fi
821
822  typeset status
823
824  echo rm $@ > out_rsync 2>&1
825  \rm $@ >> out_rsync 2>&1
826  status=$?
827
828  if [ ${status} -gt 0 ] ; then
829    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
830    cat out_rsync
831    IGCM_debug_Exit "IGCM_sys_RmRunDir"
832  else
833    \rm out_rsync
834  fi
835  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
836}
837
838#D-#==================================================
839#D-function IGCM_sys_Mv
840#D-* Purpose: generic move
841#D-* Examples:
842#D-
843function IGCM_sys_Mv {
844  IGCM_debug_PushStack "IGCM_sys_Mv" $@
845  if ( $DEBUG_sys ) ; then
846    echo "IGCM_sys_Mv :" $@
847  fi
848
849  if [ $DRYRUN = 0 ]; then
850
851    typeset status
852
853    echo mv $@ > out_rsync 2>&1
854    \mv $@ >> out_rsync 2>&1
855    status=$?
856
857    if [ ${status} -gt 0 ] ; then
858      echo "IGCM_sys_Mv : error code ${status}"
859      cat out_rsync
860      IGCM_debug_Exit "IGCM_sys_Mv"
861    else
862      \rm out_rsync
863    fi
864  fi
865  IGCM_debug_PopStack "IGCM_sys_Mv"
866}
867
868#D-#==================================================
869#D-function IGCM_sys_Put_Dir
870#D-* Purpose: Copy a complete directory on $(ARCHIVE)
871#D-* Examples:
872#D-
873function IGCM_sys_Put_Dir {
874  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
875  if ( $DEBUG_sys ) ; then
876    echo "IGCM_sys_Put_Dir :" $@
877  fi
878  if [ $DRYRUN = 0 ]; then
879    if [ ! -d ${1} ] ; then
880      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
881      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
882      return
883    fi
884
885    typeset status
886
887    # Only if we use rsync
888    #IGCM_sys_TestDirArchive $( dirname $2 )
889    #
890    #USUAL WAY
891    \cp -r $1 $2 > out_rsync 2>&1
892    status=$?
893
894    if [ ${status} -gt 0 ] ; then
895      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
896      cat out_rsync
897      IGCM_debug_Exit "IGCM_sys_Put_Dir"
898    else
899      \rm out_rsync
900    fi
901  fi
902  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
903}
904
905#D-#==================================================
906#D-function IGCM_sys_Get_Dir
907#D-* Purpose: Copy a complete directory from ${ARCHIVE}
908#D-* Examples:
909#D-
910function IGCM_sys_Get_Dir {
911  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
912  if ( $DEBUG_sys ) ; then
913    echo "IGCM_sys_Get_Dir :" $@
914  fi
915  if [ $DRYRUN = 0 ]; then
916    if [ ! -d ${1} ] ; then
917      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
918      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
919      return
920    fi
921
922    typeset status
923
924    #USUAL WAY
925    \cp -ur $1 $2 > out_rsync 2>&1
926    status=$?
927
928    if [ ${status} -gt 0 ] ; then
929      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
930      cat out_rsync
931      IGCM_debug_Exit "IGCM_sys_Get_Dir"
932    else
933      \rm out_rsync
934    fi
935  fi
936  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
937}
938
939#D-#==================================================
940#D-function IGCM_sys_Get_Master
941#D-* Purpose: Copy a complete directory from MASTER filesystem
942#D-* Examples:
943#D-
944function IGCM_sys_Get_Master {
945  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
946  if ( $DEBUG_sys ) ; then
947    echo "IGCM_sys_Get_Master :" $@
948  fi
949  if [ $DRYRUN = 0 ]; then
950    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
951      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
952      IGCM_debug_PopStack "IGCM_sys_Get_Master"
953      return
954    fi
955
956    typeset NB_ESSAI DELAI status i
957    # number of tentative
958    NB_ESSAI=3
959    # time delay between tentative
960    DELAI=2
961
962    i=0
963    while [ $i -lt $NB_ESSAI ] ; do
964      \cp -urL $1 $2 > out_rsync 2>&1
965      status=$?
966      if [ ${status} -gt 0 ]; then
967        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
968        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
969        sleep $DELAI
970      else
971        break
972      fi
973      (( i = i + 1 ))
974    done
975
976    if [ ${status} -gt 0 ] ; then
977      echo "IGCM_sys_Get_Master : error."
978      cat out_rsync
979      IGCM_debug_Exit "IGCM_sys_Get_Master"
980    else
981      \rm out_rsync
982    fi
983  fi
984  IGCM_debug_PopStack "IGCM_sys_Get_Master"
985}
986
987#====================================================
988#- Call IGCM_sys_Mirror_libIGCM now !
989if ( $MirrorlibIGCM ) ; then
990  IGCM_sys_Mirror_libIGCM
991fi
992
993#D-#==================================================
994#D-function IGCM_sys_Put_Rest
995#D-* Purpose: Put computied restarts on ${ARCHIVE}.
996#D-           File and target directory must exist.
997#D-* Examples:
998#D-
999function IGCM_sys_Put_Rest {
1000  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1001  if ( $DEBUG_sys ) ; then
1002    echo "IGCM_sys_Put_Rest :" $@
1003  fi
1004  if [ $DRYRUN = 0 ]; then
1005    if [ ! -f ${1} ] ; then
1006      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1007      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1008    fi
1009
1010    typeset status
1011    #
1012    if [ X${JobType} = XRUN ] ; then
1013      IGCM_sys_Chmod 444 ${1}
1014    fi
1015    #
1016    # Only if we use rsync
1017    #IGCM_sys_MkdirArchive $( dirname $2 )
1018    #
1019    #USUAL WAY
1020    \cp $1 $2 > out_rsync 2>&1
1021    status=$?
1022
1023#       #RSYNC WITH NETWORK SSH CALL
1024#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1025#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1026
1027#       #RSYNC WITH NFS USE
1028#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1029#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1030
1031#       status=$?
1032#       IGCM_sys_Rsync_out $status
1033
1034#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1035#       (( status=status+$? ))
1036
1037    if [ ${status} -gt 0 ] ; then
1038      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1039      cat out_rsync
1040      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1041    else
1042      \rm out_rsync
1043    fi
1044  fi
1045  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1046}
1047
1048#D-#==================================================
1049#D-function IGCM_sys_PutBuffer_Rest
1050#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1051#D-           File and target directory must exist.
1052#D-* Examples:
1053#D-
1054function IGCM_sys_PutBuffer_Rest {
1055  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1056  if ( $DEBUG_sys ) ; then
1057    echo "IGCM_sys_PutBuffer_Rest :" $@
1058  fi
1059  if [ $DRYRUN = 0 ]; then
1060    if [ ! -f ${1} ] ; then
1061      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1062      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1063    fi
1064
1065    typeset status
1066    #
1067    if [ X${JobType} = XRUN ] ; then
1068      IGCM_sys_Chmod 444 ${1}
1069    fi
1070
1071    #
1072    # USUAL WAY
1073    \cp $1 $2 > out_rsync 2>&1
1074    status=$?
1075
1076    if [ ${status} -gt 0 ] ; then
1077      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1078      [ -f ${2} ] && ls -l ${2}
1079      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1080      cat out_rsync
1081      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1082    else
1083      \rm out_rsync
1084    fi
1085  fi
1086  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1087}
1088
1089#D-#==================================================
1090#D-function IGCM_sys_PrepareTaredRestart
1091#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1092#D-* Examples:
1093#D-
1094function IGCM_sys_PrepareTaredRestart {
1095  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1096
1097  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1098}
1099
1100#D-#==================================================
1101#D-function IGCM_sys_Put_Out
1102#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1103#D-* Examples:
1104#D-
1105function IGCM_sys_Put_Out {
1106  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1107  if ( $DEBUG_sys ) ; then
1108    echo "IGCM_sys_Put_Out :" $@
1109  fi
1110
1111  typeset status
1112
1113  if [ $DRYRUN = 0 ]; then
1114    if [ ! -f ${1} ] ; then
1115      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1116      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1117      return 1
1118    fi
1119    #
1120    IGCM_sys_MkdirArchive $( dirname $2 )
1121    #
1122    if [ X${JobType} = XRUN ] ; then
1123      if [ X${3} = X ] ; then
1124        IGCM_sys_Chmod 444 ${1}
1125      fi
1126    fi
1127    #
1128
1129    echo ${RSYNC} ${RSYNC_opt} $1 $2 > out_rsync 2>&1
1130    ${RSYNC} ${RSYNC_opt} $1 $2 >> out_rsync 2>&1
1131    status=$?
1132    IGCM_sys_Rsync_out $status
1133
1134    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1135    (( status=status+$? ))
1136
1137    if [ ${status} -gt 0 ] ; then
1138      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
1139      cat out_rsync
1140      IGCM_debug_Exit "IGCM_sys_Put_Out"
1141    else
1142      \rm out_rsync
1143    fi
1144  fi
1145  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1146  return 0
1147}
1148
1149#D-#==================================================
1150#D-function IGCM_sys_PutBuffer_Out
1151#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1152#D-* Examples:
1153#D-
1154function IGCM_sys_PutBuffer_Out {
1155  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1156  if ( $DEBUG_sys ) ; then
1157    echo "IGCM_sys_PutBuffer_Out :" $@
1158  fi
1159
1160  typeset NB_ESSAI DELAI status i exist skip
1161
1162  # number of tentative
1163  NB_ESSAI=3
1164  # time delay between tentative
1165  DELAI=2
1166
1167  if [ $DRYRUN = 0 ]; then
1168    if [ ! -f ${1} ] ; then
1169      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1170      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1171      return 1
1172    fi
1173    #
1174    IGCM_sys_Mkdir $( dirname $2 )
1175    #
1176
1177    exist=false
1178    skip=false
1179    if [ -f $2 ] ; then
1180      IGCM_debug_Print 1 "$2 already exist"
1181      exist=true
1182      if [ "X$( diff $1 $2 )" = X ] ; then
1183        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1184        status=0
1185        skip=true
1186      else
1187        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1188        skip=false
1189      fi
1190    fi
1191    #
1192    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1193      IGCM_sys_Chmod u+w $2
1194    fi
1195
1196    if [ X${skip} = Xfalse ] ; then
1197      i=0
1198      while [ $i -lt $NB_ESSAI ] ; do
1199        # USUAL WAY
1200        \cp $1 $2 > out_rsync 2>&1
1201        status=$?
1202        if [ ${status} -gt 0 ]; then
1203          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1204          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1205          [ -f ${2} ] && ls -l ${2}
1206          [ -f ${2}/${1} ] && ls -l ${2}/${1}
1207          sleep $DELAI
1208        else
1209          break
1210        fi
1211        (( i = i + 1 ))
1212      done
1213    fi
1214
1215    if [ ${status} -gt 0 ] ; then
1216      echo "IGCM_sys_PutBuffer_Out : error."
1217      [ -f ${2} ] && ls -l ${2}
1218      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1219      cat out_rsync
1220      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1221    else
1222
1223      if [ X${JobType} = XRUN ] ; then
1224        if [ X${3} = X ] ; then
1225          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1226          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1227        fi
1228      fi
1229
1230      \rm out_rsync
1231    fi
1232  fi
1233  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1234  return 0
1235}
1236
1237#D-#==================================================
1238#D-function IGCM_sys_Get
1239#D-* Purpose: Get a file from ${ARCHIVE}
1240#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1241#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1242function IGCM_sys_Get {
1243  IGCM_debug_PushStack "IGCM_sys_Get" $@
1244
1245  typeset DEST status dm_liste
1246
1247  if ( $DEBUG_sys ) ; then
1248    echo "IGCM_sys_Get :" $@
1249  fi
1250  if [ $DRYRUN -le 2 ]; then
1251    if [ X${1} = X'/l' ] ; then
1252      eval set +A dm_liste \${${2}}
1253    else
1254      dm_liste=${1}
1255    fi
1256    eval DEST=\${${#}}
1257
1258    # test if the (first) file is present in the old computation :
1259    IGCM_sys_TestFileArchive ${dm_liste[0]}
1260    status=$?
1261    if [ ${status} -gt 0 ] ; then
1262      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1263      IGCM_debug_Exit "IGCM_sys_Get"
1264      return
1265    fi
1266
1267    #USUAL WAY
1268    \cp ${dm_liste[*]} ${DEST} > out_rsync 2>&1
1269    status=$?
1270
1271    if [ ${status} -gt 0 ] ; then
1272      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
1273      cat out_rsync
1274      IGCM_debug_Exit "IGCM_sys_Get"
1275    else
1276      \rm out_rsync
1277    fi
1278  fi
1279  IGCM_debug_PopStack "IGCM_sys_Get"
1280}
1281
1282#D-#==================================================
1283#D-function IGCM_sys_GetBuffer
1284#D-* Purpose: Get a file from ${SCRATCHDIR}
1285#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1286#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1287function IGCM_sys_GetBuffer {
1288  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1289
1290  typeset DEST buf_liste target file_work
1291  typeset NB_ESSAI DELAI status i
1292
1293  if ( $DEBUG_sys ) ; then
1294    echo "IGCM_sys_GetBuffer :" $@
1295  fi
1296
1297  # number of tentative
1298  NB_ESSAI=3
1299  # time delay between tentative
1300  DELAI=2
1301
1302  if [ $DRYRUN -le 2 ]; then
1303    if [ X${1} = X'/l' ] ; then
1304      eval set +A buf_liste \${${2}}
1305    else
1306      eval set +A buf_liste ${1}
1307    fi
1308    eval DEST=\${${#}}
1309
1310    #USUAL WAY
1311    if [ X${1} = X'/l' ] ; then
1312      for target in ${buf_liste[*]} ; do
1313        local_file=$( basename ${target} )
1314        i=0
1315        while [ $i -lt $NB_ESSAI ] ; do
1316          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1317          status=$?
1318          if [ ${status} -gt 0 ]; then
1319            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1320            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1321            sleep $DELAI
1322          else
1323            break
1324          fi
1325          (( i = i + 1 ))
1326        done
1327        if [ ${status} -gt 0 ] ; then
1328          echo "IGCM_sys_Get : error"
1329          cat out_rsync
1330          \rm out_rsync
1331          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1332        else
1333          \rm out_rsync
1334        fi
1335      done
1336    else
1337      i=0
1338      while [ $i -lt $NB_ESSAI ] ; do
1339        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1340        status=$?
1341        if [ ${status} -gt 0 ]; then
1342          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1343          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1344          sleep $DELAI
1345        else
1346          break
1347        fi
1348        (( i = i + 1 ))
1349      done
1350      if [ ${status} -gt 0 ] ; then
1351        echo "IGCM_sys_Get : error"
1352        cat out_rsync
1353        \rm out_rsync
1354        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1355      else
1356        \rm out_rsync
1357      fi
1358    fi
1359  fi
1360  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1361}
1362
1363#D-#==================================================
1364#D-function IGCM_sys_GetDate_FichWork
1365#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1366#D-* Examples:
1367#D-
1368function IGCM_sys_GetDate_FichWork {
1369  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1370  if ( $DEBUG_sys ) ; then
1371    echo "IGCM_sys_GetDate_FichWork :" $@
1372  fi
1373  typeset dateF
1374  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1375  eval ${2}=${dateF[5]}
1376
1377  # donne la date filesys d'un fichier sur la machine work
1378  IGCM_debug_PopStack "IGCM_sys_FichWork"
1379}
1380
1381#D-#==================================================
1382#D-function IGCM_sys_GetDate_FichArchive
1383#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1384#D-* Examples:
1385#D-
1386function IGCM_sys_GetDate_FichArchive {
1387  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1388  if ( $DEBUG_sys ) ; then
1389    echo "IGCM_sys_GetDate_FichArchive :" $@
1390  fi
1391  typeset dateF
1392  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1393  eval ${2}=${dateF[5]}
1394
1395  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1396}
1397
1398#D-#==================================================
1399#D-function IGCM_sys_Dods_Rm
1400#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1401#D-* Examples:
1402#D-
1403function IGCM_sys_Dods_Rm {
1404  if ( $DEBUG_sys ) ; then
1405    echo "IGCM_sys_Dods_Rm :" $@
1406  fi
1407  return 0
1408}
1409
1410#D-#==================================================
1411#D-function IGCM_sys_Dods_Cp
1412#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1413#D-* Examples:
1414#D-
1415function IGCM_sys_Dods_Cp {
1416  if ( $DEBUG_sys ) ; then
1417    echo "IGCM_sys_Dods_Cp :" $@
1418  fi
1419  return 0
1420}
1421
1422#D-#==================================================
1423#D-function IGCM_sys_Put_Dods
1424#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
1425#D-* Examples:
1426#D-
1427function IGCM_sys_Put_Dods {
1428  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1429  if ( $DEBUG_sys ) ; then
1430    echo "IGCM_sys_Put_Dods :" $@
1431  fi
1432  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1433}
1434
1435##############################################################
1436# REBUILD OPERATOR
1437
1438#D-#==================================================
1439#D-function IGCM_sys_rebuild
1440#D-* Purpose: rebuild parallel files
1441#D-* Examples:
1442#D-
1443function IGCM_sys_rebuild {
1444  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1445  if ( $DEBUG_sys ) ; then
1446    echo "IGCM_sys_rebuild :" $@
1447  fi
1448
1449  typeset NB_ESSAI DELAI status i firstArg
1450  # number of tentative
1451  NB_ESSAI=3
1452  # time delay between tentative
1453  DELAI=2
1454
1455  i=0
1456  while [ $i -lt $NB_ESSAI ] ; do
1457    /home/users/igcmg/rebuild/bin/rebuild -f -o $@ > out_rsync 2>&1
1458    status=$?
1459    if [ ${status} -gt 0 ] ; then
1460      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1461      cat out_rsync
1462      \rm out_rsync
1463      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1464      firstArg=${1}
1465      \rm ${firstArg}
1466      sleep $DELAI
1467    else
1468      \rm out_rsync
1469      break
1470    fi
1471    (( i = i + 1 ))
1472  done
1473
1474  if [ ${status} -gt 0 ] ; then
1475    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1476    IGCM_debug_Exit "rebuild"
1477  fi
1478
1479  IGCM_debug_PopStack "IGCM_sys_rebuild"
1480}
1481
1482#D-#==================================================
1483#D-function IGCM_sys_rebuild_station
1484#D-* Purpose: rebuild parallel files describing station
1485#D-* Examples:
1486#D-
1487function IGCM_sys_rebuild_station {
1488  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1489  typeset i list_opt file_in file_out prefix_invert list_invert
1490  if ( $DEBUG_sys ) ; then
1491    echo "IGCM_sys_rebuild_station :" $@
1492  fi
1493  list_opt=$@
1494
1495  # Invert Axis : t,x -> x,t
1496  #               t,pres,x -> x,t,pres
1497  # So that we can concatenate along x
1498  i=0
1499  for file_in in ${list_opt} ; do
1500    (( i = i + 1))
1501    [ ${i} = 1 ] && file_out=${file_in} && continue
1502    prefix_invert=$( basename ${file_in} .nc )
1503    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1504    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1505  done
1506
1507  # Concatenate
1508  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1509
1510  # Re-ivert file
1511  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1512
1513  # Station re-ordering is too expansive to be run within libICGM
1514  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1515  # This re-ordering must be done "in memory" by the cmorization process
1516  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1517  # BEGIN reordering
1518
1519  # Only LMDZ text output contains the exact ordering of the station.
1520  # We isolate this in the code below:
1521  #  0  38  -157.5000000000000  70.98591549295774
1522  #  0  54  27.49999999999999   67.18309859154928
1523  #  0  56  -62.50000000000001  82.39436619718309
1524  #  0  79  12.49999999999999   78.59154929577466
1525  #  0  116 -165.0000000000000  76.05633802816901
1526  #  0  117 130.0000000000000   70.98591549295774
1527  #  0  118 110.0000000000000   87.46478873239437
1528  #  1  40  4.999999999999995   51.97183098591550
1529#  typeset iStation iProc list_opt file_in file_out prefix_invert
1530#  typeset -Z4 j4
1531#  typeset -Z3 j3
1532
1533#  unset list_opt
1534#  set +A list_opt $@
1535
1536  # Filename after rebuild
1537#  file_out=${list_opt[0]}
1538  # Prefix of output files
1539#  prefix_invert=$( basename ${file_out} .nc )
1540  # Number of procs
1541#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1542
1543#  iProc=0
1544#  while [ ${iProc} -lt ${num_proc} ] ; do
1545    # Array containing Station as a number
1546#    unset proc_stn
1547#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | gawk ' {print $2}' )
1548    # Number of stations produced by processor proc
1549#    stationLast=${#proc_stn[*]}
1550    # Proc number on 4 digits
1551#    j4=${iProc}
1552    # Init
1553#    iStation=0
1554#    while [ ${iStation} -lt ${stationLast} ] ; do
1555      # Station number on 3 digits
1556#      j3=${proc_stn[${iStation}]}
1557      # Extract station
1558      # Invert Axis : t,x -> x,t
1559      #               t,pres,x -> x,t,pres
1560      # So that we can concatenate along x
1561#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1562#      (( iStation = iStation + 1 ))
1563#    done
1564#    (( iProc = iProc + 1 ))
1565#  done
1566
1567  # Concatenate all station along x
1568#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1569
1570  # Re-invert file
1571#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1572
1573  # END reordering
1574
1575  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1576}
1577
1578############################################################
1579# Activate Running Environnment Variables
1580
1581#D-#==================================================
1582#D-function IGCM_sys_desactiv_variables
1583#D-* Purpose: set environement variables prior to execution
1584#D-* Examples:
1585#D-
1586function IGCM_sys_activ_variables {
1587  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1588  if ( $DEBUG_sys ) ; then
1589    echo "IGCM_sys_activ_variables"
1590  fi
1591
1592# --------------------------------------------------------------------
1593#D- MPI specifications
1594# --------------------------------------------------------------------
1595
1596# --------------------------------------------------------------------
1597#D- Other specifications
1598# --------------------------------------------------------------------
1599
1600  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1601}
1602
1603############################################################
1604# Desactivate Running Environnment Variables
1605
1606#D-#==================================================
1607#D-function IGCM_sys_desactiv_variables
1608#D-* Purpose: unset environement variables after execution
1609#D-* Examples:
1610#D-
1611function IGCM_sys_desactiv_variables {
1612  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1613  if ( $DEBUG_sys ) ; then
1614    echo "IGCM_sys_desactiv_variables"
1615  fi
1616# --------------------------------------------------------------------
1617#D- MPI specifications
1618# --------------------------------------------------------------------
1619
1620# --------------------------------------------------------------------
1621#D- Other specifications
1622# --------------------------------------------------------------------
1623
1624  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1625}
1626
1627############################################################
1628# Build MPI/OMP scripts run file (dummy function)
1629
1630#D-#==================================================
1631#D-function IGCM_sys_build_run_file
1632#D-* Purpose: build run file (deprecated)
1633#D-* Examples:
1634#D-
1635function IGCM_sys_build_run_file {
1636
1637  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1638
1639}
1640
1641############################################################
1642# Build MPI/OMP scripts
1643
1644#D-#==================================================
1645#D-function IGCM_sys_build_execution_scripts
1646#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1647#D-* Examples:
1648#D-
1649function IGCM_sys_build_execution_scripts
1650{
1651  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1652  if ( $DEBUG_sys ) ; then
1653    echo "IGCM_sys_build_execution_scripts " $@
1654  fi
1655  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
1656  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1657  typeset num_corempi nombre_restant_node nombre_restant_comp
1658
1659  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1660    IGCM_debug_Exit "IGCM_sys_obelix build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1661  fi
1662
1663  if ( ${OK_PARA_MPMD} ) ; then
1664
1665    if [ -f run_file ] ; then
1666      IGCM_sys_Rm -f run_file
1667    fi
1668    touch run_file
1669
1670    if ( ${OK_PARA_OMP} ) ; then
1671
1672      #  Hosts treatment
1673
1674      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
1675
1676      i=0
1677      rm -f hosts
1678      IGCM_debug_Print 1 "sys Obelix, Hosts avaible :"
1679      for nodes in `cat hosts.tmp` ; do
1680        host[$i]=$nodes
1681        echo "${host[$i]} slots=1 max_slots=1" >> hosts
1682        IGCM_debug_Print 1 ${host[$i]}
1683        i=$((i+1))
1684      done
1685      rm -f hosts.tmp
1686
1687      listnodes=${host[*]}
1688
1689      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
1690
1691      # Initialisation
1692
1693      init_node=y
1694      node_num_current=0
1695      start_num=0
1696      init_exec=n
1697
1698      # Test : if oasis is there, we put it at the first position
1699
1700      for comp in ${config_ListOfComponents[*]} ; do
1701
1702        if [ "X${comp}" = "XCPL" ]  ; then
1703
1704          eval ExeNameIn=\${config_Executable_${comp}[0]}
1705          eval ExeNameOut=\${config_Executable_${comp}[1]}
1706
1707          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1708          echo ""  >> script_${ExeNameOut}.ksh
1709          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1710          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1711          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1712          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
1713          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1714
1715          init_node=n
1716
1717          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
1718          node_num_current=0
1719          node_current=${host[${node_num_current}]}
1720
1721          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1722
1723          init_exec=y
1724          start_num=1
1725
1726        fi
1727
1728      done
1729
1730      # Then loop on the components (except for oasis)
1731
1732      for comp in ${config_ListOfComponents[*]} ; do
1733
1734        eval ExeNameIn=\${config_Executable_${comp}[0]}
1735        eval ExeNameOut=\${config_Executable_${comp}[1]}
1736
1737        # Only if we really have an executable for the component :
1738        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
1739
1740          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1741          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1742
1743          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1744          # echo "set -vx" >> script_${ExeNameOut}.ksh
1745          echo ""  >> script_${ExeNameOut}.ksh
1746          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1747          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1748          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1749          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1750          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1751          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1752          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1753
1754          node_num=0
1755
1756          # We define the number of MPI process to be assigned for the component
1757
1758          nombre_restant_comp=${comp_proc_mpi_loc}
1759
1760          # Loop on the allocated nodes
1761
1762          for node in ${listnodes} ; do
1763
1764            # We go to the current node
1765            if [ ${node_num} = ${node_num_current} ] ; then
1766
1767              node_current=${host[${node_num_current}]}
1768
1769              # If first time on the node : initialisation
1770
1771              if [ ${init_node} = y ] ; then
1772                nombre_restant_node=${NUM_COREPERNODE}
1773              fi
1774
1775              # Test on the number of OMP threads
1776
1777              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1778                (( node_num = node_num + 1 ))
1779                node_num_current=${node_num}
1780                init_node=y
1781                continue
1782              fi
1783
1784              # Number of MPI process to assign
1785
1786              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1787
1788              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1789                num_corempi=${nombre_restant_comp}
1790              fi
1791
1792              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1793              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1794
1795              if [ ${init_exec} = y ] ; then
1796                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1797              else
1798                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1799                init_exec=y
1800              fi
1801
1802              ((  start_num = num_corempi + start_num ))
1803
1804            else
1805
1806              (( node_num = node_num + 1 ))
1807              continue
1808            fi
1809
1810            # Test on the number of core/process remaining on the node/component
1811
1812            if [ ${nombre_restant_node} = 0 ] ; then
1813              (( node_num = node_num + 1 ))
1814              node_num_current=${node_num}
1815              init_node=y
1816
1817              if [ ${nombre_restant_comp} = 0 ] ; then
1818                break 1
1819              fi
1820            else
1821
1822              node_num_current=${node_num}
1823              init_node=n
1824
1825              if [ ${nombre_restant_comp} = 0 ] ; then
1826                break 1
1827              fi
1828            fi
1829          done
1830        fi
1831      done
1832
1833    else
1834
1835      # Then first loop on the components for the coupler ie oasis
1836
1837      ## the coupler ie oasis must be the first one
1838      for comp in ${config_ListOfComponents[*]} ; do
1839
1840        eval ExeNameOut=\${config_Executable_${comp}[1]}
1841
1842        # for CPL component only
1843        if [ "X${comp}" = "XCPL" ] ; then
1844          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1845          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1846        fi
1847      done
1848
1849      # Then second loop on the components
1850
1851      for comp in ${config_ListOfComponents[*]} ; do
1852
1853        eval ExeNameOut=\${config_Executable_${comp}[1]}
1854
1855        # Only if we really have an executable for the component and not the coupler ie oasis:
1856        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1857          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1858          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1859        fi
1860      done
1861      IGCM_sys_Chmod u+x run_file
1862
1863      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1864
1865    fi
1866
1867  else # Only one executable. launch it.
1868
1869    for comp in ${config_ListOfComponents[*]} ; do
1870
1871      # Only if we really have an executable for the component :
1872      eval ExeNameOut=\${config_Executable_${comp}[1]}
1873      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1874
1875        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1876        echo ""  >> script_${ExeNameOut}.ksh
1877        if ( ${OK_PARA_OMP} ) ; then
1878          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1879          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1880        fi
1881        if  ( ${OK_PARA_MPI} ) ; then
1882          # Default : mpirun used if nb_proc gt 1
1883          # pour sortie out/err par process
1884          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
1885          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1886          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1887          EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1888        else
1889          # Default : mpirun is NOT used if nb_proc eq 1
1890          # pour sortie out/err par process
1891          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1892          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1893          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1894          EXECUTION="time ./script_${ExeNameOut}.ksh"
1895        fi
1896      fi
1897    done
1898
1899  fi
1900
1901  IGCM_debug_Print 1 "sys Obelix : execution command is"
1902  IGCM_debug_Print 1 "$EXECUTION"
1903
1904  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1905}
1906
1907############################################################
1908# Check of space available on temporary filesytems. Dummy function here
1909
1910#D-#==================================================
1911#D-function IGCM_sys_check_quota. Dummy call here
1912#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1913#D-* Examples:
1914#D-
1915function IGCM_sys_check_quota {
1916  IGCM_debug_PushStack "IGCM_sys_check_quota"
1917  if ( $DEBUG_sys ) ; then
1918    echo "IGCM_sys_check_quota"
1919  fi
1920  IGCM_debug_PopStack "IGCM_sys_check_quota"
1921}
1922
1923#D-#==================================================
1924#D-function IGCM_sys_CountJobInQueue
1925#D-* Purpose: Check if job_name is currently
1926#D-  running or in queue
1927#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1928#D-
1929function IGCM_sys_CountJobInQueue {
1930  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1931  if ( $DEBUG_sys ) ; then
1932    echo "IGCM_sys_CountJobInQueue"
1933  fi
1934  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1935}
1936
1937##############################################################
1938# NCO OPERATOR
1939
1940#D-#==================================================
1941#D-function IGCM_sys_ncap2
1942#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
1943#D-* Examples:
1944#D-
1945function IGCM_sys_ncap2 {
1946  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
1947  if ( $DEBUG_sys ) ; then
1948    echo "IGCM_sys_ncap2 :" $@
1949  fi
1950
1951  typeset NB_ESSAI DELAI status i
1952  # number of tentative
1953  NB_ESSAI=3
1954  # time delay between tentative
1955  DELAI=2
1956
1957  i=0
1958  while [ $i -lt $NB_ESSAI ] ; do
1959    ncap2 "$@" > out_rsync 2>&1
1960    status=$?
1961    if [ ${status} -gt 0 ] ; then
1962      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
1963      cat out_rsync
1964      \rm out_rsync
1965      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1966      sleep $DELAI
1967    else
1968      \rm out_rsync
1969      break
1970    fi
1971    (( i = i + 1 ))
1972  done
1973
1974  if [ ${status} -gt 0 ] ; then
1975    echo "IGCM_sys_ncap2 : ncap2 error"
1976    IGCM_debug_Exit "ncap2"
1977  fi
1978
1979  IGCM_debug_PopStack "IGCM_sys_ncap2"
1980}
1981
1982#D-#==================================================
1983#D-function IGCM_sys_ncatted
1984#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
1985#D-* Examples:
1986#D-
1987function IGCM_sys_ncatted {
1988  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1989  if ( $DEBUG_sys ) ; then
1990    echo "IGCM_sys_ncatted :" $@
1991  fi
1992
1993  typeset NB_ESSAI DELAI status i
1994  # number of tentative
1995  NB_ESSAI=3
1996  # time delay between tentative
1997  DELAI=2
1998
1999  i=0
2000  while [ $i -lt $NB_ESSAI ] ; do
2001    ncatted "$@" > out_rsync 2>&1
2002    status=$?
2003    if [ ${status} -gt 0 ] ; then
2004      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2005      cat out_rsync
2006      \rm out_rsync
2007      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2008      sleep $DELAI
2009    else
2010      \rm out_rsync
2011      break
2012    fi
2013    (( i = i + 1 ))
2014  done
2015
2016  if [ ${status} -gt 0 ] ; then
2017    echo "IGCM_sys_ncatted : ncatted error"
2018    IGCM_debug_Exit "ncatted"
2019  fi
2020
2021  IGCM_debug_PopStack "IGCM_sys_ncatted"
2022}
2023
2024#D-#==================================================
2025#D-function IGCM_sys_ncbo
2026#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
2027#D-* Examples:
2028#D-
2029function IGCM_sys_ncbo {
2030  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2031  if ( $DEBUG_sys ) ; then
2032    echo "IGCM_sys_ncbo :" $@
2033  fi
2034
2035  typeset NB_ESSAI DELAI status i
2036  # number of tentative
2037  NB_ESSAI=3
2038  # time delay between tentative
2039  DELAI=2
2040
2041  i=0
2042  while [ $i -lt $NB_ESSAI ] ; do
2043    ncbo $@ > out_rsync 2>&1
2044    status=$?
2045    if [ ${status} -gt 0 ] ; then
2046      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2047      cat out_rsync
2048      \rm out_rsync
2049      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2050      sleep $DELAI
2051    else
2052      \rm out_rsync
2053      break
2054    fi
2055    (( i = i + 1 ))
2056  done
2057
2058  if [ ${status} -gt 0 ] ; then
2059    echo "IGCM_sys_ncbo : ncbo error"
2060    IGCM_debug_Exit "ncbo"
2061  fi
2062
2063  IGCM_debug_PopStack "IGCM_sys_ncbo"
2064}
2065
2066#D-#==================================================
2067#D-function IGCM_sys_ncdif
2068#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2069#D-* Examples:
2070#D-
2071function IGCM_sys_ncdiff {
2072  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2073  if ( $DEBUG_sys ) ; then
2074    echo "IGCM_sys_ncdiff :" $@
2075  fi
2076
2077  typeset NB_ESSAI DELAI status i
2078  # number of tentative
2079  NB_ESSAI=3
2080  # time delay between tentative
2081  DELAI=2
2082
2083  i=0
2084  while [ $i -lt $NB_ESSAI ] ; do
2085    ncdiff $@ > out_rsync 2>&1
2086    status=$?
2087    if [ ${status} -gt 0 ] ; then
2088      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2089      cat out_rsync
2090      \rm out_rsync
2091      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2092      sleep $DELAI
2093    else
2094      \rm out_rsync
2095      break
2096    fi
2097    (( i = i + 1 ))
2098  done
2099
2100  if [ ${status} -gt 0 ] ; then
2101    echo "IGCM_sys_ncdiff : ncdiff error"
2102    IGCM_debug_Exit "ncdiff"
2103  fi
2104
2105  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2106}
2107
2108#D-#==================================================
2109#D-function IGCM_sys_ncea
2110#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2111#D-* Examples:
2112#D-
2113function IGCM_sys_ncea {
2114  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2115  if ( $DEBUG_sys ) ; then
2116    echo "IGCM_sys_ncea :" $@
2117  fi
2118
2119  typeset NB_ESSAI DELAI status i
2120  # number of tentative
2121  NB_ESSAI=3
2122  # time delay between tentative
2123  DELAI=2
2124
2125  i=0
2126  while [ $i -lt $NB_ESSAI ] ; do
2127    ncea $@ > out_rsync 2>&1
2128    status=$?
2129    if [ ${status} -gt 0 ] ; then
2130      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2131      cat out_rsync
2132      \rm out_rsync
2133      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2134      sleep $DELAI
2135    else
2136      \rm out_rsync
2137      break
2138    fi
2139    (( i = i + 1 ))
2140  done
2141
2142  if [ ${status} -gt 0 ] ; then
2143    echo "IGCM_sys_ncea : ncea error"
2144    IGCM_debug_Exit "ncea"
2145  fi
2146
2147  IGCM_debug_PopStack "IGCM_sys_ncea"
2148}
2149
2150#D-#==================================================
2151#D-function IGCM_sys_ncecat
2152#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2153#D-* Examples:
2154#D-
2155function IGCM_sys_ncecat {
2156  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2157  if ( $DEBUG_sys ) ; then
2158    echo "IGCM_sys_ncecat :" $@
2159  fi
2160
2161  typeset NB_ESSAI DELAI status i
2162  # number of tentative
2163  NB_ESSAI=3
2164  # time delay between tentative
2165  DELAI=2
2166
2167  i=0
2168  while [ $i -lt $NB_ESSAI ] ; do
2169    ncecat $@ > out_rsync 2>&1
2170    status=$?
2171    if [ ${status} -gt 0 ] ; then
2172      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2173      cat out_rsync
2174      \rm out_rsync
2175      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2176      sleep $DELAI
2177    else
2178      \rm out_rsync
2179      break
2180    fi
2181    (( i = i + 1 ))
2182  done
2183
2184  if [ ${status} -gt 0 ] ; then
2185    echo "IGCM_sys_ncecat : ncecat error"
2186    IGCM_debug_Exit "ncecat"
2187  fi
2188
2189  IGCM_debug_PopStack "IGCM_sys_ncecat"
2190}
2191
2192#D-#==================================================
2193#D-function IGCM_sys_ncflint
2194#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2195#D-* Examples:
2196#D-
2197function IGCM_sys_ncflint {
2198  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2199  if ( $DEBUG_sys ) ; then
2200    echo "IGCM_sys_ncflint :" $@
2201  fi
2202
2203  typeset NB_ESSAI DELAI status i
2204  # number of tentative
2205  NB_ESSAI=3
2206  # time delay between tentative
2207  DELAI=2
2208
2209  i=0
2210  while [ $i -lt $NB_ESSAI ] ; do
2211    ncflint $@ > out_rsync 2>&1
2212    status=$?
2213    if [ ${status} -gt 0 ] ; then
2214      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2215      cat out_rsync
2216      \rm out_rsync
2217      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2218      sleep $DELAI
2219    else
2220      \rm out_rsync
2221      break
2222    fi
2223    (( i = i + 1 ))
2224  done
2225
2226  if [ ${status} -gt 0 ] ; then
2227    echo "IGCM_sys_ncflint : ncflint error"
2228    IGCM_debug_Exit "ncflint"
2229  fi
2230
2231  IGCM_debug_PopStack "IGCM_sys_ncflint"
2232}
2233
2234#D-#==================================================
2235#D-function IGCM_sys_ncks
2236#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2237#D-* Examples:
2238#D-
2239function IGCM_sys_ncks {
2240  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2241  if ( $DEBUG_sys ) ; then
2242    echo "IGCM_sys_ncks :" $@
2243  fi
2244
2245  typeset NB_ESSAI DELAI status i
2246  # number of tentative
2247  NB_ESSAI=3
2248  # time delay between tentative
2249  DELAI=2
2250
2251  i=0
2252  while [ $i -lt $NB_ESSAI ] ; do
2253    ncks $@ > out_rsync 2>&1
2254    status=$?
2255    if [ ${status} -gt 0 ] ; then
2256      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2257      cat out_rsync
2258      \rm out_rsync
2259      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2260      sleep $DELAI
2261    else
2262      \rm out_rsync
2263      break
2264    fi
2265    (( i = i + 1 ))
2266  done
2267
2268  if [ ${status} -gt 0 ] ; then
2269    echo "IGCM_sys_ncks : ncks error"
2270    IGCM_debug_Exit "ncks"
2271  fi
2272
2273  IGCM_debug_PopStack "IGCM_sys_ncks"
2274}
2275
2276#D-#==================================================
2277#D-function IGCM_sys_ncpdq
2278#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2279#D-* Examples:
2280#D-
2281function IGCM_sys_ncpdq {
2282  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2283  if ( $DEBUG_sys ) ; then
2284    echo "IGCM_sys_ncpdq :" $@
2285  fi
2286
2287  typeset NB_ESSAI DELAI status i
2288  # number of tentative
2289  NB_ESSAI=3
2290  # time delay between tentative
2291  DELAI=2
2292
2293  i=0
2294  while [ $i -lt $NB_ESSAI ] ; do
2295    ncpdq $@ > out_rsync 2>&1
2296    status=$?
2297    if [ ${status} -gt 0 ] ; then
2298      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2299      cat out_rsync
2300      \rm out_rsync
2301      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2302      sleep $DELAI
2303    else
2304      \rm out_rsync
2305      break
2306    fi
2307    (( i = i + 1 ))
2308  done
2309
2310  if [ ${status} -gt 0 ] ; then
2311    echo "IGCM_sys_ncpdq : ncpdq error"
2312    IGCM_debug_Exit "ncpdq"
2313  fi
2314
2315  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2316}
2317
2318#D-#==================================================
2319#D-function IGCM_sys_ncra
2320#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2321#D-* Examples:
2322#D-
2323function IGCM_sys_ncra {
2324  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2325  if ( $DEBUG_sys ) ; then
2326    echo "IGCM_sys_ncra :" $@
2327  fi
2328
2329  typeset NB_ESSAI DELAI status i
2330  # number of tentative
2331  NB_ESSAI=3
2332  # time delay between tentative
2333  DELAI=2
2334
2335  i=0
2336  while [ $i -lt $NB_ESSAI ] ; do
2337    ncra $@ > out_rsync 2>&1
2338    status=$?
2339    if [ ${status} -gt 0 ] ; then
2340      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2341      cat out_rsync
2342      \rm out_rsync
2343      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2344      sleep $DELAI
2345    else
2346      \rm out_rsync
2347      break
2348    fi
2349    (( i = i + 1 ))
2350  done
2351
2352  if [ ${status} -gt 0 ] ; then
2353    echo "IGCM_sys_ncra : ncra error"
2354    IGCM_debug_Exit "ncra"
2355  fi
2356
2357  IGCM_debug_PopStack "IGCM_sys_ncra"
2358}
2359
2360#D-#==================================================
2361#D-function IGCM_sys_ncrcat
2362#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
2363#D-* Examples:
2364#D-
2365function IGCM_sys_ncrcat {
2366  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2367  if ( $DEBUG_sys ) ; then
2368    echo "IGCM_sys_ncrcat :" $@
2369  fi
2370
2371  typeset NB_ESSAI DELAI status i lastArg
2372  # number of tentative
2373  NB_ESSAI=3
2374  # time delay between tentative
2375  DELAI=2
2376
2377  i=0
2378  while [ $i -lt $NB_ESSAI ] ; do
2379    ncrcat $@ > out_rsync 2>&1
2380    status=$?
2381    if [ ${status} -gt 0 ] ; then
2382      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2383      cat out_rsync
2384      \rm out_rsync
2385      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2386      sleep $DELAI
2387    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
2388      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2389      cat out_rsync
2390      # remove files having corrupted time axis
2391      eval lastArg=\${$#}
2392      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2393      \rm ${lastArg}
2394      \rm out_rsync
2395      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2396      sleep $DELAI
2397    else
2398      \rm out_rsync
2399      break
2400    fi
2401    (( i = i + 1 ))
2402  done
2403
2404  if [ ${status} -gt 0 ] ; then
2405    echo "IGCM_sys_ncrcat : ncrcat error"
2406    #IGCM_debug_Exit "ncrcat"
2407  fi
2408
2409  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2410}
2411
2412#D-#==================================================
2413#D-function IGCM_sys_ncrename
2414#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
2415#D-* Examples:
2416#D-
2417function IGCM_sys_ncrename {
2418  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2419  if ( $DEBUG_sys ) ; then
2420    echo "IGCM_sys_ncrename :" $@
2421  fi
2422
2423  typeset NB_ESSAI DELAI status i
2424  # number of tentative
2425  NB_ESSAI=3
2426  # time delay between tentative
2427  DELAI=2
2428
2429  i=0
2430  while [ $i -lt $NB_ESSAI ] ; do
2431    ncrename $@ > out_rsync 2>&1
2432    status=$?
2433    if [ ${status} -gt 0 ] ; then
2434      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2435      cat out_rsync
2436      \rm out_rsync
2437      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2438      sleep $DELAI
2439    else
2440      \rm out_rsync
2441      break
2442    fi
2443    (( i = i + 1 ))
2444  done
2445
2446  if [ ${status} -gt 0 ] ; then
2447    echo "IGCM_sys_ncrename : ncrename error"
2448    IGCM_debug_Exit "ncrename"
2449  fi
2450
2451  IGCM_debug_PopStack "IGCM_sys_ncrename"
2452}
2453
2454#D-#==================================================
2455#D-function IGCM_sys_ncwa
2456#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
2457#D-* Examples:
2458#D-
2459function IGCM_sys_ncwa {
2460  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2461  if ( $DEBUG_sys ) ; then
2462    echo "IGCM_sys_ncwa :" $@
2463  fi
2464
2465  typeset NB_ESSAI DELAI status i
2466  # number of tentative
2467  NB_ESSAI=3
2468  # time delay between tentative
2469  DELAI=2
2470
2471  i=0
2472  while [ $i -lt $NB_ESSAI ] ; do
2473    ncwa $@ > out_rsync 2>&1
2474    status=$?
2475    if [ ${status} -gt 0 ] ; then
2476      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2477      cat out_rsync
2478      \rm out_rsync
2479      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2480      sleep $DELAI
2481    else
2482      \rm out_rsync
2483      break
2484    fi
2485    (( i = i + 1 ))
2486  done
2487
2488  if [ ${status} -gt 0 ] ; then
2489    echo "IGCM_sys_ncwa : ncwa error"
2490    IGCM_debug_Exit "ncwa"
2491  fi
2492
2493  IGCM_debug_PopStack "IGCM_sys_ncwa"
2494}
2495
2496##############################################################
2497# CDO OPERATOR
2498
2499#D-#==================================================
2500#D-function IGCM_sys_cdo
2501#D-* Purpose: encapsulate cdo call so as to manage error code and retry
2502#D-* Examples:
2503#D-
2504function IGCM_sys_cdo {
2505  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2506  if ( $DEBUG_sys ) ; then
2507    echo "IGCM_sys_cdo :" $@
2508  fi
2509
2510  typeset status
2511
2512  \cdo $@ > out_rsync 2>&1
2513  status=$?
2514  if [ ${status} -gt 0 ] ; then
2515    echo "IGCM_sys_cdo : error code ${status}"
2516    cat out_rsync
2517    \rm out_rsync
2518    IGCM_debug_PopStack "IGCM_sys_cdo"
2519    return 1
2520  else
2521    IGCM_debug_PopStack "IGCM_sys_cdo"
2522    return 0
2523  fi
2524
2525  IGCM_debug_PopStack "IGCM_sys_cdo"
2526}
Note: See TracBrowser for help on using the repository browser.