source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_iitm.ksh @ 1046

Last change on this file since 1046 was 1046, checked in by sdipsl, 10 years ago

Fix rebuild_station in the case empty files has been produced. see #212
File size: 69.0 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sébastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision:: 423                                      $ Revision of last commit
7# $Author:: sdipsl                                     $ Author of last commit
8# $Date:: 2011-02-18 16:49:14 +0100 (ven. 18 févr. 20#$ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for IITM IBM machine
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70ntypeset -r MASTER=iitm01
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# Submit command
78typeset SUBMIT=${SUBMIT:=/usr/lpp/LoadL/full/bin/llsubmit}
79# rsync with path
80typeset -r RSYNC=/usr/bin/rsync
81# RSYNC_opt args to rsync
82typeset -r RSYNC_opt="-va"
83# ie storage filesystem
84typeset -r STOREHOST=iitm01
85typeset -r REMOTE_RSYNC=/usr/bin/rsync
86
87#====================================================
88# Source default environment
89#====================================================
90##. /etc/profile
91
92#====================================================
93# Set environment tools (ferret, nco, cdo)
94#====================================================
95# Not applicable here
96
97#====================================================
98# Host specific DIRECTORIES
99#====================================================
100
101#====================================================
102#- Mirror libIGCM from iitm to a post-processing machine
103typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
104
105#====================================================
106#- libIGCM_POST for frontend
107typeset -r libIGCM_POST=${libIGCM}
108
109#====================================================
110#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
111typeset -r R_EXE="${MODIPSL}/bin"
112
113#====================================================
114#- SUBMIT_DIR : submission dir
115typeset SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
116
117#====================================================
118#- IN
119typeset -r R_IN=${R_IN:=/gpfs1/home/sabin}
120typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/gpfs1/home/sabin}
121
122#====================================================
123#- ARCHIVE
124typeset ARCHIVE=$( echo ${HOME} | sed -e "s/gpfs1/gpfs3/" )
125
126#====================================================
127#- STORAGE (dedicated to small/medium files)
128typeset STORAGE=${ARCHIVE}
129
130#====================================================
131#- R_OUT
132typeset R_OUT=${ARCHIVE}/IGCM_OUT
133
134#====================================================
135#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
136typeset R_FIG=${WORKDIR}/IGCM_OUT
137
138#====================================================
139#- R_BUF (ONLY FOR double copy an scratch)
140typeset -r R_BUF=${WORKDIR}/IGCM_OUT
141
142#====================================================
143#- RUN_DIR_PATH : Temporary working directory (=> TMP)
144typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${ARCHIVE}/RUN/${LOADL_STEP_ID}}
145
146#====================================================
147#- HOST_MPIRUN_COMMAND
148typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
149
150#====================================================
151#- Max number of arguments passed to nco operator or demigration command
152UNIX_MAX_LIMIT=360
153
154#====================================================
155#- set PackDefault true on curie
156PackDefault=false
157
158#====================================================
159#- Number of core per node (max number of OpenMP task)
160NUM_COREPERNODE=2
161
162#====================================================
163#- Default number of MPI task for IPSL coupled model
164#- required for backward compatibility
165#-
166DEFAULT_NUM_PROC_OCE=5
167DEFAULT_NUM_PROC_CPL=1
168(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
169DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
170
171#D-#==================================================
172#D-function IGCM_sys_ChangeArchive
173#D-* Purpose: Just a dummy call on this machine
174#D-* Examples:
175#D-
176function IGCM_sys_ChangeArchive {
177  IGCM_debug_Print 1 " dummy function : IGCM_sys_ChangeArchive "
178}
179
180#D-#==================================================
181#D-function IGCM_sys_RshMaster
182#D-* Purpose: Just a fake command to wrapp
183#D-           IGCM_card call in post-treatment
184#D-           Ulam do not see brodie filesystem
185#D-           Cesium do not see all mercure filesystem
186#D-           That's why we need this hack.
187#D-* Examples:
188#D-
189function IGCM_sys_RshMaster {
190  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
191  /bin/ksh <<-EOF
192    export libIGCM=${libIGCM}
193    export DEBUG_debug=${DEBUG_debug}
194    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
195    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
196    ${@}
197EOF
198  if [ $? -gt 0 ] ; then
199    echo "IGCM_sys_RshMaster : erreur."
200    IGCM_debug_Exit "IGCM_sys_RshMaster"
201  fi
202  IGCM_debug_PopStack "IGCM_sys_RshMaster"
203}
204
205#D-#==================================================
206#D-function IGCM_sys_RshArchive
207#D-* Purpose: Archive rsh command
208#D-* Examples:
209#D-
210function IGCM_sys_RshArchive {
211  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
212  /bin/ksh <<-EOF
213    ${@}
214EOF
215  status=$?
216  if [ ${status} -gt 0 ] ; then
217    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
218    IGCM_debug_Exit "IGCM_sys_RshArchive"
219  fi
220  IGCM_debug_PopStack "IGCM_sys_RshArchive"
221}
222
223#D-#==================================================
224#D-function IGCM_sys_RshPost
225#D-* Purpose: Post-process rsh command
226#D-* Examples:
227#D-
228function IGCM_sys_RshPost {
229  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
230  if ( $DEBUG_sys ) ; then
231    echo "IGCM_sys_RshPost :" $@
232  fi
233
234  # keep standard input to keep it for postpone if ulam don't answer
235  cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
236
237  /bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
238  if [ $? -gt 0 ] ; then
239    echo "IGCM_sys_RshPost : erreur."
240    IGCM_debug_Exit "IGCM_sys_RshPost"
241  fi
242  # delete temporary file
243  \rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
244
245  IGCM_debug_PopStack "IGCM_sys_RshPost"
246}
247
248#D-#==================================================
249#D-function IGCM_sys_SendMail
250#D-* Purpose: Send mail when simulation is over
251#D-* Examples:
252#D-
253function IGCM_sys_SendMail {
254  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
255  if ( $DEBUG_sys ) ; then
256    echo "IGCM_sys_SendMail :" $@
257  fi
258
259  if [ X${1} = XAccounting ] ; then
260    status=Accounting
261    mailText=jobAccounting.mail
262  elif ( ${ExitFlag} ) ; then
263    status=failed
264    mailText=jobEnd.mail
265  else
266    status=completed
267    mailText=jobEnd.mail
268  fi
269
270  # Update selected mail template
271  while read -r line; do
272    eval echo $line >> mail.txt ;
273  done < ${libIGCM}/libIGCM_sys/${mailText}
274
275  if [ ! -z ${config_UserChoices_MailName} ] ; then
276    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < mail.txt
277  elif [ -f ~/.forward ] ; then
278    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
279  else
280    mailx -s "${config_UserChoices_JobName} ${status}" ${USER} < mail.txt
281  fi
282
283  sleep 10
284  rm -f mail.txt
285
286  IGCM_debug_PopStack "IGCM_sys_SendMail"
287}
288
289#D-#==================================================
290#D-function IGCM_sys_Mkdir
291#D-* Purpose: Master locale mkdir command
292#D-* Examples:
293#D-
294function IGCM_sys_Mkdir {
295  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
296  if ( $DEBUG_sys ) ; then
297    echo "IGCM_sys_Mkdir :" $@
298  fi
299  if [ ! -d ${1} ]; then
300    \mkdir -p $1
301    if [ $? -gt 0 ] ; then
302      echo "IGCM_sys_Mkdir : erreur."
303      IGCM_debug_Exit "IGCM_sys_Mkdir"
304    fi
305  fi
306  # vérification :
307  if [ ! -d ${1} ] ; then
308    echo "IGCM_sys_Mkdir : erreur."
309    IGCM_debug_Exit "IGCM_sys_Mkdir"
310  fi
311  IGCM_debug_PopStack "IGCM_sys_Mkdir"
312}
313
314#D-#==================================================
315#D-function IGCM_sys_MkdirArchive
316#D-* Purpose: Mkdir on Archive
317#D-* Examples:
318#D-
319function IGCM_sys_MkdirArchive {
320  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
321  if ( $DEBUG_sys ) ; then
322    echo "IGCM_sys_MkdirArchive :" $@
323  fi
324  #- creation de repertoire sur le serveur fichier
325  mkdir -p $1
326  status=$?
327
328  if [ ${status} -gt 0 ] ; then
329    IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
330    IGCM_debug_Exit "IGCM_sys_MkdirArchive"
331  fi
332  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
333}
334
335#D-#==================================================
336#D-function IGCM_sys_MkdirWork
337#D-* Purpose: Mkdir on Work
338#D-* Examples:
339#D-
340function IGCM_sys_MkdirWork {
341  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
342  if ( $DEBUG_sys ) ; then
343    echo "IGCM_sys_MkdirWork :" $@
344  fi
345  #- creation de repertoire sur le serveur fichier
346  if [ ! -d ${1} ]; then
347    \mkdir -p $1
348    if [ $? -gt 0 ] ; then
349      echo "IGCM_sys_MkdirWork : erreur."
350      IGCM_debug_Exit "IGCM_sys_MkdirWork"
351    fi
352  fi
353  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
354}
355
356#D-#==================================================
357#D-function IGCM_sys_Cd
358#D-* Purpose: master cd command
359#D-* Examples:
360#D-
361function IGCM_sys_Cd {
362  IGCM_debug_PushStack "IGCM_sys_Cd" $@
363  if ( $DEBUG_sys ) ; then
364    echo "IGCM_sys_Cd :" $@
365  fi
366  \cd $1
367  if [ $? -gt 0 ] ; then
368    echo "IGCM_sys_Cd : erreur."
369    IGCM_debug_Exit "IGCM_sys_Cd"
370  fi
371  IGCM_debug_PopStack "IGCM_sys_Cd"
372}
373
374#D-#==================================================
375#D-function IGCM_sys_Chmod
376#D-* Purpose: Chmod
377#D-* Examples:
378#D-
379function IGCM_sys_Chmod {
380  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
381  if ( $DEBUG_sys ) ; then
382    echo "IGCM_sys_Chmod :" $@
383  fi
384  \chmod $@
385  if [ $? -gt 0 ] ; then
386    echo "IGCM_sys_Chmod : erreur."
387    IGCM_debug_Exit "IGCM_sys_Chmod"
388  fi
389  IGCM_debug_PopStack "IGCM_sys_Chmod"
390}
391
392#D-#==================================================
393#D-function IGCM_sys_FileSize
394#D-* Purpose: Filesize
395#D-* Examples:
396#D-
397function IGCM_sys_FileSize {
398  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
399
400  typeset sizeF
401  set +A sizeF -- $( ls -la ${1} )
402  if [ $? -gt 0 ] ; then
403    IGCM_debug_Exit "IGCM_sys_FileSize"
404  fi
405  eval ${2}=${sizeF[4]}
406
407  IGCM_debug_PopStack "IGCM_sys_FileSize"
408}
409
410#D-#==================================================
411#D-function IGCM_sys_TestDir
412#D-* Purpose: Test Directory that must exists
413#D-* Examples:
414#D-
415function IGCM_sys_TestDir {
416  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
417  if ( $DEBUG_sys ) ; then
418    echo "IGCM_sys_TestDir :" $@
419  fi
420  typeset ExistFlag
421  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
422  IGCM_debug_PopStack "IGCM_sys_TestDir"
423
424  return ${ExistFlag}
425}
426
427#D-#==================================================
428#D-function IGCM_sys_TestDirArchive
429#D-* Purpose: Test Directory that must exists on Archive
430#D-* Examples:
431#D-
432function IGCM_sys_TestDirArchive {
433  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
434  if ( $DEBUG_sys ) ; then
435    echo "IGCM_sys_TestDirArchive :" $@
436  fi
437  typeset ExistFlag
438  ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 || echo 1" )
439  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
440  return ${ExistFlag}
441}
442
443#D-#==================================================
444#D-function IGCM_sys_IsFileArchived
445#D-* Purpose: Test file that must NOT EXISTS on Archive
446#D-* Examples:
447#D-
448function IGCM_sys_IsFileArchived {
449  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
450  if ( $DEBUG_sys ) ; then
451    echo "IGCM_sys_IsFileArchived :" $@
452  fi
453  typeset IsArchivedFlag
454  IsArchivedFlag=$( [ "X$( echo $@ | grep \/gpfs3 )" != "X" ] && echo 0 || echo 1 )
455  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
456
457  return ${IsArchivedFlag}
458}
459
460#D-#==================================================
461#D-function IGCM_sys_TestFileArchive
462#D-* Purpose: Test file that must NOT EXISTS on Archive
463#D-* Examples:
464#D-
465function IGCM_sys_TestFileArchive {
466  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
467  typeset ExistFlag
468  ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 || echo 1" )
469  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
470
471  return ${ExistFlag}
472}
473
474#D-#==================================================
475#D-function IGCM_sys_TestFileBuffer
476#D-* Purpose: Test file that must NOT EXISTS on Buffer
477#D-* Examples:
478#D-
479function IGCM_sys_TestFileBuffer {
480  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
481  typeset ExistFlag
482  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
483  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
484
485  return ${ExistFlag}
486}
487
488#D-#==================================================
489#D-function IGCM_sys_CountFileArchive
490#D-* Purpose: Count files on Archive filesystem
491#D-* Examples:
492#D-
493function IGCM_sys_CountFileArchive {
494  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
495  #Command depends on targeted file system
496  DEBUG_sys=false IGCM_sys_IsFileArchived $1
497  if [ $? = 0 ] ; then
498    IGCM_sys_RshArchive "ls ${@} 2>/dev/null | wc -l"
499    status=$?
500  else
501    ls ${@} 2>/dev/null | wc -l
502    status=$?
503  fi
504  if [ ${status} -gt 0 ] ; then
505    echo "IGCM_sys_CountFileArchive : erreur."
506  fi
507  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
508}
509
510#D-#==================================================
511#D-function IGCM_sys_CountFileBuffer
512#D-* Purpose: Count files on Scratch filesystem
513#D-* Examples:
514#D-
515function IGCM_sys_CountFileBuffer {
516  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
517  ls ${@} 2>/dev/null | wc -l
518  if [ $? -gt 0 ] ; then
519    echo "IGCM_sys_CountFileBuffer : erreur."
520  fi
521  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
522}
523
524#D-#==================================================
525#D-function IGCM_sys_Tree
526#D-* Purpose: Tree directories with files on ${ARCHIVE}
527#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
528#D-
529function IGCM_sys_Tree {
530  IGCM_debug_PushStack "IGCM_sys_Tree" $@
531  if ( $DEBUG_sys ) ; then
532    echo "IGCM_sys_Tree :" $@
533  fi
534
535  \tree -f $@
536
537  IGCM_debug_PopStack "IGCM_sys_Tree"
538}
539
540#D-#==================================================
541#D-function IGCM_sys_Tar
542#D-* Purpose: master tar command
543#D-* Examples:
544#D-
545function IGCM_sys_Tar {
546  IGCM_debug_PushStack "IGCM_sys_Tar" $@
547  if ( $DEBUG_sys ) ; then
548    echo "IGCM_sys_Tar :" $@
549  fi
550  \tar cf $@
551  if [ $? -gt 0 ] ; then
552    echo "IGCM_sys_Tar : erreur."
553    IGCM_debug_Exit "IGCM_sys_Tar"
554  fi
555  IGCM_debug_PopStack "IGCM_sys_Tar"
556}
557
558#D-#==================================================
559#D-function IGCM_sys_UnTar
560#D-* Purpose: master un-tar command
561#D-* Examples:
562#D-
563function IGCM_sys_UnTar {
564  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
565  if ( $DEBUG_sys ) ; then
566    echo "IGCM_sys_UnTar :" $@
567  fi
568  \tar xvf $1
569  if [ $? -gt 0 ] ; then
570    echo "IGCM_sys_UnTar : erreur."
571    IGCM_debug_Exit "IGCM_sys_UnTar"
572  fi
573  IGCM_debug_PopStack "IGCM_sys_UnTar"
574}
575
576#D-#==================================================
577#D-function IGCM_sys_Qsub
578#D-* Purpose: Qsub new job
579#D-* Examples:
580#D-
581function IGCM_sys_Qsub {
582  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
583
584  typeset status
585
586  if ( $DEBUG_sys ) ; then
587    echo "IGCM_sys_Qsub :" $@
588  fi
589  # We have to change output/error file
590  [ ${#@} = 1 ] &&  REP_FOR_JOB=${SUBMIT_DIR}
591  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
592  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
593    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
594    $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
595  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM > /tmp/out_command.$$ 2>&1 ; status=$? ; cd - ;
596
597  cat /tmp/out_command.$$
598  if [ ${status} -gt 0 ] ; then
599    IGCM_debug_Print 2 "IGCM_sys_Qsub $1 : error code ${status}"
600    IGCM_debug_Exit "IGCM_sys_Qsub"
601  else
602    JobID=$( gawk {'print $4'} /tmp/out_command.$$ | tail -1 | sed -e s/\"//g )
603    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
604  fi
605  IGCM_debug_PopStack "IGCM_sys_Qsub"
606}
607
608#D-#==================================================
609#D-function IGCM_sys_QsubPost
610#D-* Purpose: Qsub new job on scalaire
611#D-* Examples:
612#D-
613function IGCM_sys_QsubPost {
614  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
615
616  typeset status
617
618  if ( $DEBUG_sys ) ; then
619    echo "IGCM_sys_QsubPost :" $@
620  fi
621  # We have to change output/error file
622  [ ${#@} = 1 ] &&  REP_FOR_JOB=${POST_DIR}
623  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
624
625  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
626    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
627    ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
628
629  cd $REP_FOR_JOB ; /usr/lpp/LoadL/full/bin/llsubmit JOB_FOR_IGCM > /tmp/out_command.$$ 2>&1 ; status=$? ; cd - ;
630
631  cat /tmp/out_command.$$
632  if [ ${status} -gt 0 ] ; then
633    IGCM_debug_Print 2 "IGCM_sys_QsubPost $1 : error code ${status}"
634    IGCM_debug_Exit "IGCM_sys_QsubPost"
635  else
636    JobID=$( gawk {'print $4'} /tmp/out_command.$$ | tail -1 | sed -e s/\"//g )
637    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
638  fi
639  IGCM_debug_PopStack "IGCM_sys_QsubPost"
640}
641
642#D-*************************
643#D- File transfer functions
644#D-*************************
645#D-
646
647#D-#==================================================
648#D-function IGCM_sys_Rsync_out
649#D-* Purpose: treat return val of rsync
650#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
651#D-  Error values and explanations can depend on your system version.
652function IGCM_sys_Rsync_out {
653  status=$1
654  if [ ! $status ] ; then
655    echo "rsync error !"
656  fi
657
658  if [ $MYLANG = "fr" ]; then
659    case $status in
660    0)  return ;;
661    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
662      echo "Erreur de syntaxe ou d'utilisation."
663      return;;
664    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
665      echo "Incompatibilité de protocole."
666      return;;
667    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
668      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
669      echo "répertoires"
670      return;;
671    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
672      echo "Action demandée non supportée : une tentative de manipulation de"
673      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
674      echo "été faite ; ou une option qui est supportée par le  client  mais"
675      echo "pas par le serveur a été spécifiée."
676      return;;
677    10) echo "Erreur de rsync ; RERR_SOCKETIO"
678      echo "Erreur dans le socket d'entrée sortie"
679      return;;
680    11) echo "Erreur de rsync ; RERR_FILEIO"
681      echo "Erreur d'entrée sortie fichier"
682      return;;
683    12) echo "Erreur de rsync ; RERR_STREAMIO"
684      echo "Erreur dans flux de donnée du protocole rsync"
685      return;;
686    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
687      echo "Erreur avec les diagnostics du programme"
688      return;;
689    14) echo "Erreur de rsync ; RERR_IPC"
690      echo "Erreur dans le code IPC"
691      return;;
692    20) echo "Erreur de rsync ; RERR_SIGNAL"
693      echo "SIGUSR1 ou SIGINT reçu"
694      return;;
695    21) echo "Erreur de rsync ; RERR_WAITCHILD"
696      echo "Une erreur retournée par waitpid()"
697      return;;
698    22) echo "Erreur de rsync ; RERR_MALLOC"
699      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
700      return;;
701    23) echo ""
702      echo "Erreur fichier inexistant"
703      return;;
704    30) echo "Erreur de rsync ; RERR_TIMEOUT"
705      echo "Temps d'attente écoulé dans l'envoi/réception de données"
706      return;;
707    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
708      return;;
709    esac
710  elif [ $MYLANG = "en" ] ; then
711    case $status in
712    0)  return;;
713    1)  echo "rsync error : Syntax or usage error "
714      return;;
715    2)  echo "rsync error : Protocol incompatibility "
716      return;;
717    3)  echo "rsync error : Errors selecting input/output files, dirs"
718      return;;
719    4)  echo "rsync error : Requested action not supported: an attempt"
720      echo "was made to manipulate 64-bit files on a platform that cannot support"
721      echo "them; or an option was specified that is supported by the client and"
722      echo "not by the server."
723      return;;
724    5)  echo "rsync error : Error starting client-server protocol"
725      return;;
726    10) echo "rsync error : Error in socket I/O "
727      return;;
728    11) echo "rsync error : Error in file I/O "
729      return;;
730    12) echo "rsync error : Error in rsync protocol data stream "
731      return;;
732    13) echo "rsync error : Errors with program diagnostics "
733      return;;
734    14) echo "rsync error : Error in IPC code "
735      return;;
736    20) echo "rsync error : Received SIGUSR1 or SIGINT "
737      return;;
738    21) echo "rsync error : Some error returned by waitpid() "
739      return;;
740    22) echo "rsync error : Error allocating core memory buffers "
741      return;;
742    23) echo "rsync error : Partial transfer due to error"
743      return;;
744    24) echo "rsync error : Partial transfer due to vanished source files"
745      return;;
746    30) echo "rsync error : Timeout in data send/receive "
747      return;;
748    *)  echo "rsync error : return code of rsync unknown :" $status
749      return;;
750    esac
751  else
752    echo "unknown language $MYLANG."
753    return
754  fi
755}
756
757#D-#==================================================
758#D-function IGCM_sys_Miror_libIGCM
759#D-* Purpose: Mirror libIGCM PATH and lib to frontend
760#D-* Examples:
761#D-
762function IGCM_sys_Mirror_libIGCM {
763  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
764  if ( $DEBUG_sys ) ; then
765    echo "IGCM_sys_Mirror_libIGCM"
766  fi
767
768  typeset status
769
770  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
771
772  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > /tmp/out_command.$$ 2>&1
773  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> /tmp/out_command.$$ 2>&1
774  status=$?
775
776  if [ ${status} -gt 0 ] ; then
777    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
778    cat /tmp/out_command.$$
779  fi
780  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
781}
782
783#D-#==================================================
784#D-function IGCM_sys_Cp
785#D-* Purpose: generic cp
786#D-* Examples:
787#D-
788function IGCM_sys_Cp {
789  IGCM_debug_PushStack "IGCM_sys_Cp" $@
790  if ( $DEBUG_sys ) ; then
791    echo "IGCM_sys_Cp :" $@
792  fi
793
794  typeset status
795
796  echo cp $@ > /tmp/out_command.$$ 2>&1
797  \cp $@ >> /tmp/out_command.$$ 2>&1
798  status=$?
799
800  if [ ${status} -gt 0 ] ; then
801    echo "IGCM_sys_Cp : error code ${status}"
802    cat /tmp/out_command.$$
803    IGCM_debug_Exit "IGCM_sys_Cp"
804  else
805    \rm /tmp/out_command.$$
806  fi
807  IGCM_debug_PopStack "IGCM_sys_Cp"
808}
809
810#D-#==================================================
811#D-function IGCM_sys_Rm
812#D-* Purpose: generic rm
813#D-* Examples:
814#D-
815function IGCM_sys_Rm {
816  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
817  if ( $DEBUG_sys ) ; then
818    echo "IGCM_sys_Rm :" $@
819  fi
820
821  typeset status
822
823  echo rm $@ > /tmp/out_command.$$ 2>&1
824  \rm $@ >> /tmp/out_command.$$ 2>&1
825  status=$?
826
827  if [ ${status} -gt 0 ] ; then
828    echo "IGCM_sys_Rm : error code ${status}"
829    cat /tmp/out_command.$$
830    IGCM_debug_Exit "IGCM_sys_Rm"
831  else
832    \rm /tmp/out_command.$$
833  fi
834  IGCM_debug_PopStack "IGCM_sys_Rm"
835}
836
837#D-#==================================================
838#D-function IGCM_sys_RmRunDir
839#D-* Purpose: rm tmpdir (dummy function most of the time batch
840#D-                      scheduler will do the job)
841#D-* Examples:
842#D-
843function IGCM_sys_RmRunDir {
844  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
845  if ( $DEBUG_sys ) ; then
846    echo "IGCM_sys_RmRunDir :" $@
847  fi
848
849  typeset status
850
851  echo rm $@ > /tmp/out_command.$$ 2>&1
852  \rm $@ >> /tmp/out_command.$$ 2>&1
853  status=$?
854
855  if [ ${status} -gt 0 ] ; then
856    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
857    cat /tmp/out_command.$$
858    IGCM_debug_Exit "IGCM_sys_RmRunDir"
859  else
860    \rm /tmp/out_command.$$
861  fi
862  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
863}
864
865#D-#==================================================
866#D-function IGCM_sys_Mv
867#D-* Purpose: generic move
868#D-* Examples:
869#D-
870function IGCM_sys_Mv {
871  IGCM_debug_PushStack "IGCM_sys_Mv" $@
872  if ( $DEBUG_sys ) ; then
873    echo "IGCM_sys_Mv :" $@
874  fi
875
876  if [ $DRYRUN = 0 ]; then
877
878    typeset status
879
880    echo mv $@ > /tmp/out_command.$$ 2>&1
881    \mv $@ >> /tmp/out_command.$$ 2>&1
882    status=$?
883
884    if [ ${status} -gt 0 ] ; then
885      echo "IGCM_sys_Mv : error code ${status}"
886      cat /tmp/out_command.$$
887      IGCM_debug_Exit "IGCM_sys_Mv"
888    else
889      \rm /tmp/out_command.$$
890    fi
891  fi
892
893  IGCM_debug_PopStack "IGCM_sys_Mv"
894}
895
896#D-#==================================================
897#D-function IGCM_sys_Put_Dir
898#D-* Purpose: Copy a complete directory on $(ARCHIVE)
899#D-* Examples:
900#D-
901function IGCM_sys_Put_Dir {
902  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
903  if ( $DEBUG_sys ) ; then
904    echo "IGCM_sys_Put_Dir :" $@
905  fi
906  if [ $DRYRUN = 0 ]; then
907    if [ ! -d ${1} ] ; then
908      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
909      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
910      return
911    fi
912
913    typeset status
914
915    # Only if we use rsync
916    #IGCM_sys_TestDirArchive $( dirname $2 )
917    #
918    #USUAL WAY
919    \cp -r $1 $2 > /tmp/out_command.$$ 2>&1
920    status=$?
921
922    if [ ${status} -gt 0 ] ; then
923      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
924      cat /tmp/out_command.$$
925      IGCM_debug_Exit "IGCM_sys_Put_Dir"
926    else
927      \rm /tmp/out_command.$$
928    fi
929  fi
930  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
931}
932
933#D-#==================================================
934#D-function IGCM_sys_Get_Dir
935#D-* Purpose: Copy a complete directory from ${ARCHIVE}
936#D-* Examples:
937#D-
938function IGCM_sys_Get_Dir {
939  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
940  if ( $DEBUG_sys ) ; then
941    echo "IGCM_sys_Get_Dir :" $@
942  fi
943  if [ $DRYRUN = 0 ]; then
944    if [ ! -d ${1} ] ; then
945      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
946      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
947      return
948    fi
949
950    typeset status
951
952    #USUAL WAY
953    \cp -ur $1 $2 > /tmp/out_command.$$ 2>&1
954    status=$?
955
956    if [ ${status} -gt 0 ] ; then
957      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
958      cat /tmp/out_command.$$
959      IGCM_debug_Exit "IGCM_sys_Get_Dir"
960    else
961      \rm /tmp/out_command.$$
962    fi
963  fi
964  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
965}
966
967#D-#==================================================
968#D-function IGCM_sys_Get_Master
969#D-* Purpose: Copy a complete directory from MASTER filesystem
970#D-* Examples:
971#D-
972function IGCM_sys_Get_Master {
973  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
974  if ( $DEBUG_sys ) ; then
975    echo "IGCM_sys_Get_Master :" $@
976  fi
977  if [ $DRYRUN = 0 ]; then
978    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
979      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
980      IGCM_debug_PopStack "IGCM_sys_Get_Master"
981      return
982    fi
983
984    typeset NB_ESSAI DELAI status i
985    # number of tentative
986    NB_ESSAI=3
987    # time delay between tentative
988    DELAI=2
989
990    i=0
991    while [ $i -lt $NB_ESSAI ] ; do
992      \cp -urL $1 $2 > /tmp/out_command.$$ 2>&1
993      status=$?
994      if [ ${status} -gt 0 ]; then
995        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
996        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
997        sleep $DELAI
998      else
999        break
1000      fi
1001      (( i = i + 1 ))
1002    done
1003
1004    if [ ${status} -gt 0 ] ; then
1005      echo "IGCM_sys_Get_Master : error."
1006      cat /tmp/out_command.$$
1007      IGCM_debug_Exit "IGCM_sys_Get_Master"
1008    else
1009      \rm /tmp/out_command.$$
1010    fi
1011  fi
1012  IGCM_debug_PopStack "IGCM_sys_Get_Master"
1013}
1014
1015#====================================================
1016#- Call IGCM_sys_Mirror_libIGCM now !
1017if ( $MirrorlibIGCM ) ; then
1018  IGCM_sys_Mirror_libIGCM
1019fi
1020
1021#D-#==================================================
1022#D-function IGCM_sys_Put_Rest
1023#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1024#D-           File and target directory must exist.
1025#D-* Examples:
1026#D-
1027function IGCM_sys_Put_Rest {
1028  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1029  if ( $DEBUG_sys ) ; then
1030    echo "IGCM_sys_Put_Rest :" $@
1031  fi
1032  if [ $DRYRUN = 0 ]; then
1033    if [ ! -f ${1} ] ; then
1034      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1035      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1036    fi
1037
1038    typeset status
1039    #
1040    if [ X${JobType} = XRUN ] ; then
1041      IGCM_sys_Chmod 444 ${1}
1042    fi
1043    #
1044    # Only if we use rsync
1045    #IGCM_sys_MkdirArchive $( dirname $2 )
1046    #
1047    #USUAL WAY
1048    \cp $1 $2 > /tmp/out_command.$$ 2>&1
1049    status=$?
1050
1051    if [ ${status} -gt 0 ] ; then
1052      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1053      cat /tmp/out_command.$$
1054      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1055    else
1056      \rm /tmp/out_command.$$
1057    fi
1058  fi
1059  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1060}
1061
1062#D-#==================================================
1063#D-function IGCM_sys_PutBuffer_Rest
1064#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1065#D-           File and target directory must exist.
1066#D-* Examples:
1067#D-
1068function IGCM_sys_PutBuffer_Rest {
1069  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1070  if ( $DEBUG_sys ) ; then
1071    echo "IGCM_sys_PutBuffer_Rest :" $@
1072  fi
1073  if [ $DRYRUN = 0 ]; then
1074    if [ ! -f ${1} ] ; then
1075      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1076      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1077    fi
1078
1079    typeset status
1080    #
1081    if [ X${JobType} = XRUN ] ; then
1082      IGCM_sys_Chmod 444 ${1}
1083    fi
1084
1085    #
1086    # USUAL WAY
1087    \cp $1 $2 > /tmp/out_command.$$ 2>&1
1088    status=$?
1089
1090    if [ ${status} -gt 0 ] ; then
1091      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1092      [ -f ${2} ] && ls -l ${2}
1093      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1094      cat /tmp/out_command.$$
1095      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1096    else
1097      \rm /tmp/out_command.$$
1098    fi
1099  fi
1100  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1101}
1102
1103#D-#==================================================
1104#D-function IGCM_sys_PrepareTaredRestart
1105#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1106#D-* Examples:
1107#D-
1108function IGCM_sys_PrepareTaredRestart {
1109  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1110
1111  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1112}
1113
1114#D-#==================================================
1115#D-function IGCM_sys_Put_Out
1116#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1117#D-* Examples:
1118#D-
1119function IGCM_sys_Put_Out {
1120  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1121  if ( $DEBUG_sys ) ; then
1122    echo "IGCM_sys_Put_Out :" $@
1123  fi
1124
1125  typeset status
1126
1127  if [ $DRYRUN = 0 ]; then
1128    if [ ! -f ${1} ] ; then
1129      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1130      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1131      return 1
1132    fi
1133    #
1134    IGCM_sys_MkdirArchive $( dirname $2 )
1135    #
1136    if [ X${JobType} = XRUN ] ; then
1137      if [ X${3} = X ] ; then
1138        IGCM_sys_Chmod 444 ${1}
1139      fi
1140    fi
1141    #
1142    #USUAL WAY
1143    \cp $1 $2 > /tmp/out_command.$$ 2>&1
1144    status=$?
1145
1146    if [ ${status} -gt 0 ] ; then
1147      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
1148      cat /tmp/out_command.$$
1149      IGCM_debug_Exit "IGCM_sys_Put_Out"
1150    else
1151      \rm /tmp/out_command.$$
1152    fi
1153  fi
1154  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1155  return 0
1156}
1157
1158#D-#==================================================
1159#D-function IGCM_sys_PutBuffer_Out
1160#D-* Purpose: Copy a file on ${WORKDIR} after having chmod it in readonly
1161#D-* Examples:
1162#D-
1163function IGCM_sys_PutBuffer_Out {
1164  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1165  if ( $DEBUG_sys ) ; then
1166    echo "IGCM_sys_PutBuffer_Out :" $@
1167  fi
1168
1169  typeset NB_ESSAI DELAI status i exist skip
1170
1171  # number of tentative
1172  NB_ESSAI=3
1173  # time delay between tentative
1174  DELAI=2
1175
1176  if [ $DRYRUN = 0 ]; then
1177    if [ ! -f ${1} ] ; then
1178      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1179      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1180      return 1
1181    fi
1182    #
1183    IGCM_sys_Mkdir $( dirname $2 )
1184    #
1185
1186    exist=false
1187    skip=false
1188    if [ -f $2 ] ; then
1189      IGCM_debug_Print 1 "$2 already exist"
1190      exist=true
1191      if [ "X$( diff $1 $2 )" = X ] ; then
1192        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1193        status=0
1194        skip=true
1195      else
1196        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1197        skip=false
1198      fi
1199    fi
1200    #
1201    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1202      IGCM_sys_Chmod u+w $2
1203    fi
1204
1205    if [ X${skip} = Xfalse ] ; then
1206      i=0
1207      while [ $i -lt $NB_ESSAI ] ; do
1208        # USUAL WAY
1209        \cp $1 $2 > /tmp/out_command.$$ 2>&1
1210        status=$?
1211        if [ ${status} -gt 0 ]; then
1212          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1213          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1214          [ -f ${2} ] && ls -l ${2}
1215          [ -f ${2}/${1} ] && ls -l ${2}/${1}
1216          sleep $DELAI
1217        else
1218          break
1219        fi
1220        (( i = i + 1 ))
1221      done
1222    fi
1223
1224    if [ ${status} -gt 0 ] ; then
1225      echo "IGCM_sys_PutBuffer_Out : error."
1226      [ -f ${2} ] && ls -l ${2}
1227      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1228      cat /tmp/out_command.$$
1229      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1230    else
1231
1232      if [ X${JobType} = XRUN ] ; then
1233        if [ X${3} = X ] ; then
1234          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1235          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1236        fi
1237      fi
1238
1239      \rm /tmp/out_command.$$
1240    fi
1241  fi
1242  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1243  return 0
1244}
1245
1246#D-#==================================================
1247#D-function IGCM_sys_Get
1248#D-* Purpose: Get a file from ${ARCHIVE}
1249#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1250#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1251function IGCM_sys_Get {
1252  IGCM_debug_PushStack "IGCM_sys_Get" $@
1253
1254  typeset DEST status dm_liste
1255
1256  if ( $DEBUG_sys ) ; then
1257    echo "IGCM_sys_Get :" $@
1258  fi
1259  if [ $DRYRUN -le 2 ]; then
1260    if [ X${1} = X'/l' ] ; then
1261      eval set +A dm_liste \${${2}}
1262    else
1263      dm_liste=${1}
1264    fi
1265    eval DEST=\${${#}}
1266
1267    # test if the (first) file is present in the old computation :
1268    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
1269    if [ $? = 0 ] ; then
1270      IGCM_sys_TestFileArchive ${dm_liste[0]}
1271      status=$?
1272    else
1273      IGCM_sys_TestFileBuffer ${dm_liste[0]}
1274      status=$?
1275    fi
1276
1277    if [ ${status} -gt 0 ] ; then
1278      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1279      IGCM_debug_Exit "IGCM_sys_Get"
1280      return
1281    fi
1282
1283    #USUAL WAY
1284    \cp ${dm_liste[*]} ${DEST} > /tmp/out_command.$$ 2>&1
1285    status=$?
1286
1287    if [ ${status} -gt 0 ] ; then
1288      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
1289      cat /tmp/out_command.$$
1290      IGCM_debug_Exit "IGCM_sys_Get"
1291    else
1292      \rm /tmp/out_command.$$
1293    fi
1294  fi
1295  IGCM_debug_PopStack "IGCM_sys_Get"
1296}
1297
1298#D-#==================================================
1299#D-function IGCM_sys_GetBuffer
1300#D-* Purpose: Get a file from ${SCRATCHDIR}
1301#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1302#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1303function IGCM_sys_GetBuffer {
1304  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1305
1306  typeset DEST buf_liste target file_work
1307  typeset NB_ESSAI DELAI status i
1308
1309  if ( $DEBUG_sys ) ; then
1310    echo "IGCM_sys_GetBuffer :" $@
1311  fi
1312
1313  # number of tentative
1314  NB_ESSAI=3
1315  # time delay between tentative
1316  DELAI=2
1317
1318  if [ $DRYRUN -le 2 ]; then
1319    if [ X${1} = X'/l' ] ; then
1320      eval set +A buf_liste \${${2}}
1321    else
1322      eval set +A buf_liste ${1}
1323    fi
1324    eval DEST=\${${#}}
1325
1326    #USUAL WAY
1327    if [ X${1} = X'/l' ] ; then
1328      for target in ${buf_liste[*]} ; do
1329        local_file=$( basename ${target} )
1330        i=0
1331        while [ $i -lt $NB_ESSAI ] ; do
1332          \cp ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
1333          status=$?
1334          if [ ${status} -gt 0 ]; then
1335            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1336            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1337            sleep $DELAI
1338          else
1339            break
1340          fi
1341          (( i = i + 1 ))
1342        done
1343        if [ ${status} -gt 0 ] ; then
1344          echo "IGCM_sys_Get : error"
1345          cat /tmp/out_command.$$
1346          \rm /tmp/out_command.$$
1347          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1348        else
1349          \rm /tmp/out_command.$$
1350        fi
1351      done
1352    else
1353      i=0
1354      while [ $i -lt $NB_ESSAI ] ; do
1355        \cp ${buf_liste} ${DEST} >> /tmp/out_command.$$ 2>&1
1356        status=$?
1357        if [ ${status} -gt 0 ]; then
1358          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1359          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1360          sleep $DELAI
1361        else
1362          break
1363        fi
1364        (( i = i + 1 ))
1365      done
1366      if [ ${status} -gt 0 ] ; then
1367        echo "IGCM_sys_Get : error"
1368        cat /tmp/out_command.$$
1369        \rm /tmp/out_command.$$
1370        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1371      else
1372        \rm /tmp/out_command.$$
1373      fi
1374    fi
1375  fi
1376  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1377}
1378
1379#D-#==================================================
1380#D-function IGCM_sys_GetDate_FichWork
1381#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1382#D-* Examples:
1383#D-
1384function IGCM_sys_GetDate_FichWork {
1385  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1386  if ( $DEBUG_sys ) ; then
1387    echo "IGCM_sys_GetDate_FichWork :" $@
1388  fi
1389  # donne la date filesys d'un fichier sur la machine work
1390  IGCM_debug_PopStack "IGCM_sys_FichWork"
1391}
1392
1393#D-#==================================================
1394#D-function IGCM_sys_GetDate_FichArchive
1395#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1396#D-* Examples:
1397#D-
1398function IGCM_sys_GetDate_FichArchive {
1399  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1400  if ( $DEBUG_sys ) ; then
1401    echo "IGCM_sys_GetDate_FichArchive :" $@
1402  fi
1403  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1404}
1405
1406#D-#==================================================
1407#D-function IGCM_sys_Dods_Rm
1408#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1409#D-* Examples:
1410#D-
1411function IGCM_sys_Dods_Rm {
1412  if ( $DEBUG_sys ) ; then
1413    echo "IGCM_sys_Dods_Rm :" $@
1414  fi
1415  return 0
1416}
1417
1418#D-#==================================================
1419#D-function IGCM_sys_Dods_Cp
1420#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1421#D-* Examples:
1422#D-
1423function IGCM_sys_Dods_Cp {
1424  if ( $DEBUG_sys ) ; then
1425    echo "IGCM_sys_Dods_Cp :" $@
1426  fi
1427  return 0
1428}
1429
1430#D-#==================================================
1431#D-function IGCM_sys_Put_Dods
1432#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
1433#D-* Examples:
1434#D-
1435function IGCM_sys_Put_Dods {
1436  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1437  if ( $DEBUG_sys ) ; then
1438    echo "IGCM_sys_Put_Dods :" $@
1439  fi
1440  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1441}
1442
1443##############################################################
1444# REBUILD OPERATOR
1445
1446#D-#==================================================
1447#D-function IGCM_sys_rebuild
1448#D-* Purpose: rebuild parallel files
1449#D-* Examples:
1450#D-
1451function IGCM_sys_rebuild {
1452  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1453  if ( $DEBUG_sys ) ; then
1454    echo "IGCM_sys_rebuild :" $@
1455  fi
1456
1457  typeset NB_ESSAI DELAI status i firstArg
1458  # number of tentative
1459  NB_ESSAI=3
1460  # time delay between tentative
1461  DELAI=2
1462
1463  i=0
1464  while [ $i -lt $NB_ESSAI ] ; do
1465    /gpfs1/home/sebastien/REBUILD/rebuild -f -o $@ > /tmp/out_command.$$ 2>&1
1466    status=$?
1467    if [ ${status} -gt 0 ] ; then
1468      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1469      cat /tmp/out_command.$$
1470      \rm /tmp/out_command.$$
1471      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1472      firstArg=${1}
1473      \rm ${firstArg}
1474      sleep $DELAI
1475    else
1476      \rm /tmp/out_command.$$
1477      break
1478    fi
1479    (( i = i + 1 ))
1480  done
1481
1482  if [ ${status} -gt 0 ] ; then
1483    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1484    IGCM_debug_Exit "rebuild"
1485  fi
1486
1487  IGCM_debug_PopStack "IGCM_sys_rebuild"
1488}
1489
1490#D-#==================================================
1491#D-function IGCM_sys_rebuild_station
1492#D-* Purpose: rebuild parallel files describing station
1493#D-* Examples:
1494#D-
1495function IGCM_sys_rebuild_station {
1496  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1497  typeset i list_opt file_in file_out prefix_invert list_invert
1498  if ( $DEBUG_sys ) ; then
1499    echo "IGCM_sys_rebuild_station :" $@
1500  fi
1501  list_opt=$@
1502
1503  # Invert Axis : t,x -> x,t
1504  #               t,pres,x -> x,t,pres
1505  # So that we can concatenate along x
1506  i=0
1507  for file_in in ${list_opt} ; do
1508    (( i = i + 1))
1509    [ ${i} = 1 ] && file_out=${file_in} && continue
1510    # detect time counter and do the job only if present
1511    var_unlim=$(ncdump -h ${file_in} | grep UNLIMITED | cut -d ' ' -f 1)
1512    if [ X${var_unlim} = Xtime_counter ] ; then
1513      prefix_invert=$( basename ${file_in} .nc )
1514      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1515      list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1516    fi
1517  done
1518
1519  # Concatenate
1520  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1521
1522  # Re-ivert file
1523  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1524
1525  # Station re-ordering is too expansive to be run within libIGCM
1526  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1527  # This re-ordering must be done "in memory" by the cmorization process
1528  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1529  # BEGIN reordering
1530
1531  # Only LMDZ text output contains the exact ordering of the station.
1532  # We isolate this in the code below:
1533  #  0  38  -157.5000000000000  70.98591549295774
1534  #  0  54  27.49999999999999   67.18309859154928
1535  #  0  56  -62.50000000000001  82.39436619718309
1536  #  0  79  12.49999999999999   78.59154929577466
1537  #  0  116 -165.0000000000000  76.05633802816901
1538  #  0  117 130.0000000000000   70.98591549295774
1539  #  0  118 110.0000000000000   87.46478873239437
1540  #  1  40  4.999999999999995   51.97183098591550
1541#  typeset iStation iProc list_opt file_in file_out prefix_invert
1542#  typeset -Z4 j4
1543#  typeset -Z3 j3
1544
1545#  unset list_opt
1546#  set +A list_opt $@
1547
1548  # Filename after rebuild
1549#  file_out=${list_opt[0]}
1550  # Prefix of output files
1551#  prefix_invert=$( basename ${file_out} .nc )
1552  # Number of procs
1553#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1554
1555#  iProc=0
1556#  while [ ${iProc} -lt ${num_proc} ] ; do
1557    # Array containing Station as a number
1558#    unset proc_stn
1559#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | gawk ' {print $2}' )
1560    # Number of stations produced by processor proc
1561#    stationLast=${#proc_stn[*]}
1562    # Proc number on 4 digits
1563#    j4=${iProc}
1564    # Init
1565#    iStation=0
1566#    while [ ${iStation} -lt ${stationLast} ] ; do
1567      # Station number on 3 digits
1568#      j3=${proc_stn[${iStation}]}
1569      # Extract station
1570      # Invert Axis : t,x -> x,t
1571      #               t,pres,x -> x,t,pres
1572      # So that we can concatenate along x
1573#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1574#      (( iStation = iStation + 1 ))
1575#    done
1576#    (( iProc = iProc + 1 ))
1577#  done
1578
1579  # Concatenate all station along x
1580#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1581
1582  # Re-invert file
1583#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1584
1585  # END reordering
1586
1587  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1588}
1589
1590############################################################
1591# Activate Running Environnment Variables
1592
1593#D-#==================================================
1594#D-function IGCM_sys_desactiv_variables
1595#D-* Purpose: set environement variables prior to execution
1596#D-* Examples:
1597#D-
1598function IGCM_sys_activ_variables {
1599  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1600  if ( $DEBUG_sys ) ; then
1601    echo "IGCM_sys_activ_variables"
1602  fi
1603
1604# --------------------------------------------------------------------
1605#D- MPI specifications
1606# --------------------------------------------------------------------
1607
1608# --------------------------------------------------------------------
1609#D- Other specifications
1610# --------------------------------------------------------------------
1611
1612  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1613}
1614
1615############################################################
1616# Desactivate Running Environnment Variables
1617
1618#D-#==================================================
1619#D-function IGCM_sys_desactiv_variables
1620#D-* Purpose: unset environement variables after execution
1621#D-* Examples:
1622#D-
1623function IGCM_sys_desactiv_variables {
1624  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1625  if ( $DEBUG_sys ) ; then
1626    echo "IGCM_sys_desactiv_variables"
1627  fi
1628# --------------------------------------------------------------------
1629#D- MPI specifications
1630# --------------------------------------------------------------------
1631
1632# --------------------------------------------------------------------
1633#D- Other specifications
1634# --------------------------------------------------------------------
1635
1636  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1637}
1638
1639############################################################
1640# Build MPI/OMP scripts run file (dummy function)
1641
1642#D-#==================================================
1643#D-function IGCM_sys_build_run_file
1644#D-* Purpose: build run file (deprecated)
1645#D-* Examples:
1646#D-
1647function IGCM_sys_build_run_file {
1648
1649  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1650
1651}
1652
1653############################################################
1654# Build MPI/OMP scripts
1655
1656#D-#==================================================
1657#D-function IGCM_sys_build_execution_scripts
1658#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1659#D-* Examples:
1660#D-
1661function IGCM_sys_build_execution_scripts
1662{
1663  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1664  if ( $DEBUG_sys ) ; then
1665    echo "IGCM_sys_build_execution_scripts " $@
1666  fi
1667
1668  typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
1669
1670  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1671    IGCM_debug_Exit "IGCM_sys_iitm build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1672  fi
1673
1674  if ( ${OK_PARA_MPMD} ) ; then
1675
1676    if [ -f run_file ] ; then
1677      IGCM_sys_Rm -f run_file
1678    fi
1679    touch run_file
1680
1681    if ( ${OK_PARA_OMP} ) ; then
1682# NEW : 2 Noeuds
1683# @ task_geometry={(0)(1,2,3)}
1684# Nombre de processus demandes
1685      echo "Job_${config_UserChoices_JobName} includes task_geometry = \c"
1686      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g"
1687      echo "Job_${config_UserChoices_JobName} includes task_geometry  with NbNodes = \c"
1688      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c
1689      NbNodes_Job=$(( $( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c ) - 1 ))
1690
1691      if [ ${NbNodes_Job} -eq 0 ] ; then
1692        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no task_geometry defined with OpenMP run."
1693        exit 1
1694      fi
1695    else
1696
1697# OLD :
1698# @ total_tasks = 3
1699# @ environment = "BATCH_NUM_PROC_TOT=3"
1700# Pas d OpenMP
1701# @ resources = ConsumableCpus(1)
1702
1703      echo "Job_${config_UserChoices_JobName} includes total_tasks = \c"
1704      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //"
1705      NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //" )
1706      NbProc_Job=${NbProc_Job:=0}
1707      if [ ${NbProc_Job} -eq 0 ] ; then
1708        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no total_tasks defined with MPI only run."
1709        exit 1
1710      fi
1711      if ( $( egrep '^# *@ *resources *= *ConsumableCpus\(1\)' ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} >/dev/null 2>&1 ) ) ; then
1712        IGCM_debug_Print 2 "ressources =  ConsumableCpus(1) line found into Job_${config_UserChoices_JobName}"
1713      else
1714        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources = line not found. Job should include resources = ConsumableCpus(1) "
1715        exit 1
1716      fi
1717    fi
1718
1719# run_file construction
1720
1721# Then first loop on the components for the coupler ie oasis
1722
1723### the coupler ie oasis must be the first one
1724    for comp in ${config_ListOfComponents[*]} ; do
1725
1726      eval ExeNameIn=\${config_Executable_${comp}[0]}
1727      eval ExeNameOut=\${config_Executable_${comp}[1]}
1728
1729      # for CPL component only
1730      if [ "X${comp}" = "XCPL" ] ; then
1731
1732        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1733        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1734
1735        if ( ${OK_PARA_MPI} ) ; then
1736
1737          (( mpi_count = 1 ))
1738          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1739            if ( ${OK_PARA_OMP} ) ; then
1740              echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
1741            else
1742              echo "./${ExeNameOut}" >> run_file
1743            fi
1744            (( mpi_count = mpi_count + 1 ))
1745          done
1746        else
1747          if ( ${OK_PARA_OMP} ) ; then
1748            echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
1749          else
1750            echo "./${ExeNameOut} " >> run_file
1751          fi
1752        fi
1753      fi
1754    done
1755
1756# Then second loop on the components
1757
1758    for comp in ${config_ListOfComponents[*]} ; do
1759
1760      eval ExeNameIn=\${config_Executable_${comp}[0]}
1761      eval ExeNameOut=\${config_Executable_${comp}[1]}
1762
1763      # Only if we really have an executable for the component and not the coupler ie oasis:
1764      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1765
1766        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1767        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1768
1769        if ( ${OK_PARA_MPI} ) ; then
1770
1771          (( mpi_count = 1 ))
1772          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1773            if ( ${OK_PARA_OMP} ) ; then
1774              echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
1775            else
1776              echo "./${ExeNameOut}" >> run_file
1777            fi
1778            (( mpi_count = mpi_count + 1 ))
1779          done
1780        else
1781          if ( ${OK_PARA_OMP} ) ; then
1782            echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} " >> run_file
1783          else
1784                    # to be tested : no MPI only OpenMP into MPMD mode
1785            echo "./${ExeNameOut} " >> run_file
1786          fi
1787        fi
1788      fi
1789    done
1790
1791    EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1792
1793    IGCM_sys_Chmod u+x run_file
1794    if ( $DEBUG_sys ) ; then
1795      echo "run_file contains : "
1796      cat run_file
1797    fi
1798
1799  else # Only one executable. launch it.
1800
1801    for comp in ${config_ListOfComponents[*]} ; do
1802
1803      # Only if we really have an executable for the component :
1804      eval ExeNameOut=\${config_Executable_${comp}[1]}
1805      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1806        EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1807      fi
1808
1809    done
1810
1811  fi
1812
1813  IGCM_debug_Print 1 "sys iitm : execution command is"
1814  IGCM_debug_Print 1 "$EXECUTION"
1815
1816  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1817}
1818
1819#D-#==================================================
1820#D-function IGCM_sys_check_path
1821#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1822#D-* do not point to an important use directory. Stop immediately in that case.
1823#D-* Examples:
1824#D-
1825function IGCM_sys_check_path {
1826  IGCM_debug_PushStack "IGCM_sys_check_path"
1827  if ( $DEBUG_sys ) ; then
1828    echo "IGCM_sys_check_path"
1829  fi
1830
1831  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1832    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1833    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1834    IGCM_debug_Exit "This will stop the job"
1835  fi
1836  IGCM_debug_PopStack "IGCM_sys_check_path"
1837}
1838
1839#D-#==================================================
1840#D-function IGCM_sys_check_quota. Dummy call here
1841#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1842#D-* Examples:
1843#D-
1844function IGCM_sys_check_quota {
1845  IGCM_debug_PushStack "IGCM_sys_check_quota"
1846  if ( $DEBUG_sys ) ; then
1847    echo "IGCM_sys_check_quota"
1848  fi
1849  IGCM_debug_PopStack "IGCM_sys_check_quota"
1850}
1851
1852#D-#==================================================
1853#D-function IGCM_sys_CountJobInQueue
1854#D-* Purpose: Check if job_name is currently
1855#D-  running or in queue
1856#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1857#D-
1858function IGCM_sys_CountJobInQueue {
1859  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1860  if ( $DEBUG_sys ) ; then
1861    echo "IGCM_sys_CountJobInQueue"
1862  fi
1863
1864  # Print only the full (-W) JobName (%jn)
1865  NbRun=$( llq -W -f %jn | grep -c "$1" )
1866
1867  eval ${2}=${NbRun}
1868
1869  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1870}
1871
1872##############################################################
1873# NCO OPERATOR
1874
1875#D-#==================================================
1876#D-function IGCM_sys_ncap2
1877#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
1878#D-* Examples:
1879#D-
1880function IGCM_sys_ncap2 {
1881  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
1882  if ( $DEBUG_sys ) ; then
1883    echo "IGCM_sys_ncap2 :" $@
1884  fi
1885
1886  typeset NB_ESSAI DELAI status i
1887  # number of tentative
1888  NB_ESSAI=3
1889  # time delay between tentative
1890  DELAI=2
1891
1892  i=0
1893  while [ $i -lt $NB_ESSAI ] ; do
1894    ncap2 "$@" > /tmp/out_command.$$ 2>&1
1895    status=$?
1896    if [ ${status} -gt 0 ] ; then
1897      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
1898      cat /tmp/out_command.$$
1899      \rm /tmp/out_command.$$
1900      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1901      sleep $DELAI
1902    else
1903      \rm /tmp/out_command.$$
1904      break
1905    fi
1906    (( i = i + 1 ))
1907  done
1908
1909  if [ ${status} -gt 0 ] ; then
1910    echo "IGCM_sys_ncap2 : ncap2 error"
1911    IGCM_debug_Exit "ncap2"
1912  fi
1913
1914  IGCM_debug_PopStack "IGCM_sys_ncap2"
1915}
1916
1917#D-#==================================================
1918#D-function IGCM_sys_ncatted
1919#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
1920#D-* Examples:
1921#D-
1922function IGCM_sys_ncatted {
1923  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1924  if ( $DEBUG_sys ) ; then
1925    echo "IGCM_sys_ncatted :" $@
1926  fi
1927
1928  typeset NB_ESSAI DELAI status i
1929  # number of tentative
1930  NB_ESSAI=3
1931  # time delay between tentative
1932  DELAI=2
1933
1934  i=0
1935  while [ $i -lt $NB_ESSAI ] ; do
1936    ncatted "$@" > /tmp/out_command.$$ 2>&1
1937    status=$?
1938    if [ ${status} -gt 0 ] ; then
1939      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
1940      cat /tmp/out_command.$$
1941      \rm /tmp/out_command.$$
1942      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1943      sleep $DELAI
1944    else
1945      \rm /tmp/out_command.$$
1946      break
1947    fi
1948    (( i = i + 1 ))
1949  done
1950
1951  if [ ${status} -gt 0 ] ; then
1952    echo "IGCM_sys_ncatted : ncatted error"
1953    IGCM_debug_Exit "ncatted"
1954  fi
1955
1956  IGCM_debug_PopStack "IGCM_sys_ncatted"
1957}
1958
1959#D-#==================================================
1960#D-function IGCM_sys_ncbo
1961#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
1962#D-* Examples:
1963#D-
1964function IGCM_sys_ncbo {
1965  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
1966  if ( $DEBUG_sys ) ; then
1967    echo "IGCM_sys_ncbo :" $@
1968  fi
1969
1970  typeset NB_ESSAI DELAI status i
1971  # number of tentative
1972  NB_ESSAI=3
1973  # time delay between tentative
1974  DELAI=2
1975
1976  i=0
1977  while [ $i -lt $NB_ESSAI ] ; do
1978    ncbo $@ > /tmp/out_command.$$ 2>&1
1979    status=$?
1980    if [ ${status} -gt 0 ] ; then
1981      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
1982      cat /tmp/out_command.$$
1983      \rm /tmp/out_command.$$
1984      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1985      sleep $DELAI
1986    else
1987      \rm /tmp/out_command.$$
1988      break
1989    fi
1990    (( i = i + 1 ))
1991  done
1992
1993  if [ ${status} -gt 0 ] ; then
1994    echo "IGCM_sys_ncbo : ncbo error"
1995    IGCM_debug_Exit "ncbo"
1996  fi
1997
1998  IGCM_debug_PopStack "IGCM_sys_ncbo"
1999}
2000
2001#D-#==================================================
2002#D-function IGCM_sys_ncdif
2003#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2004#D-* Examples:
2005#D-
2006function IGCM_sys_ncdiff {
2007  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2008  if ( $DEBUG_sys ) ; then
2009    echo "IGCM_sys_ncdiff :" $@
2010  fi
2011
2012  typeset NB_ESSAI DELAI status i
2013  # number of tentative
2014  NB_ESSAI=3
2015  # time delay between tentative
2016  DELAI=2
2017
2018  i=0
2019  while [ $i -lt $NB_ESSAI ] ; do
2020    ncdiff $@ > /tmp/out_command.$$ 2>&1
2021    status=$?
2022    if [ ${status} -gt 0 ] ; then
2023      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2024      cat /tmp/out_command.$$
2025      \rm /tmp/out_command.$$
2026      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2027      sleep $DELAI
2028    else
2029      \rm /tmp/out_command.$$
2030      break
2031    fi
2032    (( i = i + 1 ))
2033  done
2034
2035  if [ ${status} -gt 0 ] ; then
2036    echo "IGCM_sys_ncdiff : ncdiff error"
2037    IGCM_debug_Exit "ncdiff"
2038  fi
2039
2040  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2041}
2042
2043#D-#==================================================
2044#D-function IGCM_sys_ncea
2045#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2046#D-* Examples:
2047#D-
2048function IGCM_sys_ncea {
2049  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2050  if ( $DEBUG_sys ) ; then
2051    echo "IGCM_sys_ncea :" $@
2052  fi
2053
2054  typeset NB_ESSAI DELAI status i
2055  # number of tentative
2056  NB_ESSAI=3
2057  # time delay between tentative
2058  DELAI=2
2059
2060  i=0
2061  while [ $i -lt $NB_ESSAI ] ; do
2062    ncea $@ > /tmp/out_command.$$ 2>&1
2063    status=$?
2064    if [ ${status} -gt 0 ] ; then
2065      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2066      cat /tmp/out_command.$$
2067      \rm /tmp/out_command.$$
2068      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2069      sleep $DELAI
2070    else
2071      \rm /tmp/out_command.$$
2072      break
2073    fi
2074    (( i = i + 1 ))
2075  done
2076
2077  if [ ${status} -gt 0 ] ; then
2078    echo "IGCM_sys_ncea : ncea error"
2079    IGCM_debug_Exit "ncea"
2080  fi
2081
2082  IGCM_debug_PopStack "IGCM_sys_ncea"
2083}
2084
2085#D-#==================================================
2086#D-function IGCM_sys_ncecat
2087#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2088#D-* Examples:
2089#D-
2090function IGCM_sys_ncecat {
2091  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2092  if ( $DEBUG_sys ) ; then
2093    echo "IGCM_sys_ncecat :" $@
2094  fi
2095
2096  typeset NB_ESSAI DELAI status i
2097  # number of tentative
2098  NB_ESSAI=3
2099  # time delay between tentative
2100  DELAI=2
2101
2102  i=0
2103  while [ $i -lt $NB_ESSAI ] ; do
2104    ncecat $@ > /tmp/out_command.$$ 2>&1
2105    status=$?
2106    if [ ${status} -gt 0 ] ; then
2107      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2108      cat /tmp/out_command.$$
2109      \rm /tmp/out_command.$$
2110      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2111      sleep $DELAI
2112    else
2113      \rm /tmp/out_command.$$
2114      break
2115    fi
2116    (( i = i + 1 ))
2117  done
2118
2119  if [ ${status} -gt 0 ] ; then
2120    echo "IGCM_sys_ncecat : ncecat error"
2121    IGCM_debug_Exit "ncecat"
2122  fi
2123
2124  IGCM_debug_PopStack "IGCM_sys_ncecat"
2125}
2126
2127#D-#==================================================
2128#D-function IGCM_sys_ncflint
2129#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2130#D-* Examples:
2131#D-
2132function IGCM_sys_ncflint {
2133  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2134  if ( $DEBUG_sys ) ; then
2135    echo "IGCM_sys_ncflint :" $@
2136  fi
2137
2138  typeset NB_ESSAI DELAI status i
2139  # number of tentative
2140  NB_ESSAI=3
2141  # time delay between tentative
2142  DELAI=2
2143
2144  i=0
2145  while [ $i -lt $NB_ESSAI ] ; do
2146    ncflint $@ > /tmp/out_command.$$ 2>&1
2147    status=$?
2148    if [ ${status} -gt 0 ] ; then
2149      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2150      cat /tmp/out_command.$$
2151      \rm /tmp/out_command.$$
2152      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2153      sleep $DELAI
2154    else
2155      \rm /tmp/out_command.$$
2156      break
2157    fi
2158    (( i = i + 1 ))
2159  done
2160
2161  if [ ${status} -gt 0 ] ; then
2162    echo "IGCM_sys_ncflint : ncflint error"
2163    IGCM_debug_Exit "ncflint"
2164  fi
2165
2166  IGCM_debug_PopStack "IGCM_sys_ncflint"
2167}
2168
2169#D-#==================================================
2170#D-function IGCM_sys_ncks
2171#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2172#D-* Examples:
2173#D-
2174function IGCM_sys_ncks {
2175  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2176  if ( $DEBUG_sys ) ; then
2177    echo "IGCM_sys_ncks :" $@
2178  fi
2179
2180  typeset NB_ESSAI DELAI status i
2181  # number of tentative
2182  NB_ESSAI=3
2183  # time delay between tentative
2184  DELAI=2
2185
2186  i=0
2187  while [ $i -lt $NB_ESSAI ] ; do
2188    ncks $@ > /tmp/out_command.$$ 2>&1
2189    status=$?
2190    if [ ${status} -gt 0 ] ; then
2191      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2192      cat /tmp/out_command.$$
2193      \rm /tmp/out_command.$$
2194      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2195      sleep $DELAI
2196    else
2197      \rm /tmp/out_command.$$
2198      break
2199    fi
2200    (( i = i + 1 ))
2201  done
2202
2203  if [ ${status} -gt 0 ] ; then
2204    echo "IGCM_sys_ncks : ncks error"
2205    IGCM_debug_Exit "ncks"
2206  fi
2207
2208  IGCM_debug_PopStack "IGCM_sys_ncks"
2209}
2210
2211#D-#==================================================
2212#D-function IGCM_sys_ncpdq
2213#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2214#D-* Examples:
2215#D-
2216function IGCM_sys_ncpdq {
2217  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2218  if ( $DEBUG_sys ) ; then
2219    echo "IGCM_sys_ncpdq :" $@
2220  fi
2221
2222  typeset NB_ESSAI DELAI status i
2223  # number of tentative
2224  NB_ESSAI=3
2225  # time delay between tentative
2226  DELAI=2
2227
2228  i=0
2229  while [ $i -lt $NB_ESSAI ] ; do
2230    ncpdq $@ > /tmp/out_command.$$ 2>&1
2231    status=$?
2232    if [ ${status} -gt 0 ] ; then
2233      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2234      cat /tmp/out_command.$$
2235      \rm /tmp/out_command.$$
2236      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2237      sleep $DELAI
2238    else
2239      \rm /tmp/out_command.$$
2240      break
2241    fi
2242    (( i = i + 1 ))
2243  done
2244
2245  if [ ${status} -gt 0 ] ; then
2246    echo "IGCM_sys_ncpdq : ncpdq error"
2247    IGCM_debug_Exit "ncpdq"
2248  fi
2249
2250  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2251}
2252
2253#D-#==================================================
2254#D-function IGCM_sys_ncra
2255#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2256#D-* Examples:
2257#D-
2258function IGCM_sys_ncra {
2259  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2260  if ( $DEBUG_sys ) ; then
2261    echo "IGCM_sys_ncra :" $@
2262  fi
2263
2264  typeset NB_ESSAI DELAI status i
2265  # number of tentative
2266  NB_ESSAI=3
2267  # time delay between tentative
2268  DELAI=2
2269
2270  i=0
2271  while [ $i -lt $NB_ESSAI ] ; do
2272    ncra $@ > /tmp/out_command.$$ 2>&1
2273    status=$?
2274    if [ ${status} -gt 0 ] ; then
2275      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2276      cat /tmp/out_command.$$
2277      \rm /tmp/out_command.$$
2278      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2279      sleep $DELAI
2280    else
2281      \rm /tmp/out_command.$$
2282      break
2283    fi
2284    (( i = i + 1 ))
2285  done
2286
2287  if [ ${status} -gt 0 ] ; then
2288    echo "IGCM_sys_ncra : ncra error"
2289    IGCM_debug_Exit "ncra"
2290  fi
2291
2292  IGCM_debug_PopStack "IGCM_sys_ncra"
2293}
2294
2295#D-#==================================================
2296#D-function IGCM_sys_ncrcat
2297#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
2298#D-* Examples:
2299#D-
2300function IGCM_sys_ncrcat {
2301  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2302  if ( $DEBUG_sys ) ; then
2303    echo "IGCM_sys_ncrcat :" $@
2304  fi
2305
2306  typeset NB_ESSAI DELAI status i lastArg
2307  # number of tentative
2308  NB_ESSAI=3
2309  # time delay between tentative
2310  DELAI=2
2311
2312  i=0
2313  while [ $i -lt $NB_ESSAI ] ; do
2314    ncrcat $@ > /tmp/out_command.$$ 2>&1
2315    status=$?
2316    if [ ${status} -gt 0 ] ; then
2317      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2318      cat /tmp/out_command.$$
2319      \rm /tmp/out_command.$$
2320      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2321      sleep $DELAI
2322    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" /tmp/out_command.$$ )" = "X" ] ; then
2323      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2324      cat /tmp/out_command.$$
2325      # remove files having corrupted time axis
2326      eval lastArg=\${$#}
2327      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2328      \rm ${lastArg}
2329      \rm /tmp/out_command.$$
2330      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2331      sleep $DELAI
2332    else
2333      \rm /tmp/out_command.$$
2334      break
2335    fi
2336    (( i = i + 1 ))
2337  done
2338
2339  if [ ${status} -gt 0 ] ; then
2340    echo "IGCM_sys_ncrcat : ncrcat error"
2341      #IGCM_debug_Exit "ncrcat"
2342  fi
2343
2344  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2345}
2346
2347#D-#==================================================
2348#D-function IGCM_sys_ncrename
2349#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
2350#D-* Examples:
2351#D-
2352function IGCM_sys_ncrename {
2353  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2354  if ( $DEBUG_sys ) ; then
2355    echo "IGCM_sys_ncrename :" $@
2356  fi
2357
2358  typeset NB_ESSAI DELAI status i
2359  # number of tentative
2360  NB_ESSAI=3
2361  # time delay between tentative
2362  DELAI=2
2363
2364  i=0
2365  while [ $i -lt $NB_ESSAI ] ; do
2366    ncrename $@ > /tmp/out_command.$$ 2>&1
2367    status=$?
2368    if [ ${status} -gt 0 ] ; then
2369      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2370      cat /tmp/out_command.$$
2371      \rm /tmp/out_command.$$
2372      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2373      sleep $DELAI
2374    else
2375      \rm /tmp/out_command.$$
2376      break
2377    fi
2378    (( i = i + 1 ))
2379  done
2380
2381  if [ ${status} -gt 0 ] ; then
2382    echo "IGCM_sys_ncrename : ncrename error"
2383    IGCM_debug_Exit "ncrename"
2384  fi
2385
2386  IGCM_debug_PopStack "IGCM_sys_ncrename"
2387}
2388
2389#D-#==================================================
2390#D-function IGCM_sys_ncwa
2391#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
2392#D-* Examples:
2393#D-
2394function IGCM_sys_ncwa {
2395  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2396  if ( $DEBUG_sys ) ; then
2397    echo "IGCM_sys_ncwa :" $@
2398  fi
2399
2400  typeset NB_ESSAI DELAI status i
2401  # number of tentative
2402  NB_ESSAI=3
2403  # time delay between tentative
2404  DELAI=2
2405
2406  i=0
2407  while [ $i -lt $NB_ESSAI ] ; do
2408    ncwa $@ > /tmp/out_command.$$ 2>&1
2409    status=$?
2410    if [ ${status} -gt 0 ] ; then
2411      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2412      cat /tmp/out_command.$$
2413      \rm /tmp/out_command.$$
2414      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2415      sleep $DELAI
2416    else
2417      \rm /tmp/out_command.$$
2418      break
2419    fi
2420    (( i = i + 1 ))
2421  done
2422
2423  if [ ${status} -gt 0 ] ; then
2424    echo "IGCM_sys_ncwa : ncwa error"
2425    IGCM_debug_Exit "ncwa"
2426  fi
2427
2428  IGCM_debug_PopStack "IGCM_sys_ncwa"
2429}
2430
2431##############################################################
2432# CDO OPERATOR
2433
2434#D-#==================================================
2435#D-function IGCM_sys_cdo
2436#D-* Purpose: encapsulate cdo call so as to manage error code and retry
2437#D-* Examples:
2438#D-
2439function IGCM_sys_cdo {
2440  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2441  if ( $DEBUG_sys ) ; then
2442    echo "IGCM_sys_cdo :" $@
2443  fi
2444
2445  typeset status
2446
2447  \cdo $@ > /tmp/out_command.$$ 2>&1
2448  status=$?
2449  if [ ${status} -gt 0 ] ; then
2450    echo "IGCM_sys_cdo : error code ${status}"
2451    cat /tmp/out_command.$$
2452    \rm /tmp/out_command.$$
2453    IGCM_debug_PopStack "IGCM_sys_cdo"
2454    return 1
2455  else
2456    IGCM_debug_PopStack "IGCM_sys_cdo"
2457    return 0
2458  fi
2459
2460  IGCM_debug_PopStack "IGCM_sys_cdo"
2461}
Note: See TracBrowser for help on using the repository browser.