source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh @ 801

Last change on this file since 801 was 801, checked in by sdipsl, 11 years ago
  • Some adaptation so as to find and prepare tared restart on Ada.
  • Property svn:keywords set to Revision Author Date
File size: 67.6 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sébastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Ada
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe param files |            |  Chmod  |                           |
46# |          |      Qsub           |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=ada
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# Submit command
78typeset SUBMIT=${SUBMIT:=llsubmit}
79# rsync with path
80typeset -r RSYNC=/usr/bin/rsync
81# RSYNC_opt args to rsync
82typeset -r RSYNC_opt="-va"
83# ie storage filesystem
84typeset -r STOREHOST=gaya
85typeset -r REMOTE_RSYNC=/u/rech/ces/rces452/RSYNC/bin/rsync
86
87#====================================================
88# Source default environment
89#====================================================
90##. /etc/profile
91
92#====================================================
93# Set environment tools (ferret, nco, cdo)
94#====================================================
95if [ X${TaskType} = Xcomputing ] ; then
96  echo
97  #source /smplocal/pub/Modules/default/init/ksh
98  #. /smphome/rech/psl/rpsl035/.atlas_env_ada_bash
99  #export PATH=${PATH}:/smphome/rech/psl/rpsl035/AddNoise/src_X64/bin
100else
101  source /smplocal/pub/Modules/default/init/ksh
102  . /smphome/rech/psl/rpsl035/.atlas_env_ada_bash
103fi
104
105#====================================================
106# Host specific DIRECTORIES
107#====================================================
108
109#====================================================
110#- MirrorlibIGCM for frontend
111typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
112
113#====================================================
114#- libIGCM_POST for frontend
115typeset -r libIGCM_POST=${libIGCM}
116
117#====================================================
118#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
119typeset -r R_EXE="${MODIPSL}/bin"
120
121#====================================================
122#- SUBMIT_DIR : submission dir
123typeset SUBMIT_DIR=${SUBMIT_DIR:=${LOADL_STEP_INITDIR}}
124
125#====================================================
126#- IN
127typeset -r R_IN=${R_IN:=/workgpfs/rech/psl/rpsl035/IGCM}
128typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/u/rech/psl/rpsl376}
129
130#====================================================
131#- ARCHIVE (dedicated to large files)
132typeset ARCHIVE=$( echo ${HOME} | sed -e "s/smphome/u/" )
133
134#====================================================
135#- STORAGE (dedicated to small/medium files)
136typeset STORAGE=${WORKDIR}
137
138#====================================================
139#- R_OUT
140typeset R_OUT=${ARCHIVE}/IGCM_OUT
141
142#====================================================
143#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
144typeset R_FIG=${ARCHIVE}/IGCM_OUT
145
146#====================================================
147#- R_BUF (ONLY FOR double copy an scratch)
148typeset -r R_BUF=${WORKDIR}/IGCM_OUT
149
150#====================================================
151#- BIG_DIR : BIG_DIR to store files waiting for rebuild
152typeset -r BIG_DIR=${BIG_DIR:=${WORKDIR}/REBUILD}
153
154#====================================================
155#- RUN_DIR_PATH : Temporary working directory (=> TMP)
156typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
157
158#====================================================
159#- HOST_MPIRUN_COMMAND
160typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
161
162#====================================================
163#- Max number of arguments passed to nco operator or demigration command
164UNIX_MAX_LIMIT=360
165
166#====================================================
167#- set PackDefault true on curie
168PackDefault=true
169
170#====================================================
171#- Number of core per node (max number of OpenMP task)
172NUM_COREPERNODE=8
173
174#====================================================
175#- Default number of MPI task for IPSL coupled model
176#- required for backward compatibility
177#-
178DEFAULT_NUM_PROC_OCE=5
179DEFAULT_NUM_PROC_CPL=1
180(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
181DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
182
183#D-#==================================================
184#D-function IGCM_sys_ChangeArchive
185#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
186#D-* Examples:
187#D-
188function IGCM_sys_ChangeArchive {
189  IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
190
191  R_OUT=${WORKDIR}/IGCM_OUT
192  R_FIG=${WORKDIR}/IGCM_OUT
193
194  IGCM_debug_Print 1 "R_OUT   has been redefined = ${R_OUT}"
195  IGCM_debug_Print 1 "R_FIG   has been redefined = ${R_FIG}"
196
197  IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
198}
199
200#D-#==================================================
201#D-function IGCM_sys_RshMaster
202#D-* Purpose: Connection to frontend machine.
203#D-* Examples:
204#D-
205function IGCM_sys_RshMaster {
206  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
207  /bin/ksh <<-EOF
208    export libIGCM=${libIGCM}
209    export DEBUG_debug=${DEBUG_debug}
210    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
211    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
212    ${@}
213EOF
214  if [ $? -gt 0 ] ; then
215    echo "IGCM_sys_RshMaster : erreur."
216    IGCM_debug_Exit "IGCM_sys_RshMaster"
217  fi
218  IGCM_debug_PopStack "IGCM_sys_RshMaster"
219}
220
221#D-#==================================================
222#D-function IGCM_sys_RshArchive
223#D-* Purpose: Archive rsh command
224#D-* Examples:
225#D-
226function IGCM_sys_RshArchive {
227  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
228  DEBUG_sys=false IGCM_sys_IsFileArchived "$@"
229  if [ $? = 0 ] ; then
230    rsh ${STOREHOST} exec /bin/ksh <<-EOF
231    ${@}
232EOF
233    status=$?
234  else
235    /bin/ksh <<-EOF
236    ${@}
237EOF
238    status=$?
239  fi
240  if [ ${status} -gt 0 ] ; then
241    IGCM_debug_Print 2 "IGCM_sys_RshArchive : rsh or command failed error code ${status}"
242    IGCM_debug_Exit "IGCM_sys_RshArchive"
243  fi
244  IGCM_debug_PopStack "IGCM_sys_RshArchive"
245}
246
247#D-#==================================================
248#D-function IGCM_sys_RshPost
249#D-* Purpose: Post-process rsh command
250#D-* Examples:
251#D-
252function IGCM_sys_RshPost {
253  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
254  if ( $DEBUG_sys ) ; then
255    echo "IGCM_sys_RshPost :" $@
256  fi
257
258  # keep standard input to keep it for postpone if ulam don't answer
259  cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
260
261  /bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
262  if [ $? -gt 0 ] ; then
263    echo "IGCM_sys_RshPost : erreur."
264    IGCM_debug_Exit "IGCM_sys_RshPost"
265  fi
266  # delete temporary file
267  \rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
268
269  IGCM_debug_PopStack "IGCM_sys_RshPost"
270}
271
272#D-#==================================================
273#D-function IGCM_sys_SendMail
274#D-* Purpose: Send mail when simulation is over
275#D-* Examples:
276#D-
277function IGCM_sys_SendMail {
278  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
279  if ( $DEBUG_sys ) ; then
280    echo "IGCM_sys_SendMail :" $@
281  fi
282
283  if ( ${ExitFlag} ) ; then
284    status=failed
285  else
286    status=completed
287  fi
288
289  cat  << END_MAIL > job_end.mail
290Dear ${LOGIN},
291
292  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
293  Job started : ${DateBegin}
294  Job ended   : ${DateEnd}
295  Output files are available in ${R_SAVE}
296  Files to be rebuild are temporarily available in ${REBUILD_DIR}
297  Pre-packed files are temporarily available in ${R_BUFR}
298  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
299END_MAIL
300
301  if [ ! -z ${config_UserChoices_MailName} ] ; then
302    mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
303  elif [ -f ~/.forward ] ; then
304    mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
305  else
306    mail -s "${config_UserChoices_JobName} ${status}" ${USER} < job_end.mail
307  fi
308
309  sleep 10
310  rm -f job_end.mail
311
312  if [ $? -gt 0 ] ; then
313    echo "IGCM_sys_SendMail : erreur."
314    IGCM_debug_Exit "IGCM_sys_SendMail"
315  fi
316  IGCM_debug_PopStack "IGCM_sys_SendMail"
317}
318
319#D-#==================================================
320#D-function IGCM_sys_Mkdir
321#D-* Purpose: Master locale mkdir command
322#D-* Examples:
323#D-
324function IGCM_sys_Mkdir {
325  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
326  if ( $DEBUG_sys ) ; then
327    echo "IGCM_sys_Mkdir :" $@
328  fi
329  if [ ! -d ${1} ]; then
330    \mkdir -p $1
331    if [ $? -gt 0 ] ; then
332      echo "IGCM_sys_Mkdir : erreur."
333      IGCM_debug_Exit "IGCM_sys_Mkdir"
334    fi
335  fi
336  # vérification :
337  if [ ! -d ${1} ] ; then
338    echo "IGCM_sys_Mkdir : erreur."
339    IGCM_debug_Exit "IGCM_sys_Mkdir"
340  fi
341  IGCM_debug_PopStack "IGCM_sys_Mkdir"
342}
343
344#D-#==================================================
345#D-function IGCM_sys_MkdirArchive
346#D-* Purpose: Mkdir on Archive
347#D-* Examples:
348#D-
349function IGCM_sys_MkdirArchive {
350  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
351  if ( $DEBUG_sys ) ; then
352    echo "IGCM_sys_MkdirArchive :" $@
353  fi
354  #- creation de repertoire sur le serveur fichier
355  DEBUG_sys=false IGCM_sys_IsFileArchived $1
356  if [ $? = 0 ] ; then
357    rsh ${STOREHOST} -n mkdir -p $1
358    status=$?
359  else
360    mkdir -p $1
361    status=$?
362  fi 
363
364  if [ ${status} -gt 0 ] ; then
365    IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : rsh or mkdir failed error code ${status}"
366    IGCM_debug_Exit "IGCM_sys_MkdirArchive"
367  fi
368  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
369}
370
371#D-#==================================================
372#D-function IGCM_sys_MkdirWork
373#D-* Purpose: Mkdir on Work
374#D-* Examples:
375#D-
376function IGCM_sys_MkdirWork {
377  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
378  if ( $DEBUG_sys ) ; then
379    echo "IGCM_sys_MkdirWork :" $@
380  fi
381  #- creation de repertoire sur le serveur fichier
382  if [ ! -d ${1} ]; then
383    \mkdir -p $1
384    if [ $? -gt 0 ] ; then
385      echo "IGCM_sys_MkdirWork : erreur."
386      IGCM_debug_Exit "IGCM_sys_MkdirWork"
387    fi
388  fi
389  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
390}
391
392#D-#==================================================
393#D-function IGCM_sys_Cd
394#D-* Purpose: master cd command
395#D-* Examples:
396#D-
397function IGCM_sys_Cd {
398  IGCM_debug_PushStack "IGCM_sys_Cd" $@
399  if ( $DEBUG_sys ) ; then
400    echo "IGCM_sys_Cd :" $@
401  fi
402  \cd $1
403  if [ $? -gt 0 ] ; then
404    echo "IGCM_sys_Cd : erreur."
405    IGCM_debug_Exit "IGCM_sys_Cd"
406  fi
407  IGCM_debug_PopStack "IGCM_sys_Cd"
408}
409
410#D-#==================================================
411#D-function IGCM_sys_Chmod
412#D-* Purpose: Chmod
413#D-* Examples:
414#D-
415function IGCM_sys_Chmod {
416  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
417  if ( $DEBUG_sys ) ; then
418    echo "IGCM_sys_Chmod :" $@
419  fi
420  if [ $DRYRUN -le 1 ]; then
421    \chmod $@
422    if [ $? -gt 0 ] ; then
423      echo "IGCM_sys_Chmod : erreur."
424      IGCM_debug_Exit "IGCM_sys_Chmod"
425    fi
426  else
427    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
428  fi
429  IGCM_debug_PopStack "IGCM_sys_Chmod"
430}
431
432#D-#==================================================
433#D-function IGCM_sys_FileSize
434#D-* Purpose: Filesize
435#D-* Examples:
436#D-
437function IGCM_sys_FileSize {
438  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
439
440  typeset sizeF
441  set +A sizeF -- $( ls -la ${1} )
442  if [ $? -gt 0 ] ; then
443    IGCM_debug_Exit "IGCM_sys_FileSize"
444  fi
445  eval ${2}=${sizeF[4]}
446
447  IGCM_debug_PopStack "IGCM_sys_FileSize"
448}
449
450#D-#==================================================
451#D-function IGCM_sys_TestDir
452#D-* Purpose: Test Directory that must exists
453#D-* Examples:
454#D-
455function IGCM_sys_TestDir {
456  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
457  if ( $DEBUG_sys ) ; then
458    echo "IGCM_sys_TestDir :" $@
459  fi
460  typeset ExistFlag
461  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
462  IGCM_debug_PopStack "IGCM_sys_TestDir"
463
464  return ${ExistFlag}
465}
466
467#D-#==================================================
468#D-function IGCM_sys_TestDirArchive
469#D-* Purpose: Test Directory that must exists on Archive
470#D-* Examples:
471#D-
472function IGCM_sys_TestDirArchive {
473  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
474  if ( $DEBUG_sys ) ; then
475    echo "IGCM_sys_TestDirArchive :" $@
476  fi
477  typeset ExistFlag
478  #Command depends on targeted file system
479  DEBUG_sys=false IGCM_sys_IsFileArchived $1
480  if [ $? = 0 ] ; then
481    ExistFlag=$( IGCM_sys_RshArchive "[ -d $1 ] && echo 0 || echo 1" )
482  else
483    ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
484  fi
485  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
486  return ${ExistFlag}
487}
488
489#D-#==================================================
490#D-function IGCM_sys_IsFileArchived
491#D-* Purpose: Test file that must NOT EXISTS on Archive
492#D-* Examples:
493#D-
494function IGCM_sys_IsFileArchived {
495  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
496  if ( $DEBUG_sys ) ; then
497    echo "IGCM_sys_IsFileArchived :" $@
498  fi
499  typeset IsArchivedFlag
500  IsArchivedFlag=$( [ "X$( echo $1 | grep \/u\/rech )" != "X" ] && echo 0 || echo 1 )
501  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
502
503  return ${IsArchivedFlag}
504}
505
506#D-#==================================================
507#D-function IGCM_sys_TestFileArchive
508#D-* Purpose: Test file that must NOT EXISTS on Archive
509#D-* Examples:
510#D-
511function IGCM_sys_TestFileArchive {
512  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
513  typeset ExistFlag
514  #Command depends on targeted file system
515  #We want it quiet
516  DEBUG_sys=false IGCM_sys_IsFileArchived $1
517  if [ $? = 0 ] ; then
518    ExistFlag=$( IGCM_sys_RshArchive "[ -f $1 ] && echo 0 || echo 1" )
519  else
520    ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
521  fi
522  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
523
524  return ${ExistFlag}
525}
526
527#D-#==================================================
528#D-function IGCM_sys_TestFileBuffer
529#D-* Purpose: Test file that must NOT EXISTS on Buffer
530#D-* Examples:
531#D-
532function IGCM_sys_TestFileBuffer {
533  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
534  typeset ExistFlag
535  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
536  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
537
538  return ${ExistFlag}
539}
540
541#D-#==================================================
542#D-function IGCM_sys_CountFileArchive
543#D-* Purpose: Count files on Archive filesystem
544#D-* Examples:
545#D-
546function IGCM_sys_CountFileArchive {
547  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
548  #Command depends on targeted file system
549  DEBUG_sys=false IGCM_sys_IsFileArchived $1
550  if [ $? = 0 ] ; then
551    IGCM_sys_RshArchive "ls ${@} 2>/dev/null | wc -l"
552    status=$?
553  else
554    ls ${@} 2>/dev/null | wc -l
555    status=$?
556  fi
557  if [ ${status} -gt 0 ] ; then
558    echo "IGCM_sys_CountFileArchive : erreur."
559  fi
560  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
561}
562
563#D-#==================================================
564#D-function IGCM_sys_CountFileBuffer
565#D-* Purpose: Count files on Scratch filesystem
566#D-* Examples:
567#D-
568function IGCM_sys_CountFileBuffer {
569  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
570  ls ${@} 2>/dev/null | wc -l
571  if [ $? -gt 0 ] ; then
572    echo "IGCM_sys_CountFileBuffer : erreur."
573  fi
574  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
575}
576
577#D-#==================================================
578#D-function IGCM_sys_Tree
579#D-* Purpose: Tree directories with files on ${ARCHIVE}
580#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
581#D-
582function IGCM_sys_Tree {
583  IGCM_debug_PushStack "IGCM_sys_Tree" $@
584  if ( $DEBUG_sys ) ; then
585    echo "IGCM_sys_Tree :" $@
586  fi
587
588  \mfls -r $@
589
590  IGCM_debug_PopStack "IGCM_sys_Tree"
591}
592
593#D-#==================================================
594#D-function IGCM_sys_Tar
595#D-* Purpose: master tar command
596#D-* Examples:
597#D-
598function IGCM_sys_Tar {
599  IGCM_debug_PushStack "IGCM_sys_Tar" $@
600  if ( $DEBUG_sys ) ; then
601    echo "IGCM_sys_Tar :" $@
602  fi
603  \tar cf $@
604  if [ $? -gt 0 ] ; then
605    echo "IGCM_sys_Tar : erreur."
606    IGCM_debug_Exit "IGCM_sys_Tar"
607  fi
608  IGCM_debug_PopStack "IGCM_sys_Tar"
609}
610
611#D-#==================================================
612#D-function IGCM_sys_UnTar
613#D-* Purpose: master un-tar command
614#D-* Examples:
615#D-
616function IGCM_sys_UnTar {
617  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
618  if ( $DEBUG_sys ) ; then
619    echo "IGCM_sys_UnTar :" $@
620  fi
621  \tar xvf $1
622  if [ $? -gt 0 ] ; then
623    echo "IGCM_sys_UnTar : erreur."
624    IGCM_debug_Exit "IGCM_sys_UnTar"
625  fi
626  IGCM_debug_PopStack "IGCM_sys_UnTar"
627}
628
629#D-#==================================================
630#D-function IGCM_sys_Qsub
631#D-* Purpose: Qsub new job
632#D-* Examples:
633#D-
634function IGCM_sys_Qsub {
635  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
636
637  typeset ERROR
638  ERROR=0
639
640  if ( $DEBUG_sys ) ; then
641    echo "IGCM_sys_Qsub :" $@
642  fi
643  # We have to change output/error file
644  [ ${#@} = 1 ] &&  REP_FOR_JOB=${SUBMIT_DIR}
645  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
646  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
647      -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
648      $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
649  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM ; ERROR=$? ; cd - ;
650
651  if [ ${ERROR} -gt 0 ] ; then
652    echo "IGCM_sys_Qsub : erreur $@"
653    IGCM_debug_Exit "IGCM_sys_Qsub"
654  else
655    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
656  fi
657  IGCM_debug_PopStack "IGCM_sys_Qsub"
658}
659
660#D-#==================================================
661#D-function IGCM_sys_QsubPost
662#D-* Purpose: Qsub new job on scalaire
663#D-* Examples:
664#D-
665function IGCM_sys_QsubPost {
666  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
667
668  typeset ERROR
669  ERROR=0
670
671  if ( $DEBUG_sys ) ; then
672    echo "IGCM_sys_QsubPost :" $@
673  fi
674  # We have to change output/error file
675  [ ${#@} = 1 ] &&  REP_FOR_JOB=${POST_DIR}
676  [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
677
678  sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
679      -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
680      ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM
681
682  cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM ; ERROR=$? ; cd - ;
683  if [ ${ERROR} -gt 0 ] ; then
684    echo "IGCM_sys_QsubPost : erreur $@."
685    IGCM_debug_Exit "IGCM_sys_QsubPost"
686  else
687    IGCM_sys_Rm ${REP_FOR_JOB}/JOB_FOR_IGCM
688  fi
689
690  IGCM_debug_PopStack "IGCM_sys_QsubPost"
691}
692
693#D-*************************
694#D- File transfer functions
695#D-*************************
696#D-
697
698#D-#==================================================
699#D-function IGCM_sys_Rsync_out
700#D-* Purpose: treat return val of rsync
701#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
702#D-  Error values and explanations can depend on your system version.
703function IGCM_sys_Rsync_out {
704  status=$1
705  if [ ! $status ] ; then
706    echo "rsync error !"
707  fi
708
709  if [ $MYLANG = "fr" ]; then
710    case $status in
711    0)  return ;;
712    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
713      echo "Erreur de syntaxe ou d'utilisation."
714      return;;
715    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
716      echo "Incompatibilité de protocole."
717      return;;
718    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
719      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
720      echo "répertoires"
721      return;;
722    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
723      echo "Action demandée non supportée : une tentative de manipulation de"
724      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
725      echo "été faite ; ou une option qui est supportée par le  client  mais"
726      echo "pas par le serveur a été spécifiée."
727      return;;
728    10) echo "Erreur de rsync ; RERR_SOCKETIO"
729      echo "Erreur dans le socket d'entrée sortie"
730      return;;
731    11) echo "Erreur de rsync ; RERR_FILEIO"
732      echo "Erreur d'entrée sortie fichier"
733      return;;
734    12) echo "Erreur de rsync ; RERR_STREAMIO"
735      echo "Erreur dans flux de donnée du protocole rsync"
736      return;;
737    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
738      echo "Erreur avec les diagnostics du programme"
739      return;;
740    14) echo "Erreur de rsync ; RERR_IPC"
741      echo "Erreur dans le code IPC"
742      return;;
743    20) echo "Erreur de rsync ; RERR_SIGNAL"
744      echo "SIGUSR1 ou SIGINT reçu"
745      return;;
746    21) echo "Erreur de rsync ; RERR_WAITCHILD"
747      echo "Une erreur retournée par waitpid()"
748      return;;
749    22) echo "Erreur de rsync ; RERR_MALLOC"
750      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
751      return;;
752    23) echo ""
753      echo "Erreur fichier inexistant"
754      return;;
755    30) echo "Erreur de rsync ; RERR_TIMEOUT"
756      echo "Temps d'attente écoulé dans l'envoi/réception de données"
757      return;;
758    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
759      return;;
760    esac
761  elif [ $MYLANG = "en" ] ; then
762    case $status in
763    0)  return;;
764    1)  echo "rsync error : Syntax or usage error "
765      return;;
766    2)  echo "rsync error : Protocol incompatibility "
767      return;;
768    3)  echo "rsync error : Errors selecting input/output files, dirs"
769      return;;
770    4)  echo "rsync error : Requested action not supported: an attempt"
771      echo "was made to manipulate 64-bit files on a platform that cannot support"
772      echo "them; or an option was specified that is supported by the client and"
773      echo "not by the server."
774      return;;
775    5)  echo "rsync error : Error starting client-server protocol"
776      return;;
777    10) echo "rsync error : Error in socket I/O "
778      return;;
779    11) echo "rsync error : Error in file I/O "
780      return;;
781    12) echo "rsync error : Error in rsync protocol data stream "
782      return;;
783    13) echo "rsync error : Errors with program diagnostics "
784      return;;
785    14) echo "rsync error : Error in IPC code "
786      return;;
787    20) echo "rsync error : Received SIGUSR1 or SIGINT "
788      return;;
789    21) echo "rsync error : Some error returned by waitpid() "
790      return;;
791    22) echo "rsync error : Error allocating core memory buffers "
792      return;;
793    23) echo "rsync error : Partial transfer due to error"
794      return;;
795    24) echo "rsync error : Partial transfer due to vanished source files"
796      return;;
797    30) echo "rsync error : Timeout in data send/receive "
798      return;;
799    *)  echo "rsync error : return code of rsync unknown :" $status
800      return;;
801    esac
802  else
803    echo "unknown language $MYLANG."
804    return
805  fi
806}
807
808#D-#==================================================
809#D-function IGCM_sys_Miror_libIGCM
810#D-* Purpose: Mirror libIGCM PATH and lib to ulam
811#D-* Examples:
812#D-
813function IGCM_sys_Mirror_libIGCM {
814  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
815  if ( $DEBUG_sys ) ; then
816    echo "IGCM_sys_Mirror_libIGCM"
817  fi
818
819  typeset status
820
821  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
822
823  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
824  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
825  status=$?
826
827  if [ ${status} -gt 0 ] ; then
828    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
829    cat out_rsync
830  fi
831  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
832}
833
834#D-#==================================================
835#D-function IGCM_sys_Cp
836#D-* Purpose: generic cp
837#D-* Examples:
838#D-
839function IGCM_sys_Cp {
840  IGCM_debug_PushStack "IGCM_sys_Cp" $@
841  if ( $DEBUG_sys ) ; then
842    echo "IGCM_sys_Cp :" $@
843  fi
844
845  typeset status
846
847  echo cp $@ > out_rsync 2>&1
848  \cp $@ >> out_rsync 2>&1
849  status=$?
850
851  if [ ${status} -gt 0 ] ; then
852    echo "IGCM_sys_Cp : error code ${status}"
853    cat out_rsync
854    IGCM_debug_Exit "IGCM_sys_Cp"
855  else
856    \rm out_rsync
857  fi
858  IGCM_debug_PopStack "IGCM_sys_Cp"
859}
860
861#D-#==================================================
862#D-function IGCM_sys_Rm
863#D-* Purpose: generic rm
864#D-* Examples:
865#D-
866function IGCM_sys_Rm {
867  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
868  if ( $DEBUG_sys ) ; then
869    echo "IGCM_sys_Rm :" $@
870  fi
871
872  typeset status
873
874  echo rm $@ > out_rsync 2>&1
875  \rm $@ >> out_rsync 2>&1
876  status=$?
877
878  if [ ${status} -gt 0 ] ; then
879    echo "IGCM_sys_Rm : error code ${status}"
880    cat out_rsync
881    IGCM_debug_Exit "IGCM_sys_Rm"
882  else
883    \rm out_rsync
884  fi
885  IGCM_debug_PopStack "IGCM_sys_Rm"
886}
887
888#D-#==================================================
889#D-function IGCM_sys_RmRunDir
890#D-* Purpose: rm tmpdir (dummy function most of the time batch
891#D-                      scheduler will do the job)
892#D-* Examples:
893#D-
894function IGCM_sys_RmRunDir {
895  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
896  if ( $DEBUG_sys ) ; then
897    echo "IGCM_sys_RmRunDir :" $@
898    echo "Dummy call, let the scheduler do that."
899  fi
900  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
901}
902
903#D-#==================================================
904#D-function IGCM_sys_Mv
905#D-* Purpose: generic move
906#D-* Examples:
907#D-
908function IGCM_sys_Mv {
909  IGCM_debug_PushStack "IGCM_sys_Mv" $@
910  if ( $DEBUG_sys ) ; then
911    echo "IGCM_sys_Mv :" $@
912  fi
913
914  if [ $DRYRUN = 0 ]; then
915
916    typeset status
917
918    echo mv $@ > out_rsync 2>&1
919    \mv $@ >> out_rsync 2>&1
920    status=$?
921
922    if [ ${status} -gt 0 ] ; then
923      echo "IGCM_sys_Mv : error code ${status}"
924      cat out_rsync
925      IGCM_debug_Exit "IGCM_sys_Mv"
926    else
927      \rm out_rsync
928    fi
929  else
930    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
931  fi
932
933  IGCM_debug_PopStack "IGCM_sys_Mv"
934}
935
936#D-#==================================================
937#D-function IGCM_sys_Put_Dir
938#D-* Purpose: Copy a complete directory on $(ARCHIVE)
939#D-* Examples:
940#D-
941function IGCM_sys_Put_Dir {
942  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
943  if ( $DEBUG_sys ) ; then
944    echo "IGCM_sys_Put_Dir :" $@
945  fi
946  if [ $DRYRUN = 0 ]; then
947    if [ ! -d ${1} ] ; then
948      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
949      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
950      return
951    fi
952
953    typeset status
954
955    # Only if we use rsync
956    #IGCM_sys_TestDirArchive $( dirname $2 )
957    #
958    #Command depends on targeted file system
959    DEBUG_sys=false IGCM_sys_IsFileArchived $2
960    if [ $? = 0 ] ; then
961      \rcp -r $1 ${STOREHOST}:$2 > out_rsync 2>&1
962      status=$?
963    else
964      \cp -r $1 $2 > out_rsync 2>&1
965      status=$?
966    fi
967
968    if [ ${status} -gt 0 ] ; then
969      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : rcp or cp failed error code ${status}"
970      cat out_rsync
971      IGCM_debug_Exit "IGCM_sys_Put_Dir"
972    else
973      \rm out_rsync
974    fi
975  else
976    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
977  fi
978  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
979}
980
981#D-#==================================================
982#D-function IGCM_sys_Get_Dir
983#D-* Purpose: Copy a complete directory from ${ARCHIVE}
984#D-* Examples:
985#D-
986function IGCM_sys_Get_Dir {
987  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
988  if ( $DEBUG_sys ) ; then
989    echo "IGCM_sys_Get_Dir :" $@
990  fi
991  if [ $DRYRUN = 0 ]; then
992#    if [ ! -d ${1} ] ; then
993#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
994#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
995#      return
996#    fi
997
998    typeset NB_ESSAI DELAI status i
999    # number of tentative
1000    NB_ESSAI=3
1001    # time delay between tentative
1002    DELAI=2
1003
1004    # Only if we use rsync
1005    #IGCM_sys_TestDirArchive $( dirname $2 )
1006    #
1007    # Command depends on targeted filesystem
1008    DEBUG_sys=false IGCM_sys_IsFileArchived $1
1009    if [ $? = 0 ] ; then
1010      # add dmget (to demigrate all offline files) to reduce time of this command :
1011      IGCM_sys_RshArchive "dmfind $1 -state MIG -o -state OFL -o -state PAR | dmget -q -n"
1012      i=0
1013      while [ $i -lt $NB_ESSAI ] ; do
1014        \rcp -rp ${STOREHOST}:$1 $2 > out_rsync 2>&1
1015        status=$?
1016        if [ ${status} -gt 0 ]; then
1017          IGCM_debug_Print 2 "IGCM_sys_Get_Dir : rcp failed error code ${status} ${i}/${NB_ESSAI}"
1018          IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1019          sleep $DELAI
1020        else
1021          break
1022        fi
1023        (( i = i + 1 ))
1024      done
1025    else
1026      \cp -rp $1 $2 > out_rsync 2>&1
1027      status=$?
1028      if [ ${status} -gt 0 ] ; then
1029        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
1030        cat out_rsync
1031        IGCM_debug_Exit "IGCM_sys_Get_Dir"
1032      else
1033        \rm out_rsync
1034      fi
1035    fi
1036    if [ ${status} -gt 0 ] ; then
1037      echo "IGCM_sys_Get_Dir : error."
1038      cat out_rsync
1039      IGCM_debug_Exit "IGCM_sys_Get_Dir"
1040    else
1041      \rm out_rsync
1042    fi
1043  else
1044    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1045  fi
1046  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1047}
1048
1049#D-#==================================================
1050#D-function IGCM_sys_Get_Master
1051#D-* Purpose: Copy a complete directory from MASTER filesystem
1052#D-* Examples:
1053#D-
1054function IGCM_sys_Get_Master {
1055  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1056  if ( $DEBUG_sys ) ; then
1057    echo "IGCM_sys_Get_Master :" $@
1058  fi
1059  if [ $DRYRUN = 0 ]; then
1060    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1061      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1062      IGCM_debug_PopStack "IGCM_sys_Get_Master"
1063      return
1064    fi
1065
1066    typeset NB_ESSAI DELAI status i
1067    # number of tentative
1068    NB_ESSAI=3
1069    # time delay between tentative
1070    DELAI=2
1071
1072    i=0
1073    while [ $i -lt $NB_ESSAI ] ; do
1074      \cp -urL $1 $2 > out_rsync 2>&1
1075      status=$?
1076      if [ ${status} -gt 0 ]; then
1077        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
1078        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1079        sleep $DELAI
1080      else
1081        break
1082      fi
1083      (( i = i + 1 ))
1084    done
1085
1086    if [ ${status} -gt 0 ] ; then
1087      echo "IGCM_sys_Get_Master : error."
1088      cat out_rsync
1089      IGCM_debug_Exit "IGCM_sys_Get_Master"
1090    else
1091      \rm out_rsync
1092    fi
1093  else
1094    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1095  fi
1096  IGCM_debug_PopStack "IGCM_sys_Get_Master"
1097}
1098
1099#====================================================
1100#- Call IGCM_sys_Mirror_libIGCM now !
1101if ( $MirrorlibIGCM ) ; then
1102  IGCM_sys_Mirror_libIGCM
1103fi
1104
1105#D-#==================================================
1106#D-function IGCM_sys_Put_Rest
1107#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1108#D-           File and target directory must exist.
1109#D-* Examples:
1110#D-
1111function IGCM_sys_Put_Rest {
1112  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1113  if ( $DEBUG_sys ) ; then
1114    echo "IGCM_sys_Put_Rest :" $@
1115  fi
1116  if [ $DRYRUN = 0 ]; then
1117    if [ ! -f ${1} ] ; then
1118      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1119      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1120    fi
1121
1122    typeset status
1123    #
1124    if [ X${JobType} = XRUN ] ; then
1125      IGCM_sys_Chmod 444 ${1}
1126    fi
1127    #
1128    #
1129    #Command depends on targeted file system
1130    DEBUG_sys=false IGCM_sys_IsFileArchived $2
1131    if [ $? = 0 ] ; then
1132      mfput $1 $2 > out_rsync 2>&1
1133      status=$?
1134    else
1135      IGCM_sys_MkdirArchive $( dirname $2 )
1136      \cp $1 $2 > out_rsync 2>&1
1137      status=$?
1138    fi
1139
1140#       #RSYNC WITH NETWORK RSH CALL
1141#       IGCM_sys_MkdirArchive $( dirname $2 )
1142#       echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1143#       ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1144
1145#       #RSYNC WITH NFS USE
1146#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1147#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1148
1149#       status=$?
1150#       IGCM_sys_Rsync_out $status
1151
1152#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1153#       (( status=status+$? ))
1154
1155    if [ ${status} -gt 0 ] ; then
1156      echo "IGCM_sys_Put_Rest : mfput or cp failed error code ${status}"
1157      cat out_rsync
1158      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1159    else
1160      \rm out_rsync
1161    fi
1162  else
1163    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1164  fi
1165  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1166}
1167
1168#D-#==================================================
1169#D-function IGCM_sys_PutBuffer_Rest
1170#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1171#D-           File and target directory must exist.
1172#D-* Examples:
1173#D-
1174function IGCM_sys_PutBuffer_Rest {
1175  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1176  if ( $DEBUG_sys ) ; then
1177    echo "IGCM_sys_PutBuffer_Rest :" $@
1178  fi
1179  if [ $DRYRUN = 0 ]; then
1180    if [ ! -f ${1} ] ; then
1181      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1182      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1183    fi
1184
1185    typeset status
1186    #
1187    if [ X${JobType} = XRUN ] ; then
1188      IGCM_sys_Chmod 444 ${1}
1189    fi
1190
1191    #
1192    # USUAL WAY
1193    \cp $1 $2 > out_rsync 2>&1
1194    status=$?
1195
1196    if [ ${status} -gt 0 ] ; then
1197      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1198      [ -f ${2} ] && ls -l ${2}
1199      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1200      cat out_rsync
1201      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1202    else
1203      \rm out_rsync
1204    fi
1205  else
1206    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1207  fi
1208  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1209}
1210
1211#D-#==================================================
1212#D-function IGCM_sys_PrepareTaredRestart
1213#D-* Purpose: Prepare tared restart to be access by computing job.
1214#D-* Examples:
1215#D-
1216function IGCM_sys_PrepareTaredRestart {
1217  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1218  if [ $DRYRUN = 0 ]; then
1219    [ ! -f $( basename $1 ) ] && DEBUG_sys=false IGCM_sys_Get $1 .
1220  else
1221    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1222  fi
1223  echo $( basename $1 )
1224
1225  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1226}
1227
1228#D-#==================================================
1229#D-function IGCM_sys_Put_Out
1230#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1231#D-* Examples:
1232#D-
1233function IGCM_sys_Put_Out {
1234  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1235  if ( $DEBUG_sys ) ; then
1236    echo "IGCM_sys_Put_Out :" $@
1237  fi
1238
1239  typeset status
1240
1241  if [ $DRYRUN = 0 ]; then
1242    if [ ! -f ${1} ] ; then
1243      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1244      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1245      return 1
1246    fi
1247
1248    #
1249    if [ X${JobType} = XRUN ] ; then
1250      if [ X${3} = X ] ; then
1251        IGCM_sys_Chmod 444 ${1}
1252      fi
1253    fi
1254    #
1255    #
1256    #Command depends on targeted file system
1257    DEBUG_sys=false IGCM_sys_IsFileArchived $2
1258    if [ $? = 0 ] ; then
1259      mfput $1 $2 > out_rsync 2>&1
1260      status=$?
1261    else
1262      IGCM_sys_MkdirArchive $( dirname $2 )
1263      \cp $1 $2 > out_rsync 2>&1
1264      status=$?
1265    fi
1266
1267#       #RSYNC WITH NETWORK RSH CALL
1268#       IGCM_sys_MkdirArchive $( dirname $2 )
1269#       echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1270#       ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1271
1272#       #RSYNC WITH NFS USE
1273#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1274#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1275
1276#       status=$?
1277#       IGCM_sys_Rsync_out $status
1278
1279#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1280#       (( status=status+$? ))
1281
1282    if [ ${status} -gt 0 ] ; then
1283      IGCM_debug_Print 2 "IGCM_sys_Put_Out : mfput or cp failed error code ${status}"
1284      cat out_rsync
1285      IGCM_debug_Exit "IGCM_sys_Put_Out"
1286    else
1287      \rm out_rsync
1288    fi
1289  else
1290    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1291  fi
1292  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1293  return 0
1294}
1295
1296#D-#==================================================
1297#D-function IGCM_sys_PutBuffer_Out
1298#D-* Purpose: Copy a file on ${WORKDIR} after having chmod it in readonly
1299#D-* Examples:
1300#D-
1301function IGCM_sys_PutBuffer_Out {
1302  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1303  if ( $DEBUG_sys ) ; then
1304    echo "IGCM_sys_PutBuffer_Out :" $@
1305  fi
1306
1307  typeset NB_ESSAI DELAI status i exist skip
1308
1309  # number of tentative
1310  NB_ESSAI=3
1311  # time delay between tentative
1312  DELAI=2
1313
1314  if [ $DRYRUN = 0 ]; then
1315    if [ ! -f ${1} ] ; then
1316      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1317      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1318      return 1
1319    fi
1320    #
1321    IGCM_sys_Mkdir $( dirname $2 )
1322    #
1323
1324    exist=false
1325    skip=false
1326    if [ -f $2 ] ; then
1327      IGCM_debug_Print 1 "$2 already exist"
1328      exist=true
1329      if [ "X$( diff $1 $2 )" = X ] ; then
1330        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1331        status=0
1332        skip=true
1333      else
1334        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1335        skip=false
1336      fi
1337    fi
1338    #
1339    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1340      IGCM_sys_Chmod u+w $2
1341    fi
1342
1343    if [ X${skip} = Xfalse ] ; then
1344      i=0
1345      while [ $i -lt $NB_ESSAI ] ; do
1346        # USUAL WAY
1347        \cp $1 $2 > out_rsync 2>&1
1348        status=$?
1349        if [ ${status} -gt 0 ]; then
1350          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1351          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1352          [ -f ${2} ] && ls -l ${2}
1353          [ -f ${2}/${1} ] && ls -l ${2}/${1}
1354          sleep $DELAI
1355        else
1356          break
1357        fi
1358        (( i = i + 1 ))
1359      done
1360    fi
1361
1362    if [ ${status} -gt 0 ] ; then
1363      echo "IGCM_sys_PutBuffer_Out : error."
1364      [ -f ${2} ] && ls -l ${2}
1365      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1366      cat out_rsync
1367      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1368    else
1369
1370      if [ X${JobType} = XRUN ] ; then
1371        if [ X${3} = X ] ; then
1372          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1373          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1374        fi
1375      fi
1376
1377      \rm out_rsync
1378    fi
1379  else
1380    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1381  fi
1382  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1383  return 0
1384}
1385
1386#D-#==================================================
1387#D-function IGCM_sys_Get
1388#D-* Purpose: Get a file from ${ARCHIVE}
1389#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1390#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1391function IGCM_sys_Get {
1392  IGCM_debug_PushStack "IGCM_sys_Get" $@
1393
1394  typeset DEST status dm_liste
1395
1396  if ( $DEBUG_sys ) ; then
1397    echo "IGCM_sys_Get :" $@
1398  fi
1399  if [ $DRYRUN -le 2 ]; then
1400    if [ X${1} = X'/l' ] ; then
1401      # test if the first file is present in the old computation :
1402      eval set +A dm_liste \${${2}}
1403    else
1404      dm_liste=${1}
1405    fi
1406    eval DEST=\${${#}}
1407
1408    # test if the (first) file is present in the old computation :
1409    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
1410    if [ $? = 0 ] ; then
1411      IGCM_sys_TestFileArchive ${dm_liste[0]}
1412      status=$?
1413    else
1414      IGCM_sys_TestFileBuffer ${dm_liste[0]}
1415      status=$?
1416    fi
1417
1418    if [ ${status} -gt 0 ] ; then
1419      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1420      IGCM_debug_Exit "IGCM_sys_Get"
1421      return
1422    fi
1423
1424    #Command depends on targeted file system
1425    DEBUG_sys=false IGCM_sys_IsFileArchived ${dm_liste[0]}
1426    if [ $? = 0 ] ; then
1427      mfget ${dm_liste[*]} ${DEST} > out_rsync 2>&1
1428      status=$?
1429    else
1430      \cp ${dm_liste[*]} ${DEST} > out_rsync 2>&1
1431      status=$?
1432    fi
1433
1434#       #RSYNC WITH NETWORK RSH CALL
1435#       echo ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1436#       ${RSYNC} ${RSYNC_opt} --rsync-path=${REMOTE_RSYNC} -e rsh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1437
1438#       #RSYNC WITH NFS USE
1439#       echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1440#       ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1441
1442#       status=$?
1443#       IGCM_sys_Rsync_out $status
1444
1445#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1446#       (( status=status+$? ))
1447
1448    if [ ${status} -gt 0 ] ; then
1449      IGCM_debug_Print 2 "IGCM_sys_Get : mfget or cp failed error code ${status}"
1450      cat out_rsync
1451      IGCM_debug_Exit "IGCM_sys_Get"
1452    else
1453      \rm out_rsync
1454    fi
1455  else
1456    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1457  fi
1458  IGCM_debug_PopStack "IGCM_sys_Get"
1459}
1460
1461#D-#==================================================
1462#D-function IGCM_sys_GetBuffer
1463#D-* Purpose: Get a file from ${SCRATCHDIR}
1464#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1465#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1466function IGCM_sys_GetBuffer {
1467  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1468
1469  typeset DEST buf_liste target file_work
1470  typeset NB_ESSAI DELAI status i
1471
1472  if ( $DEBUG_sys ) ; then
1473    echo "IGCM_sys_GetBuffer :" $@
1474  fi
1475
1476  # number of tentative
1477  NB_ESSAI=3
1478  # time delay between tentative
1479  DELAI=2
1480
1481  if [ $DRYRUN -le 2 ]; then
1482    if [ X${1} = X'/l' ] ; then
1483      # test if the first file is present in the old computation :
1484      eval set +A buf_liste \${${2}}
1485    else
1486      eval set +A buf_liste ${1}
1487    fi
1488    eval DEST=\${${#}}
1489
1490    #USUAL WAY
1491    if [ X${1} = X'/l' ] ; then
1492      for target in ${buf_liste[*]} ; do
1493        local_file=$( basename ${target} )
1494        i=0
1495        while [ $i -lt $NB_ESSAI ] ; do
1496          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1497          status=$?
1498          if [ ${status} -gt 0 ]; then
1499            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1500            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1501            sleep $DELAI
1502          else
1503            break
1504          fi
1505          (( i = i + 1 ))
1506        done
1507        if [ ${status} -gt 0 ] ; then
1508          echo "IGCM_sys_Get : error"
1509          cat out_rsync
1510          \rm out_rsync
1511          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1512        else
1513          \rm out_rsync
1514        fi
1515      done
1516    else
1517      i=0
1518      while [ $i -lt $NB_ESSAI ] ; do
1519        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1520        status=$?
1521        if [ ${status} -gt 0 ]; then
1522          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1523          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1524          sleep $DELAI
1525        else
1526          break
1527        fi
1528        (( i = i + 1 ))
1529      done
1530      if [ ${status} -gt 0 ] ; then
1531        echo "IGCM_sys_Get : error"
1532        cat out_rsync
1533        \rm out_rsync
1534        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1535      else
1536        \rm out_rsync
1537      fi
1538    fi
1539  else
1540    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1541  fi
1542  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1543}
1544
1545#D-#==================================================
1546#D-function IGCM_sys_GetDate_FichWork
1547#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1548#D-* Examples:
1549#D-
1550function IGCM_sys_GetDate_FichWork {
1551  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1552  if ( $DEBUG_sys ) ; then
1553    echo "IGCM_sys_GetDate_FichWork :" $@
1554  fi
1555  typeset dateF
1556  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1557  eval ${2}=${dateF[5]}
1558
1559  # donne la date filesys d'un fichier sur la machine work
1560  IGCM_debug_PopStack "IGCM_sys_FichWork"
1561}
1562
1563#D-#==================================================
1564#D-function IGCM_sys_GetDate_FichArchive
1565#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1566#D-* Examples:
1567#D-
1568function IGCM_sys_GetDate_FichArchive {
1569  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1570  if ( $DEBUG_sys ) ; then
1571    echo "IGCM_sys_GetDate_FichArchive :" $@
1572  fi
1573  typeset dateF
1574  set +A dateF -- $( rsh ${STOREHOST} -n ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1575  eval ${2}=${dateF[5]}
1576
1577  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1578}
1579
1580#D-#==================================================
1581#D-function IGCM_sys_Put_Dods
1582#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1583#D-* Examples:
1584#D-
1585function IGCM_sys_Put_Dods {
1586  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1587  if ( $DEBUG_sys ) ; then
1588    echo "IGCM_sys_Put_Dods :" $@
1589  fi
1590  if [ $DRYRUN = 0 ]; then
1591    # We take our time on that
1592    sleep 10
1593    IGCM_sys_TestDirArchive ${R_SAVE}/${1}
1594    if [ $? != 0 ] ; then
1595      echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
1596      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1597      return
1598    fi
1599
1600    typeset status
1601    #
1602    rsh gaya exec /bin/ksh <<EOF
1603          cd ${R_SAVE}
1604          /usr/local/bin/dods_rm DODS/pub/${LOGIN}/${R_DODS}/${1} > /dev/null 2>&1
1605          /bin/chmod -R u+w ${R_SAVE}/${1}
1606          /usr/local/bin/dods_cp ${1} DODS/pub/${LOGIN}/${R_DODS} > /dev/null 2>&1
1607          /bin/chmod -R +rX ${R_SAVE}/${1}
1608          /bin/chmod -R u+w ${R_SAVE}/${1}
1609EOF
1610    status=$?
1611
1612    if [ ${status} -gt 0 ] ; then
1613      echo "IGCM_sys_Put_Dods : error."
1614      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1615    fi
1616  else
1617    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1618  fi
1619  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1620}
1621
1622##############################################################
1623# REBUILD OPERATOR
1624
1625function IGCM_sys_rebuild {
1626  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1627  if ( $DEBUG_sys ) ; then
1628    echo "IGCM_sys_rebuild :" $@
1629  fi
1630
1631  typeset NB_ESSAI DELAI status i firstArg
1632  # number of tentative
1633  NB_ESSAI=3
1634  # time delay between tentative
1635  DELAI=2
1636
1637  i=0
1638  while [ $i -lt $NB_ESSAI ] ; do
1639    /smphome/rech/psl/rpsl035/bin/rebuild -f -o $@ > out_rsync 2>&1
1640    status=$?
1641    if [ ${status} -gt 0 ] ; then
1642      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1643      cat out_rsync
1644      \rm out_rsync
1645      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1646      firstArg=${1}
1647      \rm ${firstArg}
1648      sleep $DELAI
1649    else
1650      \rm out_rsync
1651      break
1652    fi
1653    (( i = i + 1 ))
1654  done
1655
1656  if [ ${status} -gt 0 ] ; then
1657    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1658    IGCM_debug_Exit "rebuild"
1659  fi
1660
1661  IGCM_debug_PopStack "IGCM_sys_rebuild"
1662}
1663
1664function IGCM_sys_rebuild_station {
1665  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1666  typeset i list_opt file_in file_out prefix_invert list_invert
1667  if ( $DEBUG_sys ) ; then
1668    echo "IGCM_sys_rebuild_station :" $@
1669  fi
1670  list_opt=$@
1671
1672  # Invert Axis : t,x -> x,t
1673  #               t,pres,x -> x,t,pres
1674  # So that we can concatenate along x
1675  i=0
1676  for file_in in ${list_opt} ; do
1677    (( i = i + 1))
1678    [ ${i} = 1 ] && file_out=${file_in} && continue
1679    prefix_invert=$( basename ${file_in} .nc )
1680    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1681    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1682  done
1683
1684  # Concatenate
1685  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1686
1687  # Re-ivert file
1688  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1689
1690  # Station re-ordering is too expansive to be run within libICGM
1691  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1692  # This re-ordering must be done "in memory" by the cmorization process
1693  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1694  # BEGIN reordering
1695
1696  # Only LMDZ text output contains the exact ordering of the station.
1697  # We isolate this in the code below:
1698  #  0  38  -157.5000000000000  70.98591549295774
1699  #  0  54  27.49999999999999   67.18309859154928
1700  #  0  56  -62.50000000000001  82.39436619718309
1701  #  0  79  12.49999999999999   78.59154929577466
1702  #  0  116 -165.0000000000000  76.05633802816901
1703  #  0  117 130.0000000000000   70.98591549295774
1704  #  0  118 110.0000000000000   87.46478873239437
1705  #  1  40  4.999999999999995   51.97183098591550
1706#  typeset iStation iProc list_opt file_in file_out prefix_invert
1707#  typeset -Z4 j4
1708#  typeset -Z3 j3
1709
1710#  unset list_opt
1711#  set +A list_opt $@
1712
1713  # Filename after rebuild
1714#  file_out=${list_opt[0]}
1715  # Prefix of output files
1716#  prefix_invert=$( basename ${file_out} .nc )
1717  # Number of procs
1718#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1719
1720#  iProc=0
1721#  while [ ${iProc} -lt ${num_proc} ] ; do
1722    # Array containing Station as a number
1723#    unset proc_stn
1724#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | awk ' {print $2}' )
1725    # Number of stations produced by processor proc
1726#    stationLast=${#proc_stn[*]}
1727    # Proc number on 4 digits
1728#    j4=${iProc}
1729    # Init
1730#    iStation=0
1731#    while [ ${iStation} -lt ${stationLast} ] ; do
1732      # Station number on 3 digits
1733#      j3=${proc_stn[${iStation}]}
1734      # Extract station
1735      # Invert Axis : t,x -> x,t
1736      #               t,pres,x -> x,t,pres
1737      # So that we can concatenate along x
1738#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1739#      (( iStation = iStation + 1 ))
1740#    done
1741#    (( iProc = iProc + 1 ))
1742#  done
1743
1744  # Concatenate all station along x
1745#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1746
1747  # Re-invert file
1748#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1749
1750  # END reordering
1751
1752  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1753}
1754
1755############################################################
1756# Activate Running Environnment Variables
1757
1758function IGCM_sys_activ_variables {
1759  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1760  if ( $DEBUG_sys ) ; then
1761    echo "IGCM_sys_activ_variables "
1762  fi
1763
1764  ulimit -s unlimited
1765
1766## to be done only one time
1767## export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
1768  echo ${LD_LIBRARY_PATH} | grep -i netcdf >/dev/null 2>&1 || export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/smplocal/pub/NetCDF/4.1.3/lib:/smplocal/pub/HDF5/1.8.9/seq/lib
1769  echo LD_LIBRARY_PATH = ${LD_LIBRARY_PATH}
1770
1771  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1772}
1773
1774############################################################
1775# Desactivate Running Environnment Variables
1776
1777function IGCM_sys_desactiv_variables {
1778  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1779  if ( $DEBUG_sys ) ; then
1780    echo "IGCM_sys_desactiv_variables "
1781  fi
1782# --------------------------------------------------------------------
1783#D- MPI specifications
1784# --------------------------------------------------------------------
1785
1786# --------------------------------------------------------------------
1787#D- Other specifications
1788# --------------------------------------------------------------------
1789
1790  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1791}
1792
1793############################################################
1794# Build MPI/OMP scripts run file (dummy function)
1795
1796function IGCM_sys_build_run_file {
1797
1798IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1799
1800}
1801
1802############################################################
1803# Build MPI/OMP scripts
1804function IGCM_sys_build_execution_scripts
1805{
1806  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1807  if ( $DEBUG_sys ) ; then
1808    echo "IGCM_sys_build_execution_scripts " $@
1809  fi
1810
1811  EXECUTION=${HOST_MPIRUN_COMMAND}
1812
1813  if ( ${OK_PARA_MPMD} ) ; then
1814
1815    if [ -f run_file ] ; then
1816      IGCM_sys_Rm -f run_file
1817    fi
1818    touch run_file
1819
1820# run_file construction
1821
1822# Then first loop on the components for the coupler ie oasis
1823
1824### the coupler ie oasis must be the first one
1825    for comp in ${config_ListOfComponents[*]} ; do
1826
1827      eval ExeNameIn=\${config_Executable_${comp}[0]}
1828      eval ExeNameOut=\${config_Executable_${comp}[1]}
1829
1830      # for CPL component only
1831      if [ "X${comp}" = "XCPL" ] ; then
1832
1833        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1834        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1835
1836        if ( ${OK_PARA_MPI} ) ; then
1837          (( mpi_count = 1 ))
1838          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1839            echo "./${ExeNameOut}" >> run_file
1840            (( mpi_count = mpi_count + 1 ))
1841          done
1842        else
1843          echo "./${ExeNameOut} " >> run_file
1844        fi
1845      fi
1846    done
1847
1848# Then second loop on the components
1849
1850    for comp in ${config_ListOfComponents[*]} ; do
1851
1852      eval ExeNameIn=\${config_Executable_${comp}[0]}
1853      eval ExeNameOut=\${config_Executable_${comp}[1]}
1854
1855      # Only if we really have an executable for the component and not the coupler ie oasis:
1856      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1857
1858        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1859        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1860
1861        if ( ${OK_PARA_MPI} ) ; then
1862          (( mpi_count = 1 ))
1863          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1864            echo "./${ExeNameOut}" >> run_file
1865            (( mpi_count = mpi_count + 1 ))
1866          done
1867        else
1868            echo "./${ExeNameOut} " >> run_file
1869        fi
1870      fi
1871    done
1872
1873    EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
1874
1875    IGCM_sys_Chmod u+x run_file
1876    if ( $DEBUG_sys ) ; then
1877      echo "run_file contains : "
1878      cat run_file
1879    fi
1880
1881  else # Only one executable. launch it.
1882
1883    #
1884    for comp in ${config_ListOfComponents[*]} ; do
1885
1886      # Only if we really have an executable for the component :
1887      eval ExeNameOut=\${config_Executable_${comp}[1]}
1888      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1889        EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
1890      fi
1891
1892    done
1893
1894  fi
1895
1896  IGCM_debug_Print 1 "sys ada : execution command is "
1897  IGCM_debug_Print 1 "$EXECUTION"
1898
1899  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1900}
1901
1902############################################################
1903# Check of space available on temporary filesytems
1904function IGCM_sys_check_quota {
1905  IGCM_debug_PushStack "IGCM_sys_check_quota"
1906  if ( $DEBUG_sys ) ; then
1907    echo "IGCM_sys_check_quota "
1908  fi
1909  # Limit of quota (in %)
1910  limit_quota=90
1911  # Check of the volume
1912  volume_quota=$(quota_u -w | grep 'Quota soft' | awk '{print $5}')
1913  if [ ! X${volume_quota} = X ] ; then
1914    quota_volume=${volume_quota%%\%}
1915#    echo $quota_volume
1916    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1917      IGCM_debug_Print 1 "Please, check your quota of volume on workgpfs"
1918      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1919      IGCM_debug_Print 1 "Use the quota_u -w command to check"
1920      IGCM_debug_Print 1 "You must have more than 10% available to run"
1921      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1922      IGCM_debug_Verif_Exit
1923    fi
1924
1925  fi
1926  IGCM_debug_PopStack "IGCM_sys_check_quota"
1927}
1928
1929#D-#==================================================
1930#D-function IGCM_sys_CountJobInQueue
1931#D-* Purpose: Check if job_name is currently
1932#D-  running or in queue
1933#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1934#D-
1935
1936function IGCM_sys_CountJobInQueue {
1937  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1938  if ( $DEBUG_sys ) ; then
1939    echo "IGCM_sys_CountJobInQueue"
1940  fi
1941
1942  # Print only the full (-W) JobName (%jn)
1943  NbRun=$( llq -W -f %jn | grep -c "$1" )
1944
1945  eval ${2}=${NbRun}
1946
1947  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1948}
1949
1950##############################################################
1951# NCO OPERATOR
1952
1953function IGCM_sys_ncap2 {
1954  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
1955  if ( $DEBUG_sys ) ; then
1956    echo "IGCM_sys_ncap2 :" $@
1957  fi
1958
1959  typeset NB_ESSAI DELAI status i
1960  # number of tentative
1961  NB_ESSAI=3
1962  # time delay between tentative
1963  DELAI=2
1964
1965  i=0
1966  while [ $i -lt $NB_ESSAI ] ; do
1967    ncap2 "$@" > out_rsync 2>&1
1968    status=$?
1969    if [ ${status} -gt 0 ] ; then
1970      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
1971      cat out_rsync
1972      \rm out_rsync
1973      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1974      sleep $DELAI
1975    else
1976      \rm out_rsync
1977      break
1978    fi
1979    (( i = i + 1 ))
1980  done
1981
1982  if [ ${status} -gt 0 ] ; then
1983      echo "IGCM_sys_ncap2 : ncap2 error"
1984      IGCM_debug_Exit "ncap2"
1985  fi
1986
1987  IGCM_debug_PopStack "IGCM_sys_ncap2"
1988}
1989
1990function IGCM_sys_ncatted {
1991  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1992  if ( $DEBUG_sys ) ; then
1993    echo "IGCM_sys_ncatted :" $@
1994  fi
1995
1996  typeset NB_ESSAI DELAI status i
1997  # number of tentative
1998  NB_ESSAI=3
1999  # time delay between tentative
2000  DELAI=2
2001
2002  i=0
2003  while [ $i -lt $NB_ESSAI ] ; do
2004    ncatted "$@" > out_rsync 2>&1
2005    status=$?
2006    if [ ${status} -gt 0 ] ; then
2007      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2008      cat out_rsync
2009      \rm out_rsync
2010      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2011      sleep $DELAI
2012    else
2013      \rm out_rsync
2014      break
2015    fi
2016    (( i = i + 1 ))
2017  done
2018
2019  if [ ${status} -gt 0 ] ; then
2020      echo "IGCM_sys_ncatted : ncatted error"
2021      IGCM_debug_Exit "ncatted"
2022  fi
2023
2024  IGCM_debug_PopStack "IGCM_sys_ncatted"
2025}
2026
2027function IGCM_sys_ncbo {
2028  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2029  if ( $DEBUG_sys ) ; then
2030    echo "IGCM_sys_ncbo :" $@
2031  fi
2032
2033  typeset NB_ESSAI DELAI status i
2034  # number of tentative
2035  NB_ESSAI=3
2036  # time delay between tentative
2037  DELAI=2
2038
2039  i=0
2040  while [ $i -lt $NB_ESSAI ] ; do
2041    ncbo $@ > out_rsync 2>&1
2042    status=$?
2043    if [ ${status} -gt 0 ] ; then
2044      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2045      cat out_rsync
2046      \rm out_rsync
2047      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2048      sleep $DELAI
2049    else
2050      \rm out_rsync
2051      break
2052    fi
2053    (( i = i + 1 ))
2054  done
2055
2056  if [ ${status} -gt 0 ] ; then
2057      echo "IGCM_sys_ncbo : ncbo error"
2058      IGCM_debug_Exit "ncbo"
2059  fi
2060
2061  IGCM_debug_PopStack "IGCM_sys_ncbo"
2062}
2063
2064function IGCM_sys_ncdiff {
2065  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2066  if ( $DEBUG_sys ) ; then
2067    echo "IGCM_sys_ncdiff :" $@
2068  fi
2069
2070  typeset NB_ESSAI DELAI status i
2071  # number of tentative
2072  NB_ESSAI=3
2073  # time delay between tentative
2074  DELAI=2
2075
2076  i=0
2077  while [ $i -lt $NB_ESSAI ] ; do
2078    ncdiff $@ > out_rsync 2>&1
2079    status=$?
2080    if [ ${status} -gt 0 ] ; then
2081      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2082      cat out_rsync
2083      \rm out_rsync
2084      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2085      sleep $DELAI
2086    else
2087      \rm out_rsync
2088      break
2089    fi
2090    (( i = i + 1 ))
2091  done
2092
2093  if [ ${status} -gt 0 ] ; then
2094      echo "IGCM_sys_ncdiff : ncdiff error"
2095      IGCM_debug_Exit "ncdiff"
2096  fi
2097
2098  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2099}
2100
2101function IGCM_sys_ncea {
2102  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2103  if ( $DEBUG_sys ) ; then
2104    echo "IGCM_sys_ncea :" $@
2105  fi
2106
2107  typeset NB_ESSAI DELAI status i
2108  # number of tentative
2109  NB_ESSAI=3
2110  # time delay between tentative
2111  DELAI=2
2112
2113  i=0
2114  while [ $i -lt $NB_ESSAI ] ; do
2115    ncea $@ > out_rsync 2>&1
2116    status=$?
2117    if [ ${status} -gt 0 ] ; then
2118      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2119      cat out_rsync
2120      \rm out_rsync
2121      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2122      sleep $DELAI
2123    else
2124      \rm out_rsync
2125      break
2126    fi
2127    (( i = i + 1 ))
2128  done
2129
2130  if [ ${status} -gt 0 ] ; then
2131      echo "IGCM_sys_ncea : ncea error"
2132      IGCM_debug_Exit "ncea"
2133  fi
2134
2135  IGCM_debug_PopStack "IGCM_sys_ncea"
2136}
2137
2138function IGCM_sys_ncecat {
2139  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2140  if ( $DEBUG_sys ) ; then
2141    echo "IGCM_sys_ncecat :" $@
2142  fi
2143
2144  typeset NB_ESSAI DELAI status i
2145  # number of tentative
2146  NB_ESSAI=3
2147  # time delay between tentative
2148  DELAI=2
2149
2150  i=0
2151  while [ $i -lt $NB_ESSAI ] ; do
2152    ncecat $@ > out_rsync 2>&1
2153    status=$?
2154    if [ ${status} -gt 0 ] ; then
2155      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2156      cat out_rsync
2157      \rm out_rsync
2158      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2159      sleep $DELAI
2160    else
2161      \rm out_rsync
2162      break
2163    fi
2164    (( i = i + 1 ))
2165  done
2166
2167  if [ ${status} -gt 0 ] ; then
2168      echo "IGCM_sys_ncecat : ncecat error"
2169      IGCM_debug_Exit "ncecat"
2170  fi
2171
2172  IGCM_debug_PopStack "IGCM_sys_ncecat"
2173}
2174
2175function IGCM_sys_ncflint {
2176  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2177  if ( $DEBUG_sys ) ; then
2178    echo "IGCM_sys_ncflint :" $@
2179  fi
2180
2181  typeset NB_ESSAI DELAI status i
2182  # number of tentative
2183  NB_ESSAI=3
2184  # time delay between tentative
2185  DELAI=2
2186
2187  i=0
2188  while [ $i -lt $NB_ESSAI ] ; do
2189    ncflint $@ > out_rsync 2>&1
2190    status=$?
2191    if [ ${status} -gt 0 ] ; then
2192      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2193      cat out_rsync
2194      \rm out_rsync
2195      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2196      sleep $DELAI
2197    else
2198      \rm out_rsync
2199      break
2200    fi
2201    (( i = i + 1 ))
2202  done
2203
2204  if [ ${status} -gt 0 ] ; then
2205      echo "IGCM_sys_ncflint : ncflint error"
2206      IGCM_debug_Exit "ncflint"
2207  fi
2208
2209  IGCM_debug_PopStack "IGCM_sys_ncflint"
2210}
2211
2212function IGCM_sys_ncks {
2213  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2214  if ( $DEBUG_sys ) ; then
2215    echo "IGCM_sys_ncks :" $@
2216  fi
2217
2218  typeset NB_ESSAI DELAI status i
2219  # number of tentative
2220  NB_ESSAI=3
2221  # time delay between tentative
2222  DELAI=2
2223
2224  i=0
2225  while [ $i -lt $NB_ESSAI ] ; do
2226    ncks $@ > out_rsync 2>&1
2227    status=$?
2228    if [ ${status} -gt 0 ] ; then
2229      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2230      cat out_rsync
2231      \rm out_rsync
2232      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2233      sleep $DELAI
2234    else
2235      \rm out_rsync
2236      break
2237    fi
2238    (( i = i + 1 ))
2239  done
2240
2241  if [ ${status} -gt 0 ] ; then
2242      echo "IGCM_sys_ncks : ncks error"
2243      IGCM_debug_Exit "ncks"
2244  fi
2245
2246  IGCM_debug_PopStack "IGCM_sys_ncks"
2247}
2248
2249function IGCM_sys_ncpdq {
2250  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2251  if ( $DEBUG_sys ) ; then
2252    echo "IGCM_sys_ncpdq :" $@
2253  fi
2254
2255  typeset NB_ESSAI DELAI status i
2256  # number of tentative
2257  NB_ESSAI=3
2258  # time delay between tentative
2259  DELAI=2
2260
2261  i=0
2262  while [ $i -lt $NB_ESSAI ] ; do
2263    ncpdq $@ > out_rsync 2>&1
2264    status=$?
2265    if [ ${status} -gt 0 ] ; then
2266      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2267      cat out_rsync
2268      \rm out_rsync
2269      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2270      sleep $DELAI
2271    else
2272      \rm out_rsync
2273      break
2274    fi
2275    (( i = i + 1 ))
2276  done
2277
2278  if [ ${status} -gt 0 ] ; then
2279      echo "IGCM_sys_ncpdq : ncpdq error"
2280      IGCM_debug_Exit "ncpdq"
2281  fi
2282
2283  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2284}
2285
2286function IGCM_sys_ncra {
2287  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2288  if ( $DEBUG_sys ) ; then
2289    echo "IGCM_sys_ncra :" $@
2290  fi
2291
2292  typeset NB_ESSAI DELAI status i
2293  # number of tentative
2294  NB_ESSAI=3
2295  # time delay between tentative
2296  DELAI=2
2297
2298  i=0
2299  while [ $i -lt $NB_ESSAI ] ; do
2300    ncra $@ > out_rsync 2>&1
2301    status=$?
2302    if [ ${status} -gt 0 ] ; then
2303      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2304      cat out_rsync
2305      \rm out_rsync
2306      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2307      sleep $DELAI
2308    else
2309      \rm out_rsync
2310      break
2311    fi
2312    (( i = i + 1 ))
2313  done
2314
2315  if [ ${status} -gt 0 ] ; then
2316      echo "IGCM_sys_ncra : ncra error"
2317      IGCM_debug_Exit "ncra"
2318  fi
2319
2320  IGCM_debug_PopStack "IGCM_sys_ncra"
2321}
2322
2323function IGCM_sys_ncrcat {
2324  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2325  if ( $DEBUG_sys ) ; then
2326    echo "IGCM_sys_ncrcat :" $@
2327  fi
2328
2329  typeset NB_ESSAI DELAI status i lastArg
2330  # number of tentative
2331  NB_ESSAI=3
2332  # time delay between tentative
2333  DELAI=2
2334
2335  i=0
2336  while [ $i -lt $NB_ESSAI ] ; do
2337    ncrcat $@ > out_rsync 2>&1
2338    status=$?
2339    if [ ${status} -gt 0 ] ; then
2340      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2341      cat out_rsync
2342      \rm out_rsync
2343      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2344      sleep $DELAI
2345    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
2346      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2347      cat out_rsync
2348      # remove files having corrupted time axis
2349      eval lastArg=\${$#}
2350      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2351      \rm ${lastArg}
2352      \rm out_rsync
2353      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2354      sleep $DELAI
2355    else
2356      \rm out_rsync
2357      break
2358    fi
2359    (( i = i + 1 ))
2360  done
2361
2362  if [ ${status} -gt 0 ] ; then
2363      echo "IGCM_sys_ncrcat : ncrcat error"
2364      #IGCM_debug_Exit "ncrcat"
2365  fi
2366
2367  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2368}
2369
2370function IGCM_sys_ncrename {
2371  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2372  if ( $DEBUG_sys ) ; then
2373    echo "IGCM_sys_ncrename :" $@
2374  fi
2375
2376  typeset NB_ESSAI DELAI status i
2377  # number of tentative
2378  NB_ESSAI=3
2379  # time delay between tentative
2380  DELAI=2
2381
2382  i=0
2383  while [ $i -lt $NB_ESSAI ] ; do
2384    ncrename $@ > out_rsync 2>&1
2385    status=$?
2386    if [ ${status} -gt 0 ] ; then
2387      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2388      cat out_rsync
2389      \rm out_rsync
2390      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2391      sleep $DELAI
2392    else
2393      \rm out_rsync
2394      break
2395    fi
2396    (( i = i + 1 ))
2397  done
2398
2399  if [ ${status} -gt 0 ] ; then
2400      echo "IGCM_sys_ncrename : ncrename error"
2401      IGCM_debug_Exit "ncrename"
2402  fi
2403
2404  IGCM_debug_PopStack "IGCM_sys_ncrename"
2405}
2406
2407function IGCM_sys_ncwa {
2408  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2409  if ( $DEBUG_sys ) ; then
2410    echo "IGCM_sys_ncwa :" $@
2411  fi
2412
2413  typeset NB_ESSAI DELAI status i
2414  # number of tentative
2415  NB_ESSAI=3
2416  # time delay between tentative
2417  DELAI=2
2418
2419  i=0
2420  while [ $i -lt $NB_ESSAI ] ; do
2421    ncwa $@ > out_rsync 2>&1
2422    status=$?
2423    if [ ${status} -gt 0 ] ; then
2424      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2425      cat out_rsync
2426      \rm out_rsync
2427      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2428      sleep $DELAI
2429    else
2430      \rm out_rsync
2431      break
2432    fi
2433    (( i = i + 1 ))
2434  done
2435
2436  if [ ${status} -gt 0 ] ; then
2437      echo "IGCM_sys_ncwa : ncwa error"
2438      IGCM_debug_Exit "ncwa"
2439  fi
2440
2441  IGCM_debug_PopStack "IGCM_sys_ncwa"
2442}
2443
2444##############################################################
2445# CDO OPERATOR
2446
2447function IGCM_sys_cdo {
2448  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2449  if ( $DEBUG_sys ) ; then
2450    echo "IGCM_sys_cdo :" $@
2451  fi
2452
2453  typeset status
2454
2455  \cdo $@ > out_rsync 2>&1
2456  status=$?
2457  if [ ${status} -gt 0 ] ; then
2458    echo "IGCM_sys_cdo : error code ${status}"
2459    cat out_rsync
2460    \rm out_rsync
2461    IGCM_debug_PopStack "IGCM_sys_cdo"
2462    return 1
2463  else
2464    IGCM_debug_PopStack "IGCM_sys_cdo"
2465    return 0
2466  fi
2467
2468  IGCM_debug_PopStack "IGCM_sys_cdo"
2469}
Note: See TracBrowser for help on using the repository browser.