source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh @ 696

Last change on this file since 696 was 696, checked in by aclsce, 12 years ago

For quota check function on mercurex9 and titane, added the treatment of the case "more than 100% of the
quota, i.e with a "*" at the end of the quota value"

  • Property svn:keywords set to Revision Author Date
File size: 67.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Titane
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe param files |            |  Chmod  |                           |
46# |          |      Qsub           |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=titane
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# rsync with path
78typeset -r RSYNC=/usr/bin/rsync
79# RSYNC_opt args to rsync
80typeset -r RSYNC_opt="-va"
81# ie storage filesystem
82typeset -r STOREHOST=${MASTER}
83
84#====================================================
85# Source default environment
86#====================================================
87. /etc/profile
88
89#====================================================
90# Set environment tools (ferret, nco, cdo)
91#====================================================
92. /home/cont003/p86ipsl/.atlas_env_titane_ksh
93if ( [ X${LSB_QUEUE} = Xmono ] || [ X${LSB_QUEUE} = Xmonoext ] ) ; then
94  module switch nco/3.9.4 nco/3.9.4_netcdf4
95fi
96
97#====================================================
98# Specific for ocean additionnal diagnostic
99export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
100export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
101
102#====================================================
103# Host specific DIRECTORIES
104#====================================================
105
106# ============ CESIUM START ============ #
107
108#====================================================
109#- Mirror libIGCM from titane to cesium if needed
110#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
111#if [ ! ${ROOTSYS} = "home" ] ; then
112#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
113#else
114#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
115#fi
116
117#====================================================
118#- libIGCM_POST
119#if ( ${MirrorlibIGCM} ) ; then
120#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
121#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
122#else
123#  typeset -r libIGCM_POST=${libIGCM}
124#fi
125
126# ============ CESIUM  END  ============ #
127
128#====================================================
129#- MirrorlibIGCM for frontend
130typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
131
132#====================================================
133#- libIGCM_POST for frontend
134typeset -r libIGCM_POST=${libIGCM}
135
136#====================================================
137#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
138typeset -r R_EXE="${MODIPSL}/bin"
139
140#====================================================
141#- SUBMIT_DIR : submission dir
142typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
143
144#====================================================
145#- ARCHIVE (dedicated to large files)
146typeset -r ARCHIVE=${CCCSTOREDIR}
147
148#- ARCHIVE (dedicated to small/medium files)
149typeset -r STORAGE=${CCCWORKDIR}
150
151#====================================================
152#- IN
153typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
154typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
155
156#====================================================
157#- R_OUT
158typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
159
160#====================================================
161#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
162typeset -r R_FIG=${STORAGE}/IGCM_OUT
163
164#====================================================
165#- R_BUF (ONLY FOR double copy an scratch)
166typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
167
168#====================================================
169#- BIG_DIR : BIG_DIR to store files waiting for rebuild
170typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
171
172#====================================================
173#- OUT_POST
174typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
175
176#====================================================
177#- RUN_DIR_PATH : Temporary working directory (=> TMP)
178if ( [ ! X${LSB_QUEUE} = Xmono ] && [ ! X${LSB_QUEUE} = Xmonoext ] ) ; then
179  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
180else
181  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
182fi
183
184#====================================================
185#- HOST_MPIRUN_COMMAND
186typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
187
188#====================================================
189#- Max number of arguments passed to nco operator or demigration command
190UNIX_MAX_LIMIT=360
191
192#====================================================
193#- set PackDefault true on titane
194PackDefault=true
195
196#====================================================
197#- Number of core per node (max number of OpenMP task)
198NUM_COREPERNODE=8
199
200#====================================================
201#- Default number of MPI task for IPSL coupled model
202#- required for backward compatibility
203#-
204DEFAULT_NUM_PROC_OCE=5
205DEFAULT_NUM_PROC_CPL=1
206(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
207DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
208
209
210#D-#==================================================
211#D-function IGCM_sys_RshMaster
212#D-* Purpose: Connection to frontend machine.
213#D-* Examples:
214#D-
215function IGCM_sys_RshMaster {
216  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
217  /bin/ksh <<-EOF
218    export libIGCM=${libIGCM}
219    export DEBUG_debug=${DEBUG_debug}
220    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
221    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
222    ${@}
223EOF
224  if [ $? -gt 0 ] ; then
225    echo "IGCM_sys_RshMaster : erreur."
226    IGCM_debug_Exit "IGCM_sys_RshMaster"
227  fi
228  IGCM_debug_PopStack "IGCM_sys_RshMaster"
229}
230
231#D-#==================================================
232#D-function IGCM_sys_RshArchive
233#D-* Purpose: Archive rsh command
234#D-* Examples:
235#D-
236function IGCM_sys_RshArchive {
237  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
238  /bin/ksh <<-EOF
239    ${@}
240EOF
241  if [ $? -gt 0 ] ; then
242    echo "IGCM_sys_RshArchive : erreur."
243    IGCM_debug_Exit "IGCM_sys_RshArchive"
244  fi
245  IGCM_debug_PopStack "IGCM_sys_RshArchive"
246}
247
248#D-#==================================================
249#D-function IGCM_sys_RshPost
250#D-* Purpose: Post-process rsh command
251#D-* Examples:
252#D-
253function IGCM_sys_RshPost {
254  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
255  if ( $DEBUG_sys ) ; then
256    echo "IGCM_sys_RshPost :" $@
257  fi
258
259  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
260  #cat tmp_IGCM_sys_RshPost_$$
261  # keep standard input (stdin) for the loop onto temporary file
262  cat >tmp_IGCM_sys_RshPost_$$
263
264# ============ FRONTEND START ============ #
265
266  /bin/ksh <tmp_IGCM_sys_RshPost_$$
267  if [ $? -gt 0 ] ; then
268    echo "IGCM_sys_RshPost : erreur."
269    IGCM_debug_Exit "IGCM_sys_RshPost"
270  fi
271  # delete temporary file
272  \rm tmp_IGCM_sys_RshPost_$$
273
274# ============ FRONTEND  END  ============ #
275
276# ============ CESIUM START ============ #
277#  typeset NB_ESSAI DELAI status i
278#  if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
279#    #little hack so that rebuild submission is done on titane not an cesium
280#
281#    libIGCM_POST_sed=$( echo $libIGCM_POST | sed 's/\//\\\//g' )
282#    POST_DIR_sed=$( echo ${POST_DIR} | sed 's/\//\\\//g' )
283#    sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
284#    sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
285#    sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
286#    sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
287#    \mv tmp.txt tmp_IGCM_sys_RshPost_$$
288#
289#    echo cat tmp_IGCM_sys_RshPost_$$ AFTER
290#    cat tmp_IGCM_sys_RshPost_$$
291#
292#    /bin/ksh <tmp_IGCM_sys_RshPost_$$
293#    if [ $? -gt 0 ] ; then
294#      echo "IGCM_sys_RshPost : erreur."
295#      IGCM_debug_Exit "IGCM_sys_RshPost"
296#    fi
297#    # delete temporary file
298#    \rm tmp_IGCM_sys_RshPost_$$
299#
300#  else
301#    # number of tentative
302#    NB_ESSAI=10
303#    # time delay between tentative
304#    DELAI=10
305#    i=0
306#    while [ $i -ne $NB_ESSAI ] ; do
307#      ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
308#      status=$?
309#      if [ ${status} -ne 0 ];
310#      then
311#       sleep $DELAI
312#      else
313#       break
314#      fi
315#      let i=$i+1
316#    done
317#    # delete temporary file
318#    \rm tmp_IGCM_sys_RshPost_$$
319#
320#    if [ ${status} -gt 0 ] ; then
321#      echo "IGCM_sys_RshPost : erreur."
322#      IGCM_debug_Exit "IGCM_sys_RshPost"
323#    fi
324#  fi
325
326# ============ CESIUM  END  ============ #
327
328  IGCM_debug_PopStack "IGCM_sys_RshPost"
329}
330
331#D-#==================================================
332#D-function IGCM_sys_SendMail
333#D-* Purpose: Send mail when simulation is over
334#D-* Examples:
335#D-
336function IGCM_sys_SendMail {
337  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
338  if ( $DEBUG_sys ) ; then
339    echo "IGCM_sys_SendMail :" $@
340  fi
341
342  if ( ${ExitFlag} ) ; then
343    status=failed
344  else
345    status=completed
346  fi
347
348    cat  << END_MAIL > job_end.mail
349Dear ${LOGIN},
350
351  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
352  Job started : ${DateBegin}
353  Job ended   : ${DateEnd}
354  Output files are available in ${R_SAVE}
355  Files to be rebuild are temporarily available in ${REBUILD_DIR}
356  Pre-packed files are temporarily available in ${R_BUFR}
357  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
358END_MAIL
359
360    if [ ! -z ${config_UserChoices_MailName} ] ; then
361        mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
362    elif [ -f ~/.forward ] ; then
363        mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
364    fi
365    sleep 10
366    rm -f job_end.mail
367
368  if [ $? -gt 0 ] ; then
369    echo "IGCM_sys_SendMail : erreur."
370    IGCM_debug_Exit "IGCM_sys_SendMail"
371  fi
372  IGCM_debug_PopStack "IGCM_sys_SendMail"
373}
374
375#D-#==================================================
376#D-function IGCM_sys_Mkdir
377#D-* Purpose: Master locale mkdir command
378#D-* Examples:
379#D-
380function IGCM_sys_Mkdir {
381  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
382  if ( $DEBUG_sys ) ; then
383    echo "IGCM_sys_Mkdir :" $@
384  fi
385  if [ ! -d ${1} ]; then
386    \mkdir -p $1
387    if [ $? -gt 0 ] ; then
388      echo "IGCM_sys_Mkdir : erreur."
389      IGCM_debug_Exit "IGCM_sys_Mkdir"
390    fi
391  fi
392  # vérification :
393  if [ ! -d ${1} ] ; then
394    echo "IGCM_sys_Mkdir : erreur."
395    IGCM_debug_Exit "IGCM_sys_Mkdir"
396  fi
397  IGCM_debug_PopStack "IGCM_sys_Mkdir"
398}
399
400#D-#==================================================
401#D-function IGCM_sys_MkdirArchive
402#D-* Purpose: Mkdir on Archive
403#D-* Examples:
404#D-
405function IGCM_sys_MkdirArchive {
406  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
407  if ( $DEBUG_sys ) ; then
408    echo "IGCM_sys_MkdirArchive :" $@
409  fi
410  #- creation de repertoire sur le serveur fichier
411  if [ ! -d ${1} ]; then
412    \mkdir -p $1
413    if [ $? -gt 0 ] ; then
414      echo "IGCM_sys_MkdirArchive : erreur."
415      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
416    fi
417  fi
418  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
419}
420
421#D-#==================================================
422#D-function IGCM_sys_MkdirWork
423#D-* Purpose: Mkdir on Work
424#D-* Examples:
425#D-
426function IGCM_sys_MkdirWork {
427  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
428  if ( $DEBUG_sys ) ; then
429    echo "IGCM_sys_MkdirWork :" $@
430  fi
431  #- creation de repertoire sur le serveur fichier
432  if [ ! -d ${1} ]; then
433    \mkdir -p $1
434    if [ $? -gt 0 ] ; then
435      echo "IGCM_sys_MkdirWork : erreur."
436      IGCM_debug_Exit "IGCM_sys_MkdirWork"
437    fi
438  fi
439  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
440}
441
442#D-#==================================================
443#D-function IGCM_sys_Cd
444#D-* Purpose: master cd command
445#D-* Examples:
446#D-
447function IGCM_sys_Cd {
448  IGCM_debug_PushStack "IGCM_sys_Cd" $@
449  if ( $DEBUG_sys ) ; then
450    echo "IGCM_sys_Cd :" $@
451  fi
452  \cd $1
453  if [ $? -gt 0 ] ; then
454    echo "IGCM_sys_Cd : erreur."
455    IGCM_debug_Exit "IGCM_sys_Cd"
456  fi
457  IGCM_debug_PopStack "IGCM_sys_Cd"
458}
459
460#D-#==================================================
461#D-function IGCM_sys_Chmod
462#D-* Purpose: Chmod
463#D-* Examples:
464#D-
465function IGCM_sys_Chmod {
466  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
467  if ( $DEBUG_sys ) ; then
468    echo "IGCM_sys_Chmod :" $@
469  fi
470  if [ $DRYRUN -le 1 ]; then
471    \chmod $@
472    if [ $? -gt 0 ] ; then
473      echo "IGCM_sys_Chmod : erreur."
474      IGCM_debug_Exit "IGCM_sys_Chmod"
475    fi
476  else
477    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
478  fi
479  IGCM_debug_PopStack "IGCM_sys_Chmod"
480}
481
482#D-#==================================================
483#D-function IGCM_sys_FileSize
484#D-* Purpose: Filesize
485#D-* Examples:
486#D-
487function IGCM_sys_FileSize {
488  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
489
490  typeset sizeF
491  set +A sizeF -- $( ls -la ${1} )
492  if [ $? -gt 0 ] ; then
493    IGCM_debug_Exit "IGCM_sys_FileSize"
494  fi
495  eval ${2}=${sizeF[4]}
496
497  IGCM_debug_PopStack "IGCM_sys_FileSize"
498}
499
500#D-#==================================================
501#D-function IGCM_sys_TestDir
502#D-* Purpose: Test Directory that must exists
503#D-* Examples:
504#D-
505function IGCM_sys_TestDir {
506  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
507  if ( $DEBUG_sys ) ; then
508    echo "IGCM_sys_TestDir :" $@
509  fi
510  typeset ExistFlag
511  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
512  IGCM_debug_PopStack "IGCM_sys_TestDir"
513
514  return ${ExistFlag}
515}
516
517#D-#==================================================
518#D-function IGCM_sys_TestDirArchive
519#D-* Purpose: Test Directory that must exists on Archive
520#D-* Examples:
521#D-
522function IGCM_sys_TestDirArchive {
523  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
524  if ( $DEBUG_sys ) ; then
525    echo "IGCM_sys_TestDirArchive :" $@
526  fi
527  typeset ExistFlag
528  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
529  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
530
531  return ${ExistFlag}
532}
533
534#D-#==================================================
535#D-function IGCM_sys_IsFileArchived
536#D-* Purpose: Test file that must NOT EXISTS on Archive
537#D-* Examples:
538#D-
539function IGCM_sys_IsFileArchived {
540  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
541  if ( $DEBUG_sys ) ; then
542    echo "IGCM_sys_IsFileArchived :" $@
543  fi
544  typeset IsArchivedFlag
545  IsArchivedFlag=$( [ X$( echo $1 | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
546  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
547
548  return ${IsArchivedFlag}
549}
550
551#D-#==================================================
552#D-function IGCM_sys_TestFileArchive
553#D-* Purpose: Test file that must NOT EXISTS on Archive
554#D-* Examples:
555#D-
556function IGCM_sys_TestFileArchive {
557  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
558  typeset ExistFlag
559  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
560  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
561
562  return ${ExistFlag}
563}
564
565#D-#==================================================
566#D-function IGCM_sys_TestFileBuffer
567#D-* Purpose: Test file that must NOT EXISTS on Buffer
568#D-* Examples:
569#D-
570function IGCM_sys_TestFileBuffer {
571  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
572  typeset ExistFlag
573  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
574  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
575
576  return ${ExistFlag}
577}
578
579#D-#==================================================
580#D-function IGCM_sys_CountFileArchive
581#D-* Purpose: Count files on Archive filesystem
582#D-* Examples:
583#D-
584function IGCM_sys_CountFileArchive {
585  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
586  ls ${@} 2>/dev/null | wc -l
587  if [ $? -gt 0 ] ; then
588    echo "IGCM_sys_CountFileArchive : erreur."
589  fi
590  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
591}
592
593#D-#==================================================
594#D-function IGCM_sys_CountFileBuffer
595#D-* Purpose: Count files on Scratch filesystem
596#D-* Examples:
597#D-
598function IGCM_sys_CountFileBuffer {
599  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
600  ls ${@} 2>/dev/null | wc -l
601  if [ $? -gt 0 ] ; then
602    echo "IGCM_sys_CountFileBuffer : erreur."
603  fi
604  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
605}
606
607#D-#==================================================
608#D-function IGCM_sys_Tree
609#D-* Purpose: Tree directories with files on ${ARCHIVE}
610#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
611#D-
612function IGCM_sys_Tree {
613  IGCM_debug_PushStack "IGCM_sys_Tree" $@
614  if ( $DEBUG_sys ) ; then
615    echo "IGCM_sys_Tree :" $@
616  fi
617
618  \ls -lR ${@}
619
620  IGCM_debug_PopStack "IGCM_sys_Tree"
621}
622
623#D-#==================================================
624#D-function IGCM_sys_Tar
625#D-* Purpose: master tar command
626#D-* Examples:
627#D-
628function IGCM_sys_Tar {
629  IGCM_debug_PushStack "IGCM_sys_Tar" $@
630  if ( $DEBUG_sys ) ; then
631    echo "IGCM_sys_Tar :" $@
632  fi
633  \tar cf $@
634  if [ $? -gt 0 ] ; then
635    echo "IGCM_sys_Tar : erreur."
636    IGCM_debug_Exit "IGCM_sys_Tar"
637  fi
638  IGCM_debug_PopStack "IGCM_sys_Tar"
639}
640
641#D-#==================================================
642#D-function IGCM_sys_UnTar
643#D-* Purpose: master un-tar command
644#D-* Examples:
645#D-
646function IGCM_sys_UnTar {
647  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
648  if ( $DEBUG_sys ) ; then
649    echo "IGCM_sys_UnTar :" $@
650  fi
651  \tar xvf $1
652  if [ $? -gt 0 ] ; then
653    echo "IGCM_sys_UnTar : erreur."
654    IGCM_debug_Exit "IGCM_sys_UnTar"
655  fi
656  IGCM_debug_PopStack "IGCM_sys_UnTar"
657}
658
659#D-#==================================================
660#D-function IGCM_sys_Qsub
661#D-* Purpose: Qsub new job
662#D-* Examples:
663#D-
664function IGCM_sys_Qsub {
665  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
666  if ( $DEBUG_sys ) ; then
667    echo "IGCM_sys_Qsub :" $@
668  fi
669  typeset options
670  if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
671    options=" -o ${Script_Output} -e ${Script_Output}"
672  else
673    options=" -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
674  fi
675
676  /usr/local/bin/ccc_msub ${options} < $1
677  if [ $? -gt 0 ] ; then
678    echo "IGCM_sys_Qsub : erreur ${options} $1"
679    IGCM_debug_Exit "IGCM_sys_Qsub"
680  fi
681  IGCM_debug_PopStack "IGCM_sys_Qsub"
682}
683
684#D-#==================================================
685#D-function IGCM_sys_QsubPost
686#D-* Purpose: Qsub new job on scalaire
687#D-* Examples:
688#D-
689function IGCM_sys_QsubPost {
690  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
691  if ( $DEBUG_sys ) ; then
692    echo "IGCM_sys_QsubPost :" $@
693  fi
694
695  # Need to export listVarEnv for cesium
696  # NO Need to export listVarEnv for titane
697# ============ FRONTEND START ============ #
698
699  /usr/local/bin/ccc_msub -p ${BRIDGE_MSUB_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
700
701# ============ FRONTEND  END  ============ #
702
703# ============ CESIUM START ============ #
704#  typeset NB_ESSAI DELAI status i
705#  # number of tentative
706#  NB_ESSAI=10
707#  # time delay between tentative
708#  DELAI=10
709#  (( i = 0 ))
710#  while [ $i -lt $NB_ESSAI ] ; do
711#    /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
712#    status=$?
713#    if [ ${status} -gt 0 ]; then
714#      sleep $DELAI
715#    else
716#      break
717#    fi
718#    (( i = i + 1 ))
719#  done
720# ============ CESIUM  END  ============ #
721
722  if [ $? -gt 0 ] ; then
723    echo "IGCM_sys_QsubPost : erreur " $@
724    IGCM_debug_Exit "IGCM_sys_QsubPost"
725  fi
726  IGCM_debug_PopStack "IGCM_sys_QsubPost"
727}
728
729#D-*************************
730#D- File transfer functions
731#D-*************************
732#D-
733
734#D-#==================================================
735#D-function IGCM_sys_Rsync_out
736#D-* Purpose: treat return val of rsync
737#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
738#D-  Error values and explanations can depend on your system version.
739function IGCM_sys_Rsync_out {
740  RET=$1
741  if [ ! $RET ] ; then
742    echo "rsync error !"
743  fi
744
745  if [ $MYLANG = "fr" ]; then
746    case $RET in
747    0)  return ;;
748    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
749      echo "Erreur de syntaxe ou d'utilisation."
750      return;;
751    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
752      echo "Incompatibilité de protocole."
753      return;;
754    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
755      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
756      echo "répertoires"
757      return;;
758    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
759      echo "Action demandée non supportée : une tentative de manipulation de"
760      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
761      echo "été faite ; ou une option qui est supportée par le  client  mais"
762      echo "pas par le serveur a été spécifiée."
763      return;;
764    10) echo "Erreur de rsync ; RERR_SOCKETIO"
765      echo "Erreur dans le socket d'entrée sortie"
766      return;;
767    11) echo "Erreur de rsync ; RERR_FILEIO"
768      echo "Erreur d'entrée sortie fichier"
769      return;;
770    12) echo "Erreur de rsync ; RERR_STREAMIO"
771      echo "Erreur dans flux de donnée du protocole rsync"
772      return;;
773    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
774      echo "Erreur avec les diagnostics du programme"
775      return;;
776    14) echo "Erreur de rsync ; RERR_IPC"
777      echo "Erreur dans le code IPC"
778      return;;
779    20) echo "Erreur de rsync ; RERR_SIGNAL"
780      echo "SIGUSR1 ou SIGINT reçu"
781      return;;
782    21) echo "Erreur de rsync ; RERR_WAITCHILD"
783      echo "Une erreur retournée par waitpid()"
784      return;;
785    22) echo "Erreur de rsync ; RERR_MALLOC"
786      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
787      return;;
788    23) echo ""
789      echo "Erreur fichier inexistant"
790      return;;
791    30) echo "Erreur de rsync ; RERR_TIMEOUT"
792      echo "Temps d'attente écoulé dans l'envoi/réception de données"
793      return;;
794    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
795      return;;
796    esac
797  elif [ $MYLANG = "en" ] ; then
798    case $RET in
799    0)  return;;
800    1)  echo "rsync error : Syntax or usage error "
801      return;;
802    2)  echo "rsync error : Protocol incompatibility "
803      return;;
804    3)  echo "rsync error : Errors selecting input/output files, dirs"
805      return;;
806    4)  echo "rsync error : Requested action not supported: an attempt"
807      echo "was made to manipulate 64-bit files on a platform that cannot support"
808      echo "them; or an option was specified that is supported by the client and"
809      echo "not by the server."
810      return;;
811    5)  echo "rsync error : Error starting client-server protocol"
812      return;;
813    10) echo "rsync error : Error in socket I/O "
814      return;;
815    11) echo "rsync error : Error in file I/O "
816      return;;
817    12) echo "rsync error : Error in rsync protocol data stream "
818      return;;
819    13) echo "rsync error : Errors with program diagnostics "
820      return;;
821    14) echo "rsync error : Error in IPC code "
822      return;;
823    20) echo "rsync error : Received SIGUSR1 or SIGINT "
824      return;;
825    21) echo "rsync error : Some error returned by waitpid() "
826      return;;
827    22) echo "rsync error : Error allocating core memory buffers "
828      return;;
829    23) echo "rsync error : Partial transfer due to error"
830      return;;
831    24) echo "rsync error : Partial transfer due to vanished source files"
832      return;;
833    30) echo "rsync error : Timeout in data send/receive "
834      return;;
835    *)  echo "rsync error : return code of rsync unknown :" $RET
836      return;;
837    esac
838  else
839    echo "unknown language $MYLANG."
840    return
841  fi
842}
843
844#D-#==================================================
845#D-function IGCM_sys_Miror_libIGCM
846#D-* Purpose: Mirror libIGCM PATH and lib to cesium
847#D-* Examples:
848#D-
849function IGCM_sys_Mirror_libIGCM {
850  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
851  if ( $DEBUG_sys ) ; then
852    echo "IGCM_sys_Mirror_libIGCM"
853  fi
854
855  typeset RET DEST
856
857  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
858
859  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
860  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
861  RET=$?
862
863  if [ ${RET} -gt 0 ] ; then
864    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
865    cat out_rsync
866  fi
867  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
868}
869
870#D-#==================================================
871#D-function IGCM_sys_Cp
872#D-* Purpose: generic cp
873#D-* Examples:
874#D-
875function IGCM_sys_Cp {
876  IGCM_debug_PushStack "IGCM_sys_Cp" $@
877  if ( $DEBUG_sys ) ; then
878    echo "IGCM_sys_Cp :" $@
879  fi
880
881  typeset RET
882
883  echo cp $@ > out_rsync 2>&1
884  \cp $@ >> out_rsync 2>&1
885  RET=$?
886
887  if [ ${RET} -gt 0 ] ; then
888    echo "IGCM_sys_Cp : error."
889    cat out_rsync
890    IGCM_debug_Exit "IGCM_sys_Cp"
891  else
892    \rm out_rsync
893  fi
894  IGCM_debug_PopStack "IGCM_sys_Cp"
895}
896
897#D-#==================================================
898#D-function IGCM_sys_Rm
899#D-* Purpose: generic rm
900#D-* Examples:
901#D-
902function IGCM_sys_Rm {
903  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
904  if ( $DEBUG_sys ) ; then
905    echo "IGCM_sys_Rm :" $@
906  fi
907
908  typeset RET
909
910  echo rm $@ > out_rsync 2>&1
911  \rm $@ >> out_rsync 2>&1
912  RET=$?
913
914  if [ ${RET} -gt 0 ] ; then
915    echo "IGCM_sys_Rm : error."
916    cat out_rsync
917    IGCM_debug_Exit "IGCM_sys_Rm"
918  else
919    \rm out_rsync
920  fi
921  IGCM_debug_PopStack "IGCM_sys_Rm"
922}
923
924#D-#==================================================
925#D-function IGCM_sys_RmRunDir
926#D-* Purpose: rm tmpdir (dummy function most of the time batch
927#D-                      scheduler will do the job)
928#D-* Examples:
929#D-
930function IGCM_sys_RmRunDir {
931  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
932  if ( $DEBUG_sys ) ; then
933    echo "IGCM_sys_RmRunDir :" $@
934  fi
935
936  typeset RET
937
938  echo rm $@ > out_rsync 2>&1
939  \rm $@ >> out_rsync 2>&1
940  RET=$?
941
942  if [ ${RET} -gt 0 ] ; then
943    echo "IGCM_sys_RmRunDir : error."
944    cat out_rsync
945    IGCM_debug_Exit "IGCM_sys_RmRunDir"
946  else
947    \rm out_rsync
948  fi
949  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
950}
951
952#D-#==================================================
953#D-function IGCM_sys_Mv
954#D-* Purpose: generic move
955#D-* Examples:
956#D-
957function IGCM_sys_Mv {
958  IGCM_debug_PushStack "IGCM_sys_Mv" $@
959  if ( $DEBUG_sys ) ; then
960    echo "IGCM_sys_Mv :" $@
961  fi
962
963  if [ $DRYRUN = 0 ]; then
964
965    typeset RET
966
967    echo mv $@ > out_rsync 2>&1
968    \mv $@ >> out_rsync 2>&1
969    RET=$?
970
971    if [ ${RET} -gt 0 ] ; then
972      echo "IGCM_sys_Mv : error in mv."
973      cat out_rsync
974      IGCM_debug_Exit "IGCM_sys_Mv"
975    else
976      \rm out_rsync
977    fi
978  else
979    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
980  fi
981
982  IGCM_debug_PopStack "IGCM_sys_Mv"
983}
984
985#D-#==================================================
986#D-function IGCM_sys_Put_Dir
987#D-* Purpose: Copy a complete directory on $(ARCHIVE)
988#D-* Examples:
989#D-
990function IGCM_sys_Put_Dir {
991  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
992  if ( $DEBUG_sys ) ; then
993    echo "IGCM_sys_Put_Dir :" $@
994  fi
995  if [ $DRYRUN = 0 ]; then
996    if [ ! -d ${1} ] ; then
997      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
998      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
999      return
1000    fi
1001
1002    typeset RET
1003
1004    # Only if we use rsync
1005    #IGCM_sys_TestDirArchive $( dirname $2 )
1006    #
1007    #USUAL WAY
1008    \cp -r $1 $2 > out_rsync 2>&1
1009    RET=$?
1010
1011    if [ ${RET} -gt 0 ] ; then
1012      echo "IGCM_sys_Put_Dir : error."
1013      cat out_rsync
1014      IGCM_debug_Exit "IGCM_sys_Put_Dir"
1015    else
1016      \rm out_rsync
1017    fi
1018  else
1019    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1020  fi
1021  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1022}
1023
1024#D-#==================================================
1025#D-function IGCM_sys_Get_Dir
1026#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1027#D-* Examples:
1028#D-
1029function IGCM_sys_Get_Dir {
1030  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1031  if ( $DEBUG_sys ) ; then
1032    echo "IGCM_sys_Get_Dir :" $@
1033  fi
1034  if [ $DRYRUN = 0 ]; then
1035#    if [ ! -d ${1} ] ; then
1036#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1037#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1038#      return
1039#    fi
1040
1041    typeset RET
1042
1043    # Only if we use rsync
1044    #IGCM_sys_TestDirArchive $( dirname $2 )
1045    #
1046    # USUAL WAY
1047    # add dmfind/dmget (to demigrate all offline files) :
1048    #dmfind $1 -state MIG -o -state OFL -o -state PAR | dmget
1049    \cp -r $1 $2 > out_rsync 2>&1
1050    RET=$?
1051
1052    if [ ${RET} -gt 0 ] ; then
1053      echo "IGCM_sys_Get_Dir : error."
1054      cat out_rsync
1055      IGCM_debug_Exit "IGCM_sys_Get_Dir"
1056    else
1057      \rm out_rsync
1058    fi
1059  else
1060    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1061  fi
1062  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1063}
1064
1065#D-#==================================================
1066#D-function IGCM_sys_Get_Master
1067#D-* Purpose: Copy a complete directory from MASTER filesystem
1068#D-* Examples:
1069#D-
1070function IGCM_sys_Get_Master {
1071  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1072  if ( $DEBUG_sys ) ; then
1073    echo "IGCM_sys_Get_Master :" $@
1074  fi
1075  if [ $DRYRUN = 0 ]; then
1076    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1077      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1078      IGCM_debug_PopStack "IGCM_sys_Get_Master"
1079      return
1080    fi
1081
1082    typeset RET
1083
1084    #USUAL WAY
1085    \cp -rL $1 $2 > out_rsync 2>&1
1086    RET=$?
1087
1088    if [ ${RET} -gt 0 ] ; then
1089      echo "IGCM_sys_Get_Master : error."
1090      cat out_rsync
1091      IGCM_debug_Exit "IGCM_sys_Get_Master"
1092    else
1093      \rm out_rsync
1094    fi
1095  else
1096    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1097  fi
1098  IGCM_debug_PopStack "IGCM_sys_Get_Master"
1099}
1100
1101#====================================================
1102#- Call IGCM_sys_Mirror_libIGCM now !
1103if ( $MirrorlibIGCM ) ; then
1104  IGCM_sys_Mirror_libIGCM
1105fi
1106
1107#D-#==================================================
1108#D-function IGCM_sys_Put_Rest
1109#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1110#D-           File and target directory must exist.
1111#D-* Examples:
1112#D-
1113function IGCM_sys_Put_Rest {
1114  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1115  if ( $DEBUG_sys ) ; then
1116    echo "IGCM_sys_Put_Rest :" $@
1117  fi
1118  if [ $DRYRUN = 0 ]; then
1119    if [ ! -f ${1} ] ; then
1120      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1121      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1122    fi
1123
1124    typeset RET
1125    #
1126    if [ X${JobType} = XRUN ] ; then
1127      IGCM_sys_Chmod 444 ${1}
1128    fi
1129
1130    #
1131    # USUAL WAY
1132    \cp $1 $2 > out_rsync 2>&1
1133    RET=$?
1134
1135#       #RSYNC WITH NETWORK SSH CALL
1136#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1137#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1138
1139#       #RSYNC WITH NFS USE
1140#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1141#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1142
1143#       RET=$?
1144#       IGCM_sys_Rsync_out $RET
1145
1146#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1147#       (( RET=RET+$? ))
1148
1149    if [ ${RET} -gt 0 ] ; then
1150      echo "IGCM_sys_Put_Rest : error."
1151      cat out_rsync
1152      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1153    else
1154      \rm out_rsync
1155    fi
1156  else
1157    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1158  fi
1159  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1160}
1161
1162#D-#==================================================
1163#D-function IGCM_sys_PutBuffer_Rest
1164#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1165#D-           File and target directory must exist.
1166#D-* Examples:
1167#D-
1168function IGCM_sys_PutBuffer_Rest {
1169  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1170  if ( $DEBUG_sys ) ; then
1171    echo "IGCM_sys_PutBuffer_Rest :" $@
1172  fi
1173  if [ $DRYRUN = 0 ]; then
1174    if [ ! -f ${1} ] ; then
1175      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1176      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1177    fi
1178
1179    typeset RET
1180    #
1181    if [ X${JobType} = XRUN ] ; then
1182      IGCM_sys_Chmod 444 ${1}
1183    fi
1184
1185    #
1186    # USUAL WAY
1187    \cp $1 $2 > out_rsync 2>&1
1188    RET=$?
1189
1190    if [ ${RET} -gt 0 ] ; then
1191      echo "IGCM_sys_PutBuffer_Rest : error."
1192      cat out_rsync
1193      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1194    else
1195      \rm out_rsync
1196    fi
1197  else
1198    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1199  fi
1200  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1201}
1202
1203#D-#==================================================
1204#D-function IGCM_sys_Put_Out
1205#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1206#D-* Examples:
1207#D-
1208function IGCM_sys_Put_Out {
1209  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1210  if ( $DEBUG_sys ) ; then
1211    echo "IGCM_sys_Put_Out :" $@
1212  fi
1213
1214  typeset NB_ESSAI DELAI RET i
1215
1216  # number of tentative
1217  NB_ESSAI=3
1218  # time delay between tentative
1219  DELAI=2
1220
1221  if [ $DRYRUN = 0 ]; then
1222    if [ ! -f ${1} ] ; then
1223      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1224      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1225      return 1
1226    fi
1227    #
1228    IGCM_sys_MkdirArchive $( dirname $2 )
1229    #
1230
1231    #=====================================================
1232    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1233    #=====================================================
1234
1235    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
1236    #if [ $? -eq 0 ] ; then
1237    #    typeset WORKPATH FILEPATH
1238    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
1239    #    IGCM_sys_MkdirWork ${WORKPATH}
1240    #    FILEPATH=${WORKPATH}/$( basename $2 )
1241    #    #
1242    #    IGCM_sys_Cp ${1} ${FILEPATH}
1243    #fi
1244
1245    if [ X${JobType} = XRUN ] ; then
1246      if [ X${3} = X ] ; then
1247        IGCM_sys_Chmod 444 ${1}
1248      fi
1249    fi
1250    #
1251
1252    (( i = 0 ))
1253    while [ $i -lt $NB_ESSAI ] ; do
1254    # USUAL WAY
1255      \cp $1 $2 > out_rsync 2>&1
1256      RET=$?
1257      if [ ${RET} -gt 0 ]; then
1258        IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed ${i}/${NB_ESSAI}"
1259        IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1260        sleep $DELAI
1261      else
1262        break
1263      fi
1264      (( i = i + 1 ))
1265    done
1266
1267#       #RSYNC WITH NETWORK SSH CALL
1268#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1269#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1270
1271#       #RSYNC WITH NFS USE
1272#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1273#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1274
1275#       RET=$?
1276#       IGCM_sys_Rsync_out $RET
1277
1278#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1279#       (( RET=RET+$? ))
1280
1281    if [ ${RET} -gt 0 ] ; then
1282      echo "IGCM_sys_Put_Out : error."
1283      cat out_rsync
1284      IGCM_debug_Exit "IGCM_sys_Put_Out"
1285    else
1286      \rm out_rsync
1287    fi
1288  else
1289    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1290  fi
1291  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1292  return 0
1293}
1294
1295#D-#==================================================
1296#D-function IGCM_sys_PutBuffer_Out
1297#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1298#D-* Examples:
1299#D-
1300function IGCM_sys_PutBuffer_Out {
1301  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1302  if ( $DEBUG_sys ) ; then
1303    echo "IGCM_sys_PutBuffer_Out :" $@
1304  fi
1305  if [ $DRYRUN = 0 ]; then
1306    if [ ! -f ${1} ] ; then
1307      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1308      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1309      return 1
1310    fi
1311    #
1312    IGCM_sys_Mkdir $( dirname $2 )
1313    #
1314    typeset RET
1315
1316    if [ X${JobType} = XRUN ] ; then
1317      if [ X${3} = X ] ; then
1318        IGCM_sys_Chmod 444 ${1}
1319      fi
1320    fi
1321    #
1322    # USUAL WAY
1323    \cp $1 $2 > out_rsync 2>&1
1324    RET=$?
1325
1326    if [ ${RET} -gt 0 ] ; then
1327      echo "IGCM_sys_PutBuffer_Out : error."
1328      cat out_rsync
1329      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1330    else
1331      \rm out_rsync
1332    fi
1333  else
1334    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1335  fi
1336  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1337  return 0
1338}
1339
1340#D-#==================================================
1341#D-function IGCM_sys_Get
1342#D-* Purpose: Get a file from ${ARCHIVE}
1343#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1344#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1345function IGCM_sys_Get {
1346  IGCM_debug_PushStack "IGCM_sys_Get" $@
1347
1348  typeset DEST RET dm_liste ifile target file_work
1349  typeset NB_ESSAI DELAI status i
1350
1351  if ( $DEBUG_sys ) ; then
1352    echo "IGCM_sys_Get :" $@
1353  fi
1354
1355  # number of tentative
1356  NB_ESSAI=3
1357  # time delay between tentative
1358  DELAI=2
1359
1360  if [ $DRYRUN -le 2 ]; then
1361    if [ X${1} = X'/l' ] ; then
1362      # test if the first file is present in the old computation :
1363      eval set +A dm_liste \${${2}}
1364    else
1365      eval set +A dm_liste ${1}
1366    fi
1367    eval DEST=\${${#}}
1368
1369    #=====================================================
1370    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1371    #=====================================================
1372
1373    # Is it an R_OUT file (not R_IN) ?
1374    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
1375    #if [ $? -eq 0 ] ; then
1376    #    # Yes  ? then we try to get it in SCRATCHDIR
1377    #    set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
1378    #    if [ -f ${file_work[0]} ] ; then
1379    #   IGCM_sys_Cp ${file_work[*]} ${DEST}
1380    #   IGCM_debug_PopStack "IGCM_sys_Get"
1381    #   return
1382    #    fi
1383    #fi
1384
1385    ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1386    RET=$?
1387    if [ ${RET} -gt 0 ] ; then
1388      echo "WARNING IGCM_sys_Get : demigration error."
1389      cat out_rsync
1390      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1391    fi
1392
1393    #if [ ${RET} -gt 0 ] ; then
1394    #    if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1395    #   cat out_rsync
1396    #   echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1397    #   sleep 30
1398    #   echo "We try another time"
1399    ##  dmget ${dm_liste[*]} > out_rsync 2>&1
1400    #   ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1401    #   RET=$?
1402    #   if [ ${RET} -gt 0 ] ; then
1403    #       echo "ERROR IGCM_sys_Get : again demigration error :"
1404    #       cat out_rsync
1405    #       IGCM_debug_Exit "IGCM_sys_Get"
1406    #   fi
1407    #    else
1408    #   echo "ERROR IGCM_sys_Get : demigration error :"
1409    #   cat out_rsync
1410    #   IGCM_debug_Exit "IGCM_sys_Get"
1411    #    fi
1412    #fi
1413
1414    #USUAL WAY
1415    if [ X${1} = X'/l' ] ; then
1416      (( RET=0 ))
1417      for target in ${dm_liste[*]} ; do
1418        local_file=$( basename ${target} )
1419        (( i = 0 ))
1420        # test if the target file is present before the loop
1421        IGCM_sys_TestFileArchive ${target}
1422        RET=$?
1423        if [ ${RET} -gt 0 ] ; then
1424          echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1425          IGCM_debug_Exit "IGCM_sys_Get"
1426        else
1427          while [ $i -lt $NB_ESSAI ] ; do
1428            \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1429            status=$?
1430            if [ ${status} -gt 0 ]; then
1431              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed ${i}/${NB_ESSAI}"
1432              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1433              sleep $DELAI
1434            else
1435              break
1436            fi
1437            (( i = i + 1 ))
1438          done
1439          (( RET = RET + status ))
1440        fi
1441      done
1442    else
1443      \cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1444      RET=$?
1445    fi
1446
1447#       #RSYNC WITH NETWORK SSH CALL
1448#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1449#       ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1450
1451#       #RSYNC WITH NFS USE
1452#       echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1453#       ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1454
1455#       RET=$?
1456#       IGCM_sys_Rsync_out $RET
1457
1458#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1459#       (( RET=RET+$? ))
1460
1461    if [ ${RET} -gt 0 ] ; then
1462      echo "IGCM_sys_Get : copy error."
1463      cat out_rsync
1464      IGCM_debug_Exit "IGCM_sys_Get"
1465    else
1466      \rm out_rsync
1467    fi
1468  else
1469    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1470  fi
1471  IGCM_debug_PopStack "IGCM_sys_Get"
1472}
1473
1474#D-#==================================================
1475#D-function IGCM_sys_GetBuffer
1476#D-* Purpose: Get a file from ${SCRATCHDIR}
1477#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1478#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1479function IGCM_sys_GetBuffer {
1480  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1481
1482  typeset DEST RET buf_liste ifile target file_work
1483
1484  if ( $DEBUG_sys ) ; then
1485    echo "IGCM_sys_GetBuffer :" $@
1486  fi
1487  if [ $DRYRUN -le 2 ]; then
1488    if [ X${1} = X'/l' ] ; then
1489      # test if the first file is present in the old computation :
1490      eval set +A buf_liste \${${2}}
1491    else
1492      eval set +A buf_liste ${1}
1493    fi
1494    eval DEST=\${${#}}
1495
1496    #USUAL WAY
1497    if [ X${1} = X'/l' ] ; then
1498      (( RET=0 ))
1499      for target in ${buf_liste[*]} ; do
1500        local_file=$( basename ${target} )
1501        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1502        (( RET = RET + $? ))
1503      done
1504    else
1505      \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1506      RET=$?
1507    fi
1508
1509    if [ ${RET} -gt 0 ] ; then
1510      echo "IGCM_sys_GetBuffer : copy error."
1511      cat out_rsync
1512      IGCM_debug_Exit "IGCM_sys_GetBuffer"
1513    else
1514      \rm out_rsync
1515    fi
1516  else
1517    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1518  fi
1519  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1520}
1521
1522#D-#==================================================
1523#D-function IGCM_sys_GetDate_FichWork
1524#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1525#D-* Examples:
1526#D-
1527function IGCM_sys_GetDate_FichWork {
1528  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1529  if ( $DEBUG_sys ) ; then
1530    echo "IGCM_sys_GetDate_FichWork :" $@
1531  fi
1532  typeset dateF
1533  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1534  eval ${2}=${dateF[5]}
1535
1536    # donne la date filesys d'un fichier sur la machine work
1537  IGCM_debug_PopStack "IGCM_sys_FichWork"
1538}
1539
1540#D-#==================================================
1541#D-function IGCM_sys_GetDate_FichArchive
1542#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1543#D-* Examples:
1544#D-
1545function IGCM_sys_GetDate_FichArchive {
1546  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1547  if ( $DEBUG_sys ) ; then
1548    echo "IGCM_sys_GetDate_FichArchive :" $@
1549  fi
1550  typeset dateF
1551  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1552  eval ${2}=${dateF[5]}
1553
1554  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1555}
1556
1557#D-#==================================================
1558#D-function IGCM_sys_Dods_Rm
1559#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1560#D-* Examples:
1561#D-
1562function IGCM_sys_Dods_Rm {
1563  if ( $DEBUG_sys ) ; then
1564    echo "IGCM_sys_Dods_Rm :" $@
1565  fi
1566  typeset RET
1567  RET=0
1568  if [ $DRYRUN = 0 ]; then
1569
1570#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1571#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1572#      echo "Nothing has been done."
1573#      return
1574#    fi
1575
1576    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1577    RET=$?
1578
1579#       if [ ${RET} -gt 0 ] ; then
1580#           echo "IGCM_sys_Dods_Rm : error."
1581#           cat out_dods_rm
1582#           IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1583#       else
1584#           rm out_dods_rm
1585#       fi
1586
1587  else
1588    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1589  fi
1590  return $RET
1591}
1592
1593#D-#==================================================
1594#D-function IGCM_sys_Dods_Cp
1595#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1596#D-* Examples:
1597#D-
1598function IGCM_sys_Dods_Cp {
1599  if ( $DEBUG_sys ) ; then
1600    echo "IGCM_sys_Dods_Cp :" $@
1601  fi
1602  typeset RET
1603  RET=0
1604  if [ $DRYRUN = 0 ]; then
1605
1606#    if [ ! -d ${R_SAVE}/${1} ] ; then
1607#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1608#      echo "Nothing has been done."
1609#      return
1610#    fi
1611
1612    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1613    RET=$?
1614
1615#       if [ ${RET} -gt 0 ] ; then
1616#           echo "IGCM_sys_Dods_Cp : error."
1617#           cat out_dods_cp
1618#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1619#       else
1620#           rm out_dods_cp
1621#       fi
1622
1623  else
1624    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1625  fi
1626  return $RET
1627}
1628
1629#D-#==================================================
1630#D-function IGCM_sys_Put_Dods
1631#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1632#D-* Examples:
1633#D-
1634function IGCM_sys_Put_Dods {
1635  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1636  if ( $DEBUG_sys ) ; then
1637    echo "IGCM_sys_Put_Dods :" $@
1638  fi
1639  set -vx
1640  typeset RET
1641  if [ $DRYRUN = 0 ]; then
1642    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1643      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1644      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1645      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1646      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1647      return
1648    fi
1649
1650    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1651      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1652      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1653      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1654      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1655      return
1656    fi
1657    #
1658    if [ -d ${R_SAVE}/${1} ] ; then
1659      cd ${R_SAVE}
1660    elif [ -d ${R_FIGR}/${1} ] ; then
1661      cd ${R_FIGR}
1662    fi
1663
1664    IGCM_sys_Dods_Rm ${1}
1665    IGCM_sys_Dods_Cp ${1}
1666    RET=0
1667
1668    if [ ${RET} -gt 0 ] ; then
1669      echo "IGCM_sys_Put_Dods : error."
1670      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1671    fi
1672  else
1673    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1674  fi
1675  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1676}
1677
1678##############################################################
1679# REBUILD OPERATOR
1680
1681function IGCM_sys_rebuild {
1682  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1683  if ( $DEBUG_sys ) ; then
1684    echo "IGCM_sys_rebuild :" $@
1685  fi
1686  /home/cont003/p86ipsl/X64/bin/rebuild -f -o $@
1687  if [ $? -gt 0 ] ; then
1688    echo "IGCM_sys_rebuild : erreur ${@}."
1689    IGCM_debug_Exit "rebuild"
1690  fi
1691
1692  IGCM_debug_PopStack "IGCM_sys_rebuild"
1693}
1694
1695############################################################
1696# Activate Running Environnment Variables
1697
1698function IGCM_sys_activ_variables {
1699  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1700  if ( $DEBUG_sys ) ; then
1701    echo "IGCM_sys_activ_variables"
1702  fi
1703
1704  ulimit -s unlimited
1705
1706  export KMP_STACKSIZE=3g
1707  export KMP_LIBRARY=turnaround
1708  export MKL_SERIAL=YES
1709  unset MPI_PLACING_MODE
1710
1711  # add module load nb of threads for core placement
1712  (( max_omp = 0 ))
1713
1714  for comp in ${config_ListOfComponents[*]} ; do
1715    eval ExeNameIn=\${config_Executable_${comp}[0]}
1716    eval ExeNameOut=\${config_Executable_${comp}[1]}
1717
1718    # Only if we really have an executable for the component :
1719    if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1720      eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1721      if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
1722        (( max_omp = comp_proc_omp_loc ))
1723      fi
1724    fi
1725  done
1726
1727  ( [ X${max_omp} = X4 ] || [ X${max_omp} = X8 ] ) &&  { module load openmp/${max_omp}thds ; IGCM_debug_Print 1 "module load openmp/${max_omp}thds" ; }
1728
1729  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1730}
1731
1732############################################################
1733# Desactivate Running Environnment Variables
1734
1735function IGCM_sys_desactiv_variables {
1736  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1737  if ( $DEBUG_sys ) ; then
1738    echo "IGCM_sys_desactiv_variables"
1739  fi
1740  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1741}
1742
1743############################################################
1744# Build MPI/OMP scripts run file (dummy function)
1745
1746function IGCM_sys_build_run_file {
1747
1748IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1749
1750}
1751
1752############################################################
1753# Build MPI/OMP scripts
1754function IGCM_sys_build_execution_scripts
1755{
1756  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1757  if ( $DEBUG_sys ) ; then
1758    echo "IGCM_sys_build_execution_scripts " $@
1759  fi
1760  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
1761  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1762  typeset num_corempi nombre_restant_node nombre_restant_comp
1763  typeset IGCM_OMP
1764
1765  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1766    IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1767  fi
1768
1769  if ( ${OK_PARA_MPMD} ) ; then
1770
1771    if [ -f run_file ] ; then
1772      IGCM_sys_Rm -f run_file
1773    fi
1774    touch run_file
1775
1776    if ( ${OK_PARA_OMP} ) ; then
1777
1778      #  Hosts treatment
1779
1780      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
1781
1782      i=0
1783      rm -f hosts
1784      IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
1785      for nodes in `cat hosts.tmp` ; do
1786        host[$i]=$nodes
1787        echo "${host[$i]} slots=8 max_slots=8" >> hosts
1788        IGCM_debug_Print 1 ${host[$i]}
1789        i=$((i+1))
1790      done
1791      rm -f hosts.tmp
1792
1793      listnodes=${host[*]}
1794
1795      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
1796
1797      # Initialisation
1798
1799      init_node=y
1800      node_num_current=0
1801      start_num=0
1802      init_exec=n
1803
1804      # Test : if oasis is there, we put it at the first position
1805
1806      for comp in ${config_ListOfComponents[*]} ; do
1807
1808        if [ "X${comp}" = "XCPL" ]  ; then
1809
1810          eval ExeNameIn=\${config_Executable_${comp}[0]}
1811          eval ExeNameOut=\${config_Executable_${comp}[1]}
1812
1813          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1814          echo ""  >> script_${ExeNameOut}.ksh
1815          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1816          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1817          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1818          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
1819          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1820
1821          init_node=n
1822
1823          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
1824          node_num_current=0
1825          node_current=${host[${node_num_current}]}
1826
1827          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1828
1829          init_exec=y
1830          start_num=1
1831
1832        fi
1833
1834      done
1835
1836      # Then loop on the components (except for oasis)
1837
1838      for comp in ${config_ListOfComponents[*]} ; do
1839
1840        eval ExeNameIn=\${config_Executable_${comp}[0]}
1841        eval ExeNameOut=\${config_Executable_${comp}[1]}
1842
1843        # Only if we really have an executable for the component :
1844        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
1845
1846          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1847          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1848
1849          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1850          # echo "set -vx" >> script_${ExeNameOut}.ksh
1851          echo ""  >> script_${ExeNameOut}.ksh
1852          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1853          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1854          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1855          echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1856          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1857          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1858          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1859
1860          node_num=0
1861
1862          # We define the number of MPI process to be assigned for the component
1863
1864          nombre_restant_comp=${comp_proc_mpi_loc}
1865
1866          # Loop on the allocated nodes
1867
1868          for node in ${listnodes} ; do
1869
1870            # We go to the current node
1871            if [ ${node_num} = ${node_num_current} ] ; then
1872
1873              node_current=${host[${node_num_current}]}
1874
1875              # If first time on the node : initialisation
1876
1877              if [ ${init_node} = y ] ; then
1878                nombre_restant_node=${NUM_COREPERNODE}
1879              fi
1880
1881              # Test on the number of OMP threads
1882
1883              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1884                (( node_num = node_num + 1 ))
1885                node_num_current=${node_num}
1886                init_node=y
1887                continue
1888              fi
1889
1890              # Number of MPI process to assign
1891
1892              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1893
1894              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1895                num_corempi=${nombre_restant_comp}
1896              fi
1897
1898              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1899              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1900
1901              if [ ${init_exec} = y ] ; then
1902                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1903              else
1904                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1905                init_exec=y
1906              fi
1907
1908              ((  start_num = num_corempi + start_num ))
1909
1910            else
1911
1912              (( node_num = node_num + 1 ))
1913              continue
1914            fi
1915
1916            # Test on the number of core/process remaining on the node/component
1917
1918            if [ ${nombre_restant_node} = 0 ] ; then
1919              (( node_num = node_num + 1 ))
1920              node_num_current=${node_num}
1921              init_node=y
1922
1923              if [ ${nombre_restant_comp} = 0 ] ; then
1924                break 1
1925              fi
1926            else
1927
1928              node_num_current=${node_num}
1929              init_node=n
1930
1931              if [ ${nombre_restant_comp} = 0 ] ; then
1932                break 1
1933              fi
1934            fi
1935          done
1936        fi
1937      done
1938
1939    else
1940
1941        # Then first loop on the components for the coupler ie oasis
1942
1943        ## the coupler ie oasis must be the first one
1944        for comp in ${config_ListOfComponents[*]} ; do
1945
1946            eval ExeNameOut=\${config_Executable_${comp}[1]}
1947
1948        # for CPL component only
1949            if [ "X${comp}" = "XCPL" ] ; then
1950                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1951                echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1952            fi
1953        done
1954
1955        # Then second loop on the components
1956
1957        for comp in ${config_ListOfComponents[*]} ; do
1958
1959            eval ExeNameOut=\${config_Executable_${comp}[1]}
1960
1961            # Only if we really have an executable for the component and not the coupler ie oasis:
1962            if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1963                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1964                echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1965            fi
1966        done
1967        IGCM_sys_Chmod u+x run_file
1968
1969        EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1970
1971    fi
1972
1973  else # Only one executable.
1974      # Check has to be added with :
1975      # Job :
1976      # #MSUB -n 8 # reservation des processeurs pour le job
1977      # #MSUB -N 4 # reservation des nodes pour le job
1978      # BATCH_NUM_PROC_TOT=32
1979      #
1980      # config.card :
1981      # ATM= (gcm.e, lmdz.x, 8MPI, 4OMP)
1982      #
1983      # tested with :
1984      # Job :
1985      # #MSUB -n 32
1986      # BATCH_NUM_PROC_TOT=32
1987      #
1988      # config.card :
1989      # ATM= (gcm.e, lmdz.x)
1990
1991
1992      for comp in ${config_ListOfComponents[*]} ; do
1993
1994          # Only if we really have an executable for the component :
1995          eval ExeNameOut=\${config_Executable_${comp}[1]}
1996###       if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1997          if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1998
1999              IGCM_OMP=1
2000
2001              if ( ${OK_PARA_OMP} ) ; then
2002                  eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2003                  export OMP_NUM_THREADS=${comp_proc_omp_loc}
2004                  IGCMG_OMP=${comp_proc_omp_loc}
2005              fi
2006              if  ( ${OK_PARA_MPI} ) ; then
2007                  eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2008                  # Default : mpirun used if nb_proc gt 1
2009                  # pour sortie out/err par process ? more test required
2010                  # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
2011                  EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
2012              else
2013                  # Check if BATCH_NUM_PROC_TOT = IGCMG_OMP
2014                  if ( [  ${IGCM_OMP} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
2015                      IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> ${comp}_PROC_OMP : ${BATCH_NUM_PROC_TOT} <> ${BRIDGE_MSUB_NPROC} "
2016                      IGCM_debug_Print 1 "sys Titane : Check #MSUB -n xx into Job"
2017                      IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
2018                  fi
2019                  # Default : mpirun is NOT used if nb_proc eq 1
2020                  # pour sortie out/err par process ? more test required
2021                  # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2022                  EXECUTION="/usr/bin/time ./${ExeNameOut}"
2023              fi
2024          fi
2025      done
2026
2027  fi
2028
2029  IGCM_debug_Print 1 "sys Titane : La commande d execution est "
2030  IGCM_debug_Print 1 "$EXECUTION"
2031
2032  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
2033}
2034
2035############################################################
2036# Check of space available on temporary filesytems
2037function IGCM_sys_check_quota {
2038  IGCM_debug_PushStack "IGCM_sys_check_quota"
2039  if ( $DEBUG_sys ) ; then
2040    echo "IGCM_sys_check_quota "
2041  fi
2042  # Limit of quota (in %)
2043  limit_quota=90
2044
2045  # Check of the volume
2046  volume_quota=$(ccc_quota | grep ' scratch' | awk '{print $2}')
2047  volume_avail=$(ccc_quota | grep ' scratch' | awk '{print $3}')
2048
2049  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2050
2051    unit_avail=${volume_avail: -1}
2052    unit_quota=${volume_quota: -1}
2053
2054    if [ "${unit_quota}" = "*" ] ; then
2055        IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2056        IGCM_debug_Print 1 "More than 100% of your quota is used"
2057        IGCM_debug_Print 1 "Use the ccc_quota command to check"
2058        IGCM_debug_Print 1 "You must have more than 10% available to run"
2059        IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2060        IGCM_debug_Verif_Exit
2061    fi
2062
2063    temp_avail=${volume_avail%%${volume_avail: -1}*}
2064    temp_quota=${volume_quota%%${volume_quota: -1}*}
2065
2066    if [ ! ${unit_avail} = ${unit_quota} ] ; then
2067
2068    # Convertion
2069      if [ ${volume_avail: -1} = "T" ] ; then
2070        (( temp_avail = temp_avail * 1000000000000 ))
2071      elif [ ${volume_avail: -1} = "G" ] ; then
2072        (( temp_avail = temp_avail * 1000000000 ))
2073      elif [ ${volume_avail: -1} = "M" ] ; then
2074        (( temp_avail = temp_avail * 1000000 ))
2075      elif [ ${volume_avail: -1} = "k" ] ; then
2076        (( temp_avail = temp_avail * 1000 ))
2077      else
2078        (( temp_avail = volume_avail ))
2079      fi
2080      if [ ${volume_quota: -1} = "T" ] ; then
2081        (( temp_quota = temp_quota * 1000000000000 ))
2082      elif [ ${volume_quota: -1} = "G" ] ; then
2083        (( temp_quota = temp_quota * 1000000000 ))
2084      elif [ ${volume_quota: -1} = "M" ] ; then
2085        (( temp_quota = temp_quota * 1000000 ))
2086      elif [ ${volume_quota: -1} = "k" ] ; then
2087        (( temp_quota = temp_quota * 1000 ))
2088      else
2089        (( temp_quota = volume_quota ))
2090      fi
2091    fi
2092
2093    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
2094#    echo "volume ratio is " $quota_volume
2095
2096    if [ ${quota_volume} -ge ${limit_quota} ] ; then
2097      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2098      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2099      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2100      IGCM_debug_Print 1 "You must have more than 10% available to run"
2101      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2102      IGCM_debug_Verif_Exit
2103    fi
2104
2105  fi
2106
2107# Check of the number of inodes
2108
2109  inode_quota=$(ccc_quota | grep ' scratch' | awk '{print $6}')
2110  inode_avail=$(ccc_quota | grep ' scratch' | awk '{print $7}')
2111
2112  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2113
2114    unit_avail=${inode_avail: -1}
2115    unit_quota=${inode_quota: -1}
2116
2117    if [ "${unit_quota}" = "*" ] ; then
2118        IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2119        IGCM_debug_Print 1 "More than 100% of your quota is used"
2120        IGCM_debug_Print 1 "Use the ccc_quota command to check"
2121        IGCM_debug_Print 1 "You must have more than 10% available to run"
2122        IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2123        IGCM_debug_Verif_Exit
2124    fi
2125
2126    temp_avail=${inode_avail%%${inode_avail: -1}*}
2127    temp_quota=${inode_quota%%${inode_quota: -1}*}
2128
2129    if [ ! ${unit_avail} = ${unit_quota} ] ; then
2130
2131    # Convertion
2132      if [ ${inode_avail: -1} = "T" ] ; then
2133        (( temp_avail = temp_avail * 1000000000000 ))
2134      elif [ ${inode_avail: -1} = "G" ] ; then
2135        (( temp_avail = temp_avail * 1000000000 ))
2136      elif [ ${inode_avail: -1} = "M" ] ; then
2137        (( temp_avail = temp_avail * 1000000 ))
2138      elif [ ${inode_avail: -1} = "k" ] ; then
2139        (( temp_avail = temp_avail * 1000 ))
2140      else
2141        (( temp_avail = inode_avail ))
2142      fi
2143
2144      if [ ${inode_quota: -1} = "T" ] ; then
2145        (( temp_quota = temp_quota * 1000000000000 ))
2146      elif [ ${inode_quota: -1} = "G" ] ; then
2147        (( temp_quota = temp_quota * 1000000000 ))
2148      elif [ ${inode_quota: -1} = "M" ] ; then
2149        (( temp_quota = temp_quota * 1000000 ))
2150      elif [ ${inode_quota: -1} = "k" ] ; then
2151        (( temp_quota = temp_quota * 1000 ))
2152      else
2153        (( temp_quota = inode_quota ))
2154      fi
2155    fi
2156    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
2157#    echo "inode ratio is " $quota_inode
2158
2159    if [ ${quota_inode} -ge ${limit_quota} ] ; then
2160      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2161      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2162      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2163      IGCM_debug_Print 1 "You must have more than 10% available to run"
2164      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2165      IGCM_debug_Verif_Exit
2166    fi
2167
2168  fi
2169  IGCM_debug_PopStack "IGCM_sys_check_quota"
2170}
2171
2172##############################################################
2173# NCO OPERATOR
2174
2175function IGCM_sys_ncap2 {
2176  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2177  if ( $DEBUG_sys ) ; then
2178    echo "IGCM_sys_ncap2 :" $@
2179  fi
2180  ncap2 "$@"
2181  if [ $? -gt 0 ] ; then
2182    echo "IGCM_sys_ncap2 : erreur ${@}."
2183    IGCM_debug_Exit "ncap2"
2184  fi
2185
2186  IGCM_debug_PopStack "IGCM_sys_ncap2"
2187}
2188
2189function IGCM_sys_ncatted {
2190  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2191  if ( $DEBUG_sys ) ; then
2192    echo "IGCM_sys_ncatted :" $@
2193  fi
2194  ncatted "$@"
2195  if [ $? -gt 0 ] ; then
2196    echo "IGCM_sys_ncatted : erreur ${@}."
2197    IGCM_debug_Exit "ncatted"
2198  fi
2199
2200  IGCM_debug_PopStack "IGCM_sys_ncatted"
2201}
2202
2203function IGCM_sys_ncbo {
2204  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2205  if ( $DEBUG_sys ) ; then
2206    echo "IGCM_sys_ncbo :" $@
2207  fi
2208  ncbo $@
2209  if [ $? -gt 0 ] ; then
2210    echo "IGCM_sys_ncbo : erreur ${@}."
2211    IGCM_debug_Exit "ncbo"
2212  fi
2213
2214  IGCM_debug_PopStack "IGCM_sys_ncbo"
2215}
2216
2217function IGCM_sys_ncdiff {
2218  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2219  if ( $DEBUG_sys ) ; then
2220    echo "IGCM_sys_ncdiff :" $@
2221  fi
2222  ncdiff $@
2223  if [ $? -gt 0 ] ; then
2224    echo "IGCM_sys_ncdiff : erreur ${@}."
2225    IGCM_debug_Exit "ncdiff"
2226  fi
2227
2228  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2229}
2230
2231function IGCM_sys_ncea {
2232  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2233  if ( $DEBUG_sys ) ; then
2234    echo "IGCM_sys_ncea :" $@
2235  fi
2236  ncea $@
2237  if [ $? -gt 0 ] ; then
2238    echo "IGCM_sys_ncea : erreur ${@}."
2239    IGCM_debug_Exit "ncea"
2240  fi
2241
2242  IGCM_debug_PopStack "IGCM_sys_ncea"
2243}
2244
2245function IGCM_sys_ncecat {
2246  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2247  if ( $DEBUG_sys ) ; then
2248    echo "IGCM_sys_ncecat :" $@
2249  fi
2250  ncecat $@
2251  if [ $? -gt 0 ] ; then
2252    echo "IGCM_sys_ncecat : erreur ${@}."
2253    IGCM_debug_Exit "ncecat"
2254  fi
2255
2256  IGCM_debug_PopStack "IGCM_sys_ncecat"
2257}
2258
2259function IGCM_sys_ncflint {
2260  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2261  if ( $DEBUG_sys ) ; then
2262    echo "IGCM_sys_ncflint :" $@
2263  fi
2264  ncflint $@
2265  if [ $? -gt 0 ] ; then
2266    echo "IGCM_sys_ncflint : erreur ${@}."
2267    IGCM_debug_Exit "ncflint"
2268  fi
2269
2270  IGCM_debug_PopStack "IGCM_sys_ncflint"
2271}
2272
2273function IGCM_sys_ncks {
2274  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2275  if ( $DEBUG_sys ) ; then
2276    echo "IGCM_sys_ncks :" $@
2277  fi
2278  ncks $@
2279  if [ $? -gt 0 ] ; then
2280    echo "IGCM_sys_ncks : erreur ${@}."
2281    IGCM_debug_Exit "ncks"
2282  fi
2283
2284  IGCM_debug_PopStack "IGCM_sys_ncks"
2285}
2286
2287function IGCM_sys_ncpdq {
2288  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2289  if ( $DEBUG_sys ) ; then
2290    echo "IGCM_sys_ncpdq :" $@
2291  fi
2292  ncpdq $@
2293  if [ $? -gt 0 ] ; then
2294    echo "IGCM_sys_ncpdq : erreur ${@}."
2295    IGCM_debug_Exit "ncpdq"
2296  fi
2297
2298  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2299}
2300
2301function IGCM_sys_ncra {
2302  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2303  if ( $DEBUG_sys ) ; then
2304    echo "IGCM_sys_ncra :" $@
2305  fi
2306  ncra $@
2307  if [ $? -gt 0 ] ; then
2308    echo "IGCM_sys_ncra : erreur ${@}."
2309    IGCM_debug_Exit "ncra"
2310  fi
2311
2312  IGCM_debug_PopStack "IGCM_sys_ncra"
2313}
2314
2315function IGCM_sys_ncrcat {
2316  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2317  if ( $DEBUG_sys ) ; then
2318    echo "IGCM_sys_ncrcat :" $@
2319  fi
2320  ncrcat --omp_num_threads 2 $@
2321  if [ $? -gt 0 ] ; then
2322    echo "IGCM_sys_ncrcat : erreur ${@}."
2323#       IGCM_debug_Exit "ncrcat"
2324  fi
2325
2326  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2327}
2328
2329function IGCM_sys_ncrename {
2330  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2331  if ( $DEBUG_sys ) ; then
2332    echo "IGCM_sys_ncrename :" $@
2333  fi
2334  ncrename $@
2335  if [ $? -gt 0 ] ; then
2336    echo "IGCM_sys_ncrename : erreur ${@}."
2337    IGCM_debug_Exit "ncrename"
2338  fi
2339
2340  IGCM_debug_PopStack "IGCM_sys_ncrename"
2341}
2342
2343function IGCM_sys_ncwa {
2344  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2345  if ( $DEBUG_sys ) ; then
2346    echo "IGCM_sys_ncwa :" $@
2347  fi
2348  ncwa $@
2349  if [ $? -gt 0 ] ; then
2350    echo "IGCM_sys_ncwa : erreur ${@}."
2351    IGCM_debug_Exit "ncwa"
2352  fi
2353
2354  IGCM_debug_PopStack "IGCM_sys_ncwa"
2355}
2356
2357##############################################################
2358# CDO OPERATOR
2359
2360function IGCM_sys_cdo {
2361  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2362  if ( $DEBUG_sys ) ; then
2363    echo "IGCM_sys_cdo :" $@
2364  fi
2365  \cdo $@
2366  if [ $? -gt 0 ] ; then
2367    echo "IGCM_sys_cdo : erreur ${@}."
2368    IGCM_debug_PopStack "IGCM_sys_cdo"
2369    return 1
2370  else
2371    IGCM_debug_PopStack "IGCM_sys_cdo"
2372    return 0
2373  fi
2374
2375  IGCM_debug_PopStack "IGCM_sys_cdo"
2376}
Note: See TracBrowser for help on using the repository browser.