source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh @ 654

Last change on this file since 654 was 654, checked in by aclsce, 12 years ago

Added the print of quota values when the quota limit is reached.

  • Property svn:keywords set to Revision Author Date
File size: 65.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Titane
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe param files |            |  Chmod  |                           |
46# |          |      Qsub           |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=titane
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# rsync with path
78typeset -r RSYNC=/usr/bin/rsync
79# RSYNC_opt args to rsync
80typeset -r RSYNC_opt="-va"
81# ie storage filesystem
82typeset -r STOREHOST=${MASTER}
83
84#====================================================
85# Source default environment
86#====================================================
87. /etc/profile
88
89#====================================================
90# Set environment tools (ferret, nco, cdo)
91#====================================================
92. /home/cont003/p86ipsl/.atlas_env_titane_ksh
93if ( [ X${LSB_QUEUE} = Xmono ] || [ X${LSB_QUEUE} = Xmonoext ] ) ; then
94  module switch nco/3.9.4 nco/3.9.4_netcdf4
95fi
96
97#====================================================
98# Specific for ocean additionnal diagnostic
99export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
100export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
101
102#====================================================
103# Host specific DIRECTORIES
104#====================================================
105
106# ============ CESIUM START ============ #
107
108#====================================================
109#- Mirror libIGCM from titane to cesium if needed
110#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
111#if [ ! ${ROOTSYS} = "home" ] ; then
112#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
113#else
114#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
115#fi
116
117#====================================================
118#- libIGCM_POST
119#if ( ${MirrorlibIGCM} ) ; then
120#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
121#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
122#else
123#  typeset -r libIGCM_POST=${libIGCM}
124#fi
125
126# ============ CESIUM  END  ============ #
127
128#====================================================
129#- MirrorlibIGCM for frontend
130typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
131
132#====================================================
133#- libIGCM_POST for frontend
134typeset -r libIGCM_POST=${libIGCM}
135
136#====================================================
137#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
138typeset -r R_EXE="${MODIPSL}/bin"
139
140#====================================================
141#- SUBMIT_DIR : submission dir
142typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
143
144#====================================================
145#- ARCHIVE (dedicated to large files)
146typeset -r ARCHIVE=${CCCSTOREDIR}
147
148#- ARCHIVE (dedicated to small/medium files)
149typeset -r STORAGE=${CCCWORKDIR}
150
151#====================================================
152#- IN
153typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
154typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
155
156#====================================================
157#- R_OUT
158typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
159
160#====================================================
161#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
162typeset -r R_FIG=${STORAGE}/IGCM_OUT
163
164#====================================================
165#- R_BUF (ONLY FOR double copy an scratch)
166typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
167
168#====================================================
169#- BIG_DIR : BIG_DIR to store files waiting for rebuild
170typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
171
172#====================================================
173#- OUT_POST
174typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
175
176#====================================================
177#- RUN_DIR_PATH : Temporary working directory (=> TMP)
178if ( [ ! X${LSB_QUEUE} = Xmono ] && [ ! X${LSB_QUEUE} = Xmonoext ] ) ; then
179  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
180else
181  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
182fi
183
184#====================================================
185#- HOST_MPIRUN_COMMAND
186typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
187
188#====================================================
189#- Max number of arguments passed to nco operator or demigration command
190UNIX_MAX_LIMIT=360
191
192#====================================================
193#- set PackDefault true on titane
194PackDefault=true
195
196NUM_COREPERNODE=8
197
198#====================================================
199#- Default number of MPI task for IPSL coupled model
200#- required for backward compatibility
201#-
202DEFAULT_NUM_PROC_OCE=5
203DEFAULT_NUM_PROC_CPL=1
204(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
205DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
206
207
208#D-#==================================================
209#D-function IGCM_sys_RshMaster
210#D-* Purpose: Connection to frontend machine.
211#D-* Examples:
212#D-
213function IGCM_sys_RshMaster {
214  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
215  /bin/ksh <<-EOF
216    export libIGCM=${libIGCM}
217    export DEBUG_debug=${DEBUG_debug}
218    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
219    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
220    ${@}
221EOF
222  if [ $? -gt 0 ] ; then
223    echo "IGCM_sys_RshMaster : erreur."
224    IGCM_debug_Exit "IGCM_sys_RshMaster"
225  fi
226  IGCM_debug_PopStack "IGCM_sys_RshMaster"
227}
228
229#D-#==================================================
230#D-function IGCM_sys_RshArchive
231#D-* Purpose: Archive rsh command
232#D-* Examples:
233#D-
234function IGCM_sys_RshArchive {
235  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
236  /bin/ksh <<-EOF
237    ${@}
238EOF
239  if [ $? -gt 0 ] ; then
240    echo "IGCM_sys_RshArchive : erreur."
241    IGCM_debug_Exit "IGCM_sys_RshArchive"
242  fi
243  IGCM_debug_PopStack "IGCM_sys_RshArchive"
244}
245
246#D-#==================================================
247#D-function IGCM_sys_RshPost
248#D-* Purpose: Post-process rsh command
249#D-* Examples:
250#D-
251function IGCM_sys_RshPost {
252  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
253  if ( $DEBUG_sys ) ; then
254    echo "IGCM_sys_RshPost :" $@
255  fi
256
257  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
258  #cat tmp_IGCM_sys_RshPost_$$
259  # keep standard input (stdin) for the loop onto temporary file
260  cat >tmp_IGCM_sys_RshPost_$$
261
262# ============ FRONTEND START ============ #
263
264  /bin/ksh <tmp_IGCM_sys_RshPost_$$
265  if [ $? -gt 0 ] ; then
266    echo "IGCM_sys_RshPost : erreur."
267    IGCM_debug_Exit "IGCM_sys_RshPost"
268  fi
269  # delete temporary file
270  \rm tmp_IGCM_sys_RshPost_$$
271
272# ============ FRONTEND  END  ============ #
273
274# ============ CESIUM START ============ #
275#  typeset NB_ESSAI DELAI status i
276#  if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
277#    #little hack so that rebuild submission is done on titane not an cesium
278#
279#    libIGCM_POST_sed=$( echo $libIGCM_POST | sed 's/\//\\\//g' )
280#    POST_DIR_sed=$( echo ${POST_DIR} | sed 's/\//\\\//g' )
281#    sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
282#    sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
283#    sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
284#    sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
285#    \mv tmp.txt tmp_IGCM_sys_RshPost_$$
286#
287#    echo cat tmp_IGCM_sys_RshPost_$$ AFTER
288#    cat tmp_IGCM_sys_RshPost_$$
289#
290#    /bin/ksh <tmp_IGCM_sys_RshPost_$$
291#    if [ $? -gt 0 ] ; then
292#      echo "IGCM_sys_RshPost : erreur."
293#      IGCM_debug_Exit "IGCM_sys_RshPost"
294#    fi
295#    # delete temporary file
296#    \rm tmp_IGCM_sys_RshPost_$$
297#
298#  else
299#    # number of tentative
300#    NB_ESSAI=10
301#    # time delay between tentative
302#    DELAI=10
303#    i=0
304#    while [ $i -ne $NB_ESSAI ] ; do
305#      ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
306#      status=$?
307#      if [ ${status} -ne 0 ];
308#      then
309#       sleep $DELAI
310#      else
311#       break
312#      fi
313#      let i=$i+1
314#    done
315#    # delete temporary file
316#    \rm tmp_IGCM_sys_RshPost_$$
317#
318#    if [ ${status} -gt 0 ] ; then
319#      echo "IGCM_sys_RshPost : erreur."
320#      IGCM_debug_Exit "IGCM_sys_RshPost"
321#    fi
322#  fi
323
324# ============ CESIUM  END  ============ #
325
326  IGCM_debug_PopStack "IGCM_sys_RshPost"
327}
328
329#D-#==================================================
330#D-function IGCM_sys_SendMail
331#D-* Purpose: Send mail when simulation is over
332#D-* Examples:
333#D-
334function IGCM_sys_SendMail {
335  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
336  if ( $DEBUG_sys ) ; then
337    echo "IGCM_sys_SendMail :" $@
338  fi
339
340  if ( ${ExitFlag} ) ; then
341    status=failed
342  else
343    status=completed
344  fi
345
346    cat  << END_MAIL > job_end.mail
347Dear ${LOGIN},
348
349  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
350  Job started : ${DateBegin}
351  Job ended   : ${DateEnd}
352  Output files are available in ${R_SAVE}
353  Files to be rebuild are temporarily available in ${REBUILD_DIR}
354  Pre-packed files are temporarily available in ${R_BUFR}
355  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
356END_MAIL
357
358    if [ ! -z ${config_UserChoices_MailName} ] ; then
359        mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
360    elif [ -f ~/.forward ] ; then
361        mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
362    fi
363    sleep 10
364    rm -f job_end.mail
365
366  if [ $? -gt 0 ] ; then
367    echo "IGCM_sys_SendMail : erreur."
368    IGCM_debug_Exit "IGCM_sys_SendMail"
369  fi
370  IGCM_debug_PopStack "IGCM_sys_SendMail"
371}
372
373#D-#==================================================
374#D-function IGCM_sys_Mkdir
375#D-* Purpose: Master locale mkdir command
376#D-* Examples:
377#D-
378function IGCM_sys_Mkdir {
379  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
380  if ( $DEBUG_sys ) ; then
381    echo "IGCM_sys_Mkdir :" $@
382  fi
383  if [ ! -d ${1} ]; then
384    \mkdir -p $1
385    if [ $? -gt 0 ] ; then
386      echo "IGCM_sys_Mkdir : erreur."
387      IGCM_debug_Exit "IGCM_sys_Mkdir"
388    fi
389  fi
390  # vérification :
391  if [ ! -d ${1} ] ; then
392    echo "IGCM_sys_Mkdir : erreur."
393    IGCM_debug_Exit "IGCM_sys_Mkdir"
394  fi
395  IGCM_debug_PopStack "IGCM_sys_Mkdir"
396}
397
398#D-#==================================================
399#D-function IGCM_sys_MkdirArchive
400#D-* Purpose: Mkdir on Archive
401#D-* Examples:
402#D-
403function IGCM_sys_MkdirArchive {
404  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
405  if ( $DEBUG_sys ) ; then
406    echo "IGCM_sys_MkdirArchive :" $@
407  fi
408  #- creation de repertoire sur le serveur fichier
409  if [ ! -d ${1} ]; then
410    \mkdir -p $1
411    if [ $? -gt 0 ] ; then
412      echo "IGCM_sys_MkdirArchive : erreur."
413      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
414    fi
415  fi
416  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
417}
418
419#D-#==================================================
420#D-function IGCM_sys_MkdirWork
421#D-* Purpose: Mkdir on Work
422#D-* Examples:
423#D-
424function IGCM_sys_MkdirWork {
425  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
426  if ( $DEBUG_sys ) ; then
427    echo "IGCM_sys_MkdirWork :" $@
428  fi
429  #- creation de repertoire sur le serveur fichier
430  if [ ! -d ${1} ]; then
431    \mkdir -p $1
432    if [ $? -gt 0 ] ; then
433      echo "IGCM_sys_MkdirWork : erreur."
434      IGCM_debug_Exit "IGCM_sys_MkdirWork"
435    fi
436  fi
437  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
438}
439
440#D-#==================================================
441#D-function IGCM_sys_Cd
442#D-* Purpose: master cd command
443#D-* Examples:
444#D-
445function IGCM_sys_Cd {
446  IGCM_debug_PushStack "IGCM_sys_Cd" $@
447  if ( $DEBUG_sys ) ; then
448    echo "IGCM_sys_Cd :" $@
449  fi
450  \cd $1
451  if [ $? -gt 0 ] ; then
452    echo "IGCM_sys_Cd : erreur."
453    IGCM_debug_Exit "IGCM_sys_Cd"
454  fi
455  IGCM_debug_PopStack "IGCM_sys_Cd"
456}
457
458#D-#==================================================
459#D-function IGCM_sys_Chmod
460#D-* Purpose: Chmod
461#D-* Examples:
462#D-
463function IGCM_sys_Chmod {
464  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
465  if ( $DEBUG_sys ) ; then
466    echo "IGCM_sys_Chmod :" $@
467  fi
468  if [ $DRYRUN -le 1 ]; then
469    \chmod $@
470    if [ $? -gt 0 ] ; then
471      echo "IGCM_sys_Chmod : erreur."
472      IGCM_debug_Exit "IGCM_sys_Chmod"
473    fi
474  else
475    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
476  fi
477  IGCM_debug_PopStack "IGCM_sys_Chmod"
478}
479
480#D-#==================================================
481#D-function IGCM_sys_FileSize
482#D-* Purpose: Filesize
483#D-* Examples:
484#D-
485function IGCM_sys_FileSize {
486  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
487
488  typeset sizeF
489  set +A sizeF -- $( ls -la ${1} )
490  if [ $? -gt 0 ] ; then
491    IGCM_debug_Exit "IGCM_sys_FileSize"
492  fi
493  eval ${2}=${sizeF[4]}
494
495  IGCM_debug_PopStack "IGCM_sys_FileSize"
496}
497
498#D-#==================================================
499#D-function IGCM_sys_TestDir
500#D-* Purpose: Test Directory that must exists
501#D-* Examples:
502#D-
503function IGCM_sys_TestDir {
504  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
505  if ( $DEBUG_sys ) ; then
506    echo "IGCM_sys_TestDir :" $@
507  fi
508  typeset ExistFlag
509  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
510  IGCM_debug_PopStack "IGCM_sys_TestDir"
511
512  return ${ExistFlag}
513}
514
515#D-#==================================================
516#D-function IGCM_sys_TestDirArchive
517#D-* Purpose: Test Directory that must exists on Archive
518#D-* Examples:
519#D-
520function IGCM_sys_TestDirArchive {
521  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
522  if ( $DEBUG_sys ) ; then
523    echo "IGCM_sys_TestDirArchive :" $@
524  fi
525  typeset ExistFlag
526  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
527  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
528
529  return ${ExistFlag}
530}
531
532#D-#==================================================
533#D-function IGCM_sys_IsFileArchived
534#D-* Purpose: Test file that must NOT EXISTS on Archive
535#D-* Examples:
536#D-
537function IGCM_sys_IsFileArchived {
538  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
539  if ( $DEBUG_sys ) ; then
540    echo "IGCM_sys_IsFileArchived :" $@
541  fi
542  typeset IsArchivedFlag
543  IsArchivedFlag=$( [ X$( echo $1 | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
544  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
545
546  return ${IsArchivedFlag}
547}
548
549#D-#==================================================
550#D-function IGCM_sys_TestFileArchive
551#D-* Purpose: Test file that must NOT EXISTS on Archive
552#D-* Examples:
553#D-
554function IGCM_sys_TestFileArchive {
555  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
556  typeset ExistFlag
557  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
558  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
559
560  return ${ExistFlag}
561}
562
563#D-#==================================================
564#D-function IGCM_sys_TestFileBuffer
565#D-* Purpose: Test file that must NOT EXISTS on Buffer
566#D-* Examples:
567#D-
568function IGCM_sys_TestFileBuffer {
569  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
570  typeset ExistFlag
571  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
572  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
573
574  return ${ExistFlag}
575}
576
577#D-#==================================================
578#D-function IGCM_sys_CountFileArchive
579#D-* Purpose: Count files on Archive filesystem
580#D-* Examples:
581#D-
582function IGCM_sys_CountFileArchive {
583  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
584  ls ${@} 2>/dev/null | wc -l
585  if [ $? -gt 0 ] ; then
586    echo "IGCM_sys_CountFileArchive : erreur."
587  fi
588  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
589}
590
591#D-#==================================================
592#D-function IGCM_sys_CountFileBuffer
593#D-* Purpose: Count files on Scratch filesystem
594#D-* Examples:
595#D-
596function IGCM_sys_CountFileBuffer {
597  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
598  ls ${@} 2>/dev/null | wc -l
599  if [ $? -gt 0 ] ; then
600    echo "IGCM_sys_CountFileBuffer : erreur."
601  fi
602  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
603}
604
605#D-#==================================================
606#D-function IGCM_sys_Tree
607#D-* Purpose: Tree directories with files on ${ARCHIVE}
608#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
609#D-
610function IGCM_sys_Tree {
611  IGCM_debug_PushStack "IGCM_sys_Tree" $@
612  if ( $DEBUG_sys ) ; then
613    echo "IGCM_sys_Tree :" $@
614  fi
615
616  \ls -lR ${@}
617
618  IGCM_debug_PopStack "IGCM_sys_Tree"
619}
620
621#D-#==================================================
622#D-function IGCM_sys_Tar
623#D-* Purpose: master tar command
624#D-* Examples:
625#D-
626function IGCM_sys_Tar {
627  IGCM_debug_PushStack "IGCM_sys_Tar" $@
628  if ( $DEBUG_sys ) ; then
629    echo "IGCM_sys_Tar :" $@
630  fi
631  \tar cf $@
632  if [ $? -gt 0 ] ; then
633    echo "IGCM_sys_Tar : erreur."
634    IGCM_debug_Exit "IGCM_sys_Tar"
635  fi
636  IGCM_debug_PopStack "IGCM_sys_Tar"
637}
638
639#D-#==================================================
640#D-function IGCM_sys_UnTar
641#D-* Purpose: master un-tar command
642#D-* Examples:
643#D-
644function IGCM_sys_UnTar {
645  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
646  if ( $DEBUG_sys ) ; then
647    echo "IGCM_sys_UnTar :" $@
648  fi
649  \tar xvf $1
650  if [ $? -gt 0 ] ; then
651    echo "IGCM_sys_UnTar : erreur."
652    IGCM_debug_Exit "IGCM_sys_UnTar"
653  fi
654  IGCM_debug_PopStack "IGCM_sys_UnTar"
655}
656
657#D-#==================================================
658#D-function IGCM_sys_Qsub
659#D-* Purpose: Qsub new job
660#D-* Examples:
661#D-
662function IGCM_sys_Qsub {
663  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
664  if ( $DEBUG_sys ) ; then
665    echo "IGCM_sys_Qsub :" $@
666  fi
667  typeset options
668  if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
669    options=" -o ${Script_Output} -e ${Script_Output}"
670  else
671    options=" -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
672  fi
673
674  /usr/local/bin/ccc_msub ${options} < $1
675  if [ $? -gt 0 ] ; then
676    echo "IGCM_sys_Qsub : erreur ${options} $1"
677    IGCM_debug_Exit "IGCM_sys_Qsub"
678  fi
679  IGCM_debug_PopStack "IGCM_sys_Qsub"
680}
681
682#D-#==================================================
683#D-function IGCM_sys_QsubPost
684#D-* Purpose: Qsub new job on scalaire
685#D-* Examples:
686#D-
687function IGCM_sys_QsubPost {
688  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
689  if ( $DEBUG_sys ) ; then
690    echo "IGCM_sys_QsubPost :" $@
691  fi
692
693  # Need to export listVarEnv for cesium
694  # NO Need to export listVarEnv for titane
695# ============ FRONTEND START ============ #
696
697  /usr/local/bin/ccc_msub -p ${BRIDGE_MSUB_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
698
699# ============ FRONTEND  END  ============ #
700
701# ============ CESIUM START ============ #
702#  typeset NB_ESSAI DELAI status i
703#  # number of tentative
704#  NB_ESSAI=10
705#  # time delay between tentative
706#  DELAI=10
707#  (( i = 0 ))
708#  while [ $i -lt $NB_ESSAI ] ; do
709#    /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
710#    status=$?
711#    if [ ${status} -gt 0 ]; then
712#      sleep $DELAI
713#    else
714#      break
715#    fi
716#    (( i = i + 1 ))
717#  done
718# ============ CESIUM  END  ============ #
719
720  if [ $? -gt 0 ] ; then
721    echo "IGCM_sys_QsubPost : erreur " $@
722    IGCM_debug_Exit "IGCM_sys_QsubPost"
723  fi
724  IGCM_debug_PopStack "IGCM_sys_QsubPost"
725}
726
727#D-*************************
728#D- File transfer functions
729#D-*************************
730#D-
731
732#D-#==================================================
733#D-function IGCM_sys_Rsync_out
734#D-* Purpose: treat return val of rsync
735#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
736#D-  Error values and explanations can depend on your system version.
737function IGCM_sys_Rsync_out {
738  RET=$1
739  if [ ! $RET ] ; then
740    echo "rsync error !"
741  fi
742
743  if [ $MYLANG = "fr" ]; then
744    case $RET in
745    0)  return ;;
746    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
747      echo "Erreur de syntaxe ou d'utilisation."
748      return;;
749    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
750      echo "Incompatibilité de protocole."
751      return;;
752    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
753      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
754      echo "répertoires"
755      return;;
756    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
757      echo "Action demandée non supportée : une tentative de manipulation de"
758      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
759      echo "été faite ; ou une option qui est supportée par le  client  mais"
760      echo "pas par le serveur a été spécifiée."
761      return;;
762    10) echo "Erreur de rsync ; RERR_SOCKETIO"
763      echo "Erreur dans le socket d'entrée sortie"
764      return;;
765    11) echo "Erreur de rsync ; RERR_FILEIO"
766      echo "Erreur d'entrée sortie fichier"
767      return;;
768    12) echo "Erreur de rsync ; RERR_STREAMIO"
769      echo "Erreur dans flux de donnée du protocole rsync"
770      return;;
771    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
772      echo "Erreur avec les diagnostics du programme"
773      return;;
774    14) echo "Erreur de rsync ; RERR_IPC"
775      echo "Erreur dans le code IPC"
776      return;;
777    20) echo "Erreur de rsync ; RERR_SIGNAL"
778      echo "SIGUSR1 ou SIGINT reçu"
779      return;;
780    21) echo "Erreur de rsync ; RERR_WAITCHILD"
781      echo "Une erreur retournée par waitpid()"
782      return;;
783    22) echo "Erreur de rsync ; RERR_MALLOC"
784      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
785      return;;
786    23) echo ""
787      echo "Erreur fichier inexistant"
788      return;;
789    30) echo "Erreur de rsync ; RERR_TIMEOUT"
790      echo "Temps d'attente écoulé dans l'envoi/réception de données"
791      return;;
792    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
793      return;;
794    esac
795  elif [ $MYLANG = "en" ] ; then
796    case $RET in
797    0)  return;;
798    1)  echo "rsync error : Syntax or usage error "
799      return;;
800    2)  echo "rsync error : Protocol incompatibility "
801      return;;
802    3)  echo "rsync error : Errors selecting input/output files, dirs"
803      return;;
804    4)  echo "rsync error : Requested action not supported: an attempt"
805      echo "was made to manipulate 64-bit files on a platform that cannot support"
806      echo "them; or an option was specified that is supported by the client and"
807      echo "not by the server."
808      return;;
809    5)  echo "rsync error : Error starting client-server protocol"
810      return;;
811    10) echo "rsync error : Error in socket I/O "
812      return;;
813    11) echo "rsync error : Error in file I/O "
814      return;;
815    12) echo "rsync error : Error in rsync protocol data stream "
816      return;;
817    13) echo "rsync error : Errors with program diagnostics "
818      return;;
819    14) echo "rsync error : Error in IPC code "
820      return;;
821    20) echo "rsync error : Received SIGUSR1 or SIGINT "
822      return;;
823    21) echo "rsync error : Some error returned by waitpid() "
824      return;;
825    22) echo "rsync error : Error allocating core memory buffers "
826      return;;
827    23) echo "rsync error : Partial transfer due to error"
828      return;;
829    24) echo "rsync error : Partial transfer due to vanished source files"
830      return;;
831    30) echo "rsync error : Timeout in data send/receive "
832      return;;
833    *)  echo "rsync error : return code of rsync unknown :" $RET
834      return;;
835    esac
836  else
837    echo "unknown language $MYLANG."
838    return
839  fi
840}
841
842#D-#==================================================
843#D-function IGCM_sys_Miror_libIGCM
844#D-* Purpose: Mirror libIGCM PATH and lib to cesium
845#D-* Examples:
846#D-
847function IGCM_sys_Mirror_libIGCM {
848  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
849  if ( $DEBUG_sys ) ; then
850    echo "IGCM_sys_Mirror_libIGCM"
851  fi
852
853  typeset RET DEST
854
855  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
856
857  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
858  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
859  RET=$?
860
861  if [ ${RET} -gt 0 ] ; then
862    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
863    cat out_rsync
864  fi
865  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
866}
867
868#====================================================
869#- Call IGCM_sys_Mirror_libIGCM now !
870if ( $MirrorlibIGCM ) ; then
871  IGCM_sys_Mirror_libIGCM
872fi
873
874#D-#==================================================
875#D-function IGCM_sys_Cp
876#D-* Purpose: generic cp
877#D-* Examples:
878#D-
879function IGCM_sys_Cp {
880  IGCM_debug_PushStack "IGCM_sys_Cp" $@
881  if ( $DEBUG_sys ) ; then
882    echo "IGCM_sys_Cp :" $@
883  fi
884
885  typeset RET
886
887  echo cp $@ > out_rsync 2>&1
888  \cp $@ >> out_rsync 2>&1
889  RET=$?
890
891  if [ ${RET} -gt 0 ] ; then
892    echo "IGCM_sys_Cp : error."
893    cat out_rsync
894    IGCM_debug_Exit "IGCM_sys_Cp"
895  else
896    \rm out_rsync
897  fi
898  IGCM_debug_PopStack "IGCM_sys_Cp"
899}
900
901#D-#==================================================
902#D-function IGCM_sys_Rm
903#D-* Purpose: generic rm
904#D-* Examples:
905#D-
906function IGCM_sys_Rm {
907  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
908  if ( $DEBUG_sys ) ; then
909    echo "IGCM_sys_Rm :" $@
910  fi
911
912  typeset RET
913
914  echo rm $@ > out_rsync 2>&1
915  \rm $@ >> out_rsync 2>&1
916  RET=$?
917
918  if [ ${RET} -gt 0 ] ; then
919    echo "IGCM_sys_Rm : error."
920    cat out_rsync
921    IGCM_debug_Exit "IGCM_sys_Rm"
922  else
923    \rm out_rsync
924  fi
925  IGCM_debug_PopStack "IGCM_sys_Rm"
926}
927
928#D-#==================================================
929#D-function IGCM_sys_RmRunDir
930#D-* Purpose: rm tmpdir (dummy function most of the time batch
931#D-                      scheduler will do the job)
932#D-* Examples:
933#D-
934function IGCM_sys_RmRunDir {
935  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
936  if ( $DEBUG_sys ) ; then
937    echo "IGCM_sys_RmRunDir :" $@
938  fi
939
940  typeset RET
941
942  echo rm $@ > out_rsync 2>&1
943  \rm $@ >> out_rsync 2>&1
944  RET=$?
945
946  if [ ${RET} -gt 0 ] ; then
947    echo "IGCM_sys_RmRunDir : error."
948    cat out_rsync
949    IGCM_debug_Exit "IGCM_sys_RmRunDir"
950  else
951    \rm out_rsync
952  fi
953  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
954}
955
956#D-#==================================================
957#D-function IGCM_sys_Mv
958#D-* Purpose: generic move
959#D-* Examples:
960#D-
961function IGCM_sys_Mv {
962  IGCM_debug_PushStack "IGCM_sys_Mv" $@
963  if ( $DEBUG_sys ) ; then
964    echo "IGCM_sys_Mv :" $@
965  fi
966
967  if [ $DRYRUN = 0 ]; then
968
969    typeset RET
970
971    echo mv $@ > out_rsync 2>&1
972    \mv $@ >> out_rsync 2>&1
973    RET=$?
974
975    if [ ${RET} -gt 0 ] ; then
976      echo "IGCM_sys_Mv : error in mv."
977      cat out_rsync
978      IGCM_debug_Exit "IGCM_sys_Mv"
979    else
980      \rm out_rsync
981    fi
982  else
983    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
984  fi
985
986  IGCM_debug_PopStack "IGCM_sys_Mv"
987}
988
989#D-#==================================================
990#D-function IGCM_sys_Put_Dir
991#D-* Purpose: Copy a complete directory on $(ARCHIVE)
992#D-* Examples:
993#D-
994function IGCM_sys_Put_Dir {
995  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
996  if ( $DEBUG_sys ) ; then
997    echo "IGCM_sys_Put_Dir :" $@
998  fi
999  if [ $DRYRUN = 0 ]; then
1000    if [ ! -d ${1} ] ; then
1001      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
1002      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1003      return
1004    fi
1005
1006    typeset RET
1007
1008    # Only if we use rsync
1009    #IGCM_sys_TestDirArchive $( dirname $2 )
1010    #
1011    #USUAL WAY
1012    \cp -r $1 $2 > out_rsync 2>&1
1013    RET=$?
1014
1015    if [ ${RET} -gt 0 ] ; then
1016      echo "IGCM_sys_Put_Dir : error."
1017      cat out_rsync
1018      IGCM_debug_Exit "IGCM_sys_Put_Dir"
1019    else
1020      \rm out_rsync
1021    fi
1022  else
1023    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1024  fi
1025  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1026}
1027
1028#D-#==================================================
1029#D-function IGCM_sys_Get_Dir
1030#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1031#D-* Examples:
1032#D-
1033function IGCM_sys_Get_Dir {
1034  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1035  if ( $DEBUG_sys ) ; then
1036    echo "IGCM_sys_Get_Dir :" $@
1037  fi
1038  if [ $DRYRUN = 0 ]; then
1039#    if [ ! -d ${1} ] ; then
1040#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1041#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1042#      return
1043#    fi
1044
1045    typeset RET
1046
1047    # Only if we use rsync
1048    #IGCM_sys_TestDirArchive $( dirname $2 )
1049    #
1050    # USUAL WAY
1051    # add dmfind/dmget (to demigrate all offline files) :
1052    #dmfind $1 -state MIG -o -state OFL -o -state PAR | dmget
1053    \cp -r $1 $2 > out_rsync 2>&1
1054    RET=$?
1055
1056    if [ ${RET} -gt 0 ] ; then
1057      echo "IGCM_sys_Get_Dir : error."
1058      cat out_rsync
1059      IGCM_debug_Exit "IGCM_sys_Get_Dir"
1060    else
1061      \rm out_rsync
1062    fi
1063  else
1064    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1065  fi
1066  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1067}
1068
1069#D-#==================================================
1070#D-function IGCM_sys_Get_Master
1071#D-* Purpose: Copy a complete directory from MASTER filesystem
1072#D-* Examples:
1073#D-
1074function IGCM_sys_Get_Master {
1075  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1076  if ( $DEBUG_sys ) ; then
1077    echo "IGCM_sys_Get_Master :" $@
1078  fi
1079  if [ $DRYRUN = 0 ]; then
1080    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1081      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1082      IGCM_debug_PopStack "IGCM_sys_Get_Master"
1083      return
1084    fi
1085
1086    typeset RET
1087
1088    #USUAL WAY
1089    \cp -r $1 $2 > out_rsync 2>&1
1090    RET=$?
1091
1092    if [ ${RET} -gt 0 ] ; then
1093      echo "IGCM_sys_Get_Master : error."
1094      cat out_rsync
1095      IGCM_debug_Exit "IGCM_sys_Get_Master"
1096    else
1097      \rm out_rsync
1098    fi
1099  else
1100    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1101  fi
1102  IGCM_debug_PopStack "IGCM_sys_Get_Master"
1103}
1104
1105#D-#==================================================
1106#D-function IGCM_sys_Put_Rest
1107#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1108#D-           File and target directory must exist.
1109#D-* Examples:
1110#D-
1111function IGCM_sys_Put_Rest {
1112  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1113  if ( $DEBUG_sys ) ; then
1114    echo "IGCM_sys_Put_Rest :" $@
1115  fi
1116  if [ $DRYRUN = 0 ]; then
1117    if [ ! -f ${1} ] ; then
1118      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1119      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1120    fi
1121
1122    typeset RET
1123    #
1124    if [ X${JobType} = XRUN ] ; then
1125      IGCM_sys_Chmod 444 ${1}
1126    fi
1127
1128    #
1129    # USUAL WAY
1130    \cp $1 $2 > out_rsync 2>&1
1131    RET=$?
1132
1133#       #RSYNC WITH NETWORK SSH CALL
1134#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1135#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1136
1137#       #RSYNC WITH NFS USE
1138#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1139#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1140
1141#       RET=$?
1142#       IGCM_sys_Rsync_out $RET
1143
1144#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1145#       (( RET=RET+$? ))
1146
1147    if [ ${RET} -gt 0 ] ; then
1148      echo "IGCM_sys_Put_Rest : error."
1149      cat out_rsync
1150      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1151    else
1152      \rm out_rsync
1153    fi
1154  else
1155    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1156  fi
1157  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1158}
1159
1160#D-#==================================================
1161#D-function IGCM_sys_PutBuffer_Rest
1162#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1163#D-           File and target directory must exist.
1164#D-* Examples:
1165#D-
1166function IGCM_sys_PutBuffer_Rest {
1167  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1168  if ( $DEBUG_sys ) ; then
1169    echo "IGCM_sys_PutBuffer_Rest :" $@
1170  fi
1171  if [ $DRYRUN = 0 ]; then
1172    if [ ! -f ${1} ] ; then
1173      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1174      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1175    fi
1176
1177    typeset RET
1178    #
1179    if [ X${JobType} = XRUN ] ; then
1180      IGCM_sys_Chmod 444 ${1}
1181    fi
1182
1183    #
1184    # USUAL WAY
1185    \cp $1 $2 > out_rsync 2>&1
1186    RET=$?
1187
1188    if [ ${RET} -gt 0 ] ; then
1189      echo "IGCM_sys_PutBuffer_Rest : error."
1190      cat out_rsync
1191      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1192    else
1193      \rm out_rsync
1194    fi
1195  else
1196    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1197  fi
1198  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1199}
1200
1201#D-#==================================================
1202#D-function IGCM_sys_Put_Out
1203#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1204#D-* Examples:
1205#D-
1206function IGCM_sys_Put_Out {
1207  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1208  if ( $DEBUG_sys ) ; then
1209    echo "IGCM_sys_Put_Out :" $@
1210  fi
1211  if [ $DRYRUN = 0 ]; then
1212    if [ ! -f ${1} ] ; then
1213      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1214      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1215      return 1
1216    fi
1217    #
1218    IGCM_sys_MkdirArchive $( dirname $2 )
1219    #
1220    typeset RET
1221
1222    #=====================================================
1223    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1224    #=====================================================
1225
1226    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
1227    #if [ $? -eq 0 ] ; then
1228    #    typeset WORKPATH FILEPATH
1229    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
1230    #    IGCM_sys_MkdirWork ${WORKPATH}
1231    #    FILEPATH=${WORKPATH}/$( basename $2 )
1232    #    #
1233    #    IGCM_sys_Cp ${1} ${FILEPATH}
1234    #fi
1235
1236    if [ X${JobType} = XRUN ] ; then
1237      if [ X${3} = X ] ; then
1238        IGCM_sys_Chmod 444 ${1}
1239      fi
1240    fi
1241    #
1242    # USUAL WAY
1243    \cp $1 $2 > out_rsync 2>&1
1244    RET=$?
1245
1246#       #RSYNC WITH NETWORK SSH CALL
1247#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1248#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1249
1250#       #RSYNC WITH NFS USE
1251#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1252#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1253
1254#       RET=$?
1255#       IGCM_sys_Rsync_out $RET
1256
1257#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1258#       (( RET=RET+$? ))
1259
1260    if [ ${RET} -gt 0 ] ; then
1261      echo "IGCM_sys_Put_Out : error."
1262      cat out_rsync
1263      IGCM_debug_Exit "IGCM_sys_Put_Out"
1264    else
1265      \rm out_rsync
1266    fi
1267  else
1268    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1269  fi
1270  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1271  return 0
1272}
1273
1274#D-#==================================================
1275#D-function IGCM_sys_PutBuffer_Out
1276#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1277#D-* Examples:
1278#D-
1279function IGCM_sys_PutBuffer_Out {
1280  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1281  if ( $DEBUG_sys ) ; then
1282    echo "IGCM_sys_PutBuffer_Out :" $@
1283  fi
1284  if [ $DRYRUN = 0 ]; then
1285    if [ ! -f ${1} ] ; then
1286      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1287      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1288      return 1
1289    fi
1290    #
1291    IGCM_sys_Mkdir $( dirname $2 )
1292    #
1293    typeset RET
1294
1295    if [ X${JobType} = XRUN ] ; then
1296      if [ X${3} = X ] ; then
1297        IGCM_sys_Chmod 444 ${1}
1298      fi
1299    fi
1300    #
1301    # USUAL WAY
1302    \cp $1 $2 > out_rsync 2>&1
1303    RET=$?
1304
1305    if [ ${RET} -gt 0 ] ; then
1306      echo "IGCM_sys_PutBuffer_Out : error."
1307      cat out_rsync
1308      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1309    else
1310      \rm out_rsync
1311    fi
1312  else
1313    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1314  fi
1315  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1316  return 0
1317}
1318
1319#D-#==================================================
1320#D-function IGCM_sys_Get
1321#D-* Purpose: Get a file from ${ARCHIVE}
1322#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1323#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1324function IGCM_sys_Get {
1325  IGCM_debug_PushStack "IGCM_sys_Get" $@
1326
1327  typeset DEST RET dm_liste ifile target file_work
1328
1329  if ( $DEBUG_sys ) ; then
1330    echo "IGCM_sys_Get :" $@
1331  fi
1332  if [ $DRYRUN -le 2 ]; then
1333    if [ X${1} = X'/l' ] ; then
1334      # test if the first file is present in the old computation :
1335      eval set +A dm_liste \${${2}}
1336    else
1337      eval set +A dm_liste ${1}
1338    fi
1339    eval DEST=\${${#}}
1340
1341    #=====================================================
1342    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1343    #=====================================================
1344
1345    # Is it an R_OUT file (not R_IN) ?
1346    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
1347    #if [ $? -eq 0 ] ; then
1348    #    # Yes  ? then we try to get it in SCRATCHDIR
1349    #    set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
1350    #    if [ -f ${file_work[0]} ] ; then
1351    #   IGCM_sys_Cp ${file_work[*]} ${DEST}
1352    #   IGCM_debug_PopStack "IGCM_sys_Get"
1353    #   return
1354    #    fi
1355    #fi
1356
1357    # test if the (first) file is present in the old computation :
1358    IGCM_sys_TestFileArchive ${dm_liste[0]}
1359    RET=$?
1360    if [ ${RET} -gt 0 ] ; then
1361      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1362      IGCM_debug_Exit "IGCM_sys_Get"
1363    fi
1364
1365    #dmget ${dm_liste[*]} > out_rsync 2>&1
1366    ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1367    RET=$?
1368    if [ ${RET} -gt 0 ] ; then
1369      echo "WARNING IGCM_sys_Get : demigration error."
1370      cat out_rsync
1371      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1372    fi
1373
1374    #if [ ${RET} -gt 0 ] ; then
1375    #    if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1376    #   cat out_rsync
1377    #   echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1378    #   sleep 30
1379    #   echo "We try another time"
1380    ##  dmget ${dm_liste[*]} > out_rsync 2>&1
1381    #   ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1382    #   RET=$?
1383    #   if [ ${RET} -gt 0 ] ; then
1384    #       echo "ERROR IGCM_sys_Get : again demigration error :"
1385    #       cat out_rsync
1386    #       IGCM_debug_Exit "IGCM_sys_Get"
1387    #   fi
1388    #    else
1389    #   echo "ERROR IGCM_sys_Get : demigration error :"
1390    #   cat out_rsync
1391    #   IGCM_debug_Exit "IGCM_sys_Get"
1392    #    fi
1393    #fi
1394
1395    #USUAL WAY
1396    if [ X${1} = X'/l' ] ; then
1397      (( RET=0 ))
1398      for target in ${dm_liste[*]} ; do
1399        local_file=$( basename ${target} )
1400        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1401        (( RET = RET + $? ))
1402      done
1403    else
1404      \cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1405      RET=$?
1406    fi
1407
1408#       #RSYNC WITH NETWORK SSH CALL
1409#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1410#       ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1411
1412#       #RSYNC WITH NFS USE
1413#       echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1414#       ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1415
1416#       RET=$?
1417#       IGCM_sys_Rsync_out $RET
1418
1419#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1420#       (( RET=RET+$? ))
1421
1422    if [ ${RET} -gt 0 ] ; then
1423      echo "IGCM_sys_Get : copy error."
1424      cat out_rsync
1425      IGCM_debug_Exit "IGCM_sys_Get"
1426    else
1427      \rm out_rsync
1428    fi
1429  else
1430    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1431  fi
1432  IGCM_debug_PopStack "IGCM_sys_Get"
1433}
1434
1435#D-#==================================================
1436#D-function IGCM_sys_GetBuffer
1437#D-* Purpose: Get a file from ${SCRATCHDIR}
1438#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1439#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1440function IGCM_sys_GetBuffer {
1441  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1442
1443  typeset DEST RET buf_liste ifile target file_work
1444
1445  if ( $DEBUG_sys ) ; then
1446    echo "IGCM_sys_GetBuffer :" $@
1447  fi
1448  if [ $DRYRUN -le 2 ]; then
1449    if [ X${1} = X'/l' ] ; then
1450      # test if the first file is present in the old computation :
1451      eval set +A buf_liste \${${2}}
1452    else
1453      eval set +A buf_liste ${1}
1454    fi
1455    eval DEST=\${${#}}
1456
1457    #USUAL WAY
1458    if [ X${1} = X'/l' ] ; then
1459      (( RET=0 ))
1460      for target in ${buf_liste[*]} ; do
1461        local_file=$( basename ${target} )
1462        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1463        (( RET = RET + $? ))
1464      done
1465    else
1466      \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1467      RET=$?
1468    fi
1469
1470    if [ ${RET} -gt 0 ] ; then
1471      echo "IGCM_sys_GetBuffer : copy error."
1472      cat out_rsync
1473      IGCM_debug_Exit "IGCM_sys_GetBuffer"
1474    else
1475      \rm out_rsync
1476    fi
1477  else
1478    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1479  fi
1480  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1481}
1482
1483#D-#==================================================
1484#D-function IGCM_sys_GetDate_FichWork
1485#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1486#D-* Examples:
1487#D-
1488function IGCM_sys_GetDate_FichWork {
1489  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1490  if ( $DEBUG_sys ) ; then
1491    echo "IGCM_sys_GetDate_FichWork :" $@
1492  fi
1493  typeset dateF
1494  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1495  eval ${2}=${dateF[5]}
1496
1497    # donne la date filesys d'un fichier sur la machine work
1498  IGCM_debug_PopStack "IGCM_sys_FichWork"
1499}
1500
1501#D-#==================================================
1502#D-function IGCM_sys_GetDate_FichArchive
1503#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1504#D-* Examples:
1505#D-
1506function IGCM_sys_GetDate_FichArchive {
1507  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1508  if ( $DEBUG_sys ) ; then
1509    echo "IGCM_sys_GetDate_FichArchive :" $@
1510  fi
1511  typeset dateF
1512  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1513  eval ${2}=${dateF[5]}
1514
1515  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1516}
1517
1518#D-#==================================================
1519#D-function IGCM_sys_Dods_Rm
1520#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1521#D-* Examples:
1522#D-
1523function IGCM_sys_Dods_Rm {
1524  if ( $DEBUG_sys ) ; then
1525    echo "IGCM_sys_Dods_Rm :" $@
1526  fi
1527  typeset RET
1528  RET=0
1529  if [ $DRYRUN = 0 ]; then
1530
1531#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1532#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1533#      echo "Nothing has been done."
1534#      return
1535#    fi
1536
1537    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1538    RET=$?
1539
1540#       if [ ${RET} -gt 0 ] ; then
1541#           echo "IGCM_sys_Dods_Rm : error."
1542#           cat out_dods_rm
1543#           IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1544#       else
1545#           rm out_dods_rm
1546#       fi
1547
1548  else
1549    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1550  fi
1551  return $RET
1552}
1553
1554#D-#==================================================
1555#D-function IGCM_sys_Dods_Cp
1556#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1557#D-* Examples:
1558#D-
1559function IGCM_sys_Dods_Cp {
1560  if ( $DEBUG_sys ) ; then
1561    echo "IGCM_sys_Dods_Cp :" $@
1562  fi
1563  typeset RET
1564  RET=0
1565  if [ $DRYRUN = 0 ]; then
1566
1567#    if [ ! -d ${R_SAVE}/${1} ] ; then
1568#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1569#      echo "Nothing has been done."
1570#      return
1571#    fi
1572
1573    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1574    RET=$?
1575
1576#       if [ ${RET} -gt 0 ] ; then
1577#           echo "IGCM_sys_Dods_Cp : error."
1578#           cat out_dods_cp
1579#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1580#       else
1581#           rm out_dods_cp
1582#       fi
1583
1584  else
1585    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1586  fi
1587  return $RET
1588}
1589
1590#D-#==================================================
1591#D-function IGCM_sys_Put_Dods
1592#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1593#D-* Examples:
1594#D-
1595function IGCM_sys_Put_Dods {
1596  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1597  if ( $DEBUG_sys ) ; then
1598    echo "IGCM_sys_Put_Dods :" $@
1599  fi
1600  set -vx
1601  typeset RET
1602  if [ $DRYRUN = 0 ]; then
1603    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1604      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1605      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1606      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1607      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1608      return
1609    fi
1610
1611    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1612      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1613      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1614      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1615      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1616      return
1617    fi
1618    #
1619    if [ -d ${R_SAVE}/${1} ] ; then
1620      cd ${R_SAVE}
1621    elif [ -d ${R_FIGR}/${1} ] ; then
1622      cd ${R_FIGR}
1623    fi
1624
1625    IGCM_sys_Dods_Rm ${1}
1626    IGCM_sys_Dods_Cp ${1}
1627    RET=0
1628
1629    if [ ${RET} -gt 0 ] ; then
1630      echo "IGCM_sys_Put_Dods : error."
1631      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1632    fi
1633  else
1634    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1635  fi
1636  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1637}
1638
1639##############################################################
1640# REBUILD OPERATOR
1641
1642function IGCM_sys_rebuild {
1643  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1644  if ( $DEBUG_sys ) ; then
1645    echo "IGCM_sys_rebuild :" $@
1646  fi
1647  /home/cont003/p86ipsl/X64/bin/rebuild -f -o $@
1648  if [ $? -gt 0 ] ; then
1649    echo "IGCM_sys_rebuild : erreur ${@}."
1650    IGCM_debug_Exit "rebuild"
1651  fi
1652
1653  IGCM_debug_PopStack "IGCM_sys_rebuild"
1654}
1655
1656############################################################
1657# Activate Running Environnment Variables
1658
1659function IGCM_sys_activ_variables {
1660  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1661  if ( $DEBUG_sys ) ; then
1662    echo "IGCM_sys_activ_variables"
1663  fi
1664
1665  typeset max_omp
1666
1667  ulimit -s unlimited
1668
1669  (( max_omp = 0 ))
1670  for comp in ${config_ListOfComponents[*]} ; do
1671
1672    eval ExeNameIn=\${config_Executable_${comp}[0]}
1673    eval ExeNameOut=\${config_Executable_${comp}[1]}
1674
1675    # Only if we really have an executable for the component :
1676    if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
1677      eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1678      if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
1679        (( max_omp = comp_proc_omp_loc ))
1680      fi
1681    fi
1682  done
1683  if [ ${max_omp} -gt 1 ] ; then
1684    module load openmp/${max_omp}thds
1685  fi
1686
1687  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1688}
1689
1690############################################################
1691# Desactivate Running Environnment Variables
1692
1693function IGCM_sys_desactiv_variables {
1694  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1695  if ( $DEBUG_sys ) ; then
1696    echo "IGCM_sys_desactiv_variables"
1697  fi
1698  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1699}
1700
1701############################################################
1702# Build MPI/OMP scripts run file (dummy function)
1703
1704function IGCM_sys_build_run_file {
1705
1706IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1707
1708}
1709
1710############################################################
1711# Build MPI/OMP scripts
1712function IGCM_sys_build_execution_scripts
1713{
1714  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1715  if ( $DEBUG_sys ) ; then
1716    echo "IGCM_sys_build_execution_scripts " $@
1717  fi
1718  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
1719  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1720  typeset num_corempi nombre_restant_node nombre_restant_comp
1721
1722  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1723    IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1724  fi
1725
1726  if ( ${OK_PARA_MPMD} ) ; then
1727
1728    if [ -f run_file ] ; then
1729      IGCM_sys_Rm -f run_file
1730    fi
1731    touch run_file
1732
1733    if ( ${OK_PARA_OMP} ) ; then
1734
1735      #  Hosts treatment
1736
1737      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
1738
1739      i=0
1740      rm -f hosts
1741      IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
1742      for nodes in `cat hosts.tmp` ; do
1743        host[$i]=$nodes
1744        echo "${host[$i]} slots=8 max_slots=8" >> hosts
1745        IGCM_debug_Print 1 ${host[$i]}
1746        i=$((i+1))
1747      done
1748      rm -f hosts.tmp
1749
1750      listnodes=${host[*]}
1751
1752      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
1753
1754      # Initialisation
1755
1756      init_node=y
1757      node_num_current=0
1758      start_num=0
1759      init_exec=n
1760
1761      # Test : if oasis is there, we put it at the first position
1762
1763      for comp in ${config_ListOfComponents[*]} ; do
1764
1765        if [ "X${comp}" = "XCPL" ]  ; then
1766
1767          eval ExeNameIn=\${config_Executable_${comp}[0]}
1768          eval ExeNameOut=\${config_Executable_${comp}[1]}
1769
1770          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1771          echo ""  >> script_${ExeNameOut}.ksh
1772          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1773          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1774          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1775          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
1776          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1777
1778          init_node=n
1779
1780          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
1781          node_num_current=0
1782          node_current=${host[${node_num_current}]}
1783
1784          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1785
1786          init_exec=y
1787          start_num=1
1788
1789        fi
1790
1791      done
1792
1793      # Then loop on the components (except for oasis)
1794
1795      for comp in ${config_ListOfComponents[*]} ; do
1796
1797        eval ExeNameIn=\${config_Executable_${comp}[0]}
1798        eval ExeNameOut=\${config_Executable_${comp}[1]}
1799
1800        # Only if we really have an executable for the component :
1801        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
1802
1803          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1804          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1805
1806          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1807          # echo "set -vx" >> script_${ExeNameOut}.ksh
1808          echo ""  >> script_${ExeNameOut}.ksh
1809          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1810          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1811          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1812          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1813          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1814          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1815          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1816
1817          node_num=0
1818
1819          # We define the number of MPI process to be assigned for the component
1820
1821          nombre_restant_comp=${comp_proc_mpi_loc}
1822
1823          # Loop on the allocated nodes
1824
1825          for node in ${listnodes} ; do
1826
1827            # We go to the current node
1828            if [ ${node_num} = ${node_num_current} ] ; then
1829
1830              node_current=${host[${node_num_current}]}
1831
1832              # If first time on the node : initialisation
1833
1834              if [ ${init_node} = y ] ; then
1835                nombre_restant_node=${NUM_COREPERNODE}
1836              fi
1837
1838              # Test on the number of OMP threads
1839
1840              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1841                (( node_num = node_num + 1 ))
1842                node_num_current=${node_num}
1843                init_node=y
1844                continue
1845              fi
1846
1847              # Number of MPI process to assign
1848
1849              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1850
1851              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1852                num_corempi=${nombre_restant_comp}
1853              fi
1854
1855              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1856              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1857
1858              if [ ${init_exec} = y ] ; then
1859                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1860              else
1861                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1862                init_exec=y
1863              fi
1864
1865              ((  start_num = num_corempi + start_num ))
1866
1867            else
1868
1869              (( node_num = node_num + 1 ))
1870              continue
1871            fi
1872
1873            # Test on the number of core/process remaining on the node/component
1874
1875            if [ ${nombre_restant_node} = 0 ] ; then
1876              (( node_num = node_num + 1 ))
1877              node_num_current=${node_num}
1878              init_node=y
1879
1880              if [ ${nombre_restant_comp} = 0 ] ; then
1881                break 1
1882              fi
1883            else
1884
1885              node_num_current=${node_num}
1886              init_node=n
1887
1888              if [ ${nombre_restant_comp} = 0 ] ; then
1889                break 1
1890              fi
1891            fi
1892          done
1893        fi
1894      done
1895
1896    else
1897
1898        # Then first loop on the components for the coupler ie oasis
1899
1900        ## the coupler ie oasis must be the first one
1901        for comp in ${config_ListOfComponents[*]} ; do
1902
1903            eval ExeNameOut=\${config_Executable_${comp}[1]}
1904
1905        # for CPL component only
1906            if [ "X${comp}" = "XCPL" ] ; then
1907                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1908                echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1909            fi
1910        done
1911
1912        # Then second loop on the components
1913
1914        for comp in ${config_ListOfComponents[*]} ; do
1915
1916            eval ExeNameOut=\${config_Executable_${comp}[1]}
1917
1918            # Only if we really have an executable for the component and not the coupler ie oasis:
1919            if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1920                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1921                echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1922            fi
1923        done
1924        IGCM_sys_Chmod u+x run_file
1925
1926        EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1927
1928    fi
1929
1930  else # Only one executable. launch it.
1931
1932      for comp in ${config_ListOfComponents[*]} ; do
1933
1934          # Only if we really have an executable for the component :
1935          eval ExeNameOut=\${config_Executable_${comp}[1]}
1936          if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1937
1938              echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1939              echo ""  >> script_${ExeNameOut}.ksh
1940              if ( ${OK_PARA_OMP} ) ; then
1941                  eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1942                  echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1943              fi
1944              if  ( ${OK_PARA_MPI} ) ; then
1945                  # Default : mpirun used if nb_proc gt 1
1946                  # pour sortie out/err par process
1947                  # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
1948                  echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1949                  IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1950                  EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1951              else
1952                  # Default : mpirun is NOT used if nb_proc eq 1
1953                  # pour sortie out/err par process
1954                  # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1955                  echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1956                  IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1957                  EXECUTION="time ./script_${ExeNameOut}.ksh"
1958              fi
1959          fi
1960      done
1961
1962  fi
1963
1964  IGCM_debug_Print 1 "sys Titane : La commande d execution est "
1965  IGCM_debug_Print 1 $EXECUTION
1966
1967  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1968}
1969
1970############################################################
1971# Check of space available on temporary filesytems
1972function IGCM_sys_check_quota {
1973  IGCM_debug_PushStack "IGCM_sys_check_quota"
1974  if ( $DEBUG_sys ) ; then
1975    echo "IGCM_sys_check_quota "
1976  fi
1977  # Limit of quota (in %)
1978  limit_quota=90
1979
1980  # Check of the volume
1981  volume_quota=$(ccc_quota | grep ' scratch' | awk '{print $2}')
1982  volume_avail=$(ccc_quota | grep ' scratch' | awk '{print $3}')
1983
1984  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1985
1986    unit_avail=${volume_avail: -1}
1987    unit_quota=${volume_quota: -1}
1988
1989    temp_avail=${volume_avail%%${volume_avail: -1}*}
1990    temp_quota=${volume_quota%%${volume_quota: -1}*}
1991
1992    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1993
1994    # Convertion
1995      if [ ${volume_avail: -1} = "T" ] ; then
1996        (( temp_avail = temp_avail * 1000000000000 ))
1997      elif [ ${volume_avail: -1} = "G" ] ; then
1998        (( temp_avail = temp_avail * 1000000000 ))
1999      elif [ ${volume_avail: -1} = "M" ] ; then
2000        (( temp_avail = temp_avail * 1000000 ))
2001      elif [ ${volume_avail: -1} = "k" ] ; then
2002        (( temp_avail = temp_avail * 1000 ))
2003      else
2004        (( temp_avail = volume_avail ))
2005      fi
2006      if [ ${volume_quota: -1} = "T" ] ; then
2007        (( temp_quota = temp_quota * 1000000000000 ))
2008      elif [ ${volume_quota: -1} = "G" ] ; then
2009        (( temp_quota = temp_quota * 1000000000 ))
2010      elif [ ${volume_quota: -1} = "M" ] ; then
2011        (( temp_quota = temp_quota * 1000000 ))
2012      elif [ ${volume_quota: -1} = "k" ] ; then
2013        (( temp_quota = temp_quota * 1000 ))
2014      else
2015        (( temp_quota = volume_quota ))
2016      fi
2017    fi
2018
2019    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
2020#    echo "volume ratio is " $quota_volume
2021
2022    if [ ${quota_volume} -ge ${limit_quota} ] ; then
2023      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2024      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2025      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2026      IGCM_debug_Print 1 "You must have more than 10% available to run"
2027      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2028      IGCM_debug_Verif_Exit
2029    fi
2030
2031  fi
2032
2033# Check of the number of inodes
2034
2035  inode_quota=$(ccc_quota | grep ' scratch' | awk '{print $6}')
2036  inode_avail=$(ccc_quota | grep ' scratch' | awk '{print $7}')
2037
2038  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2039
2040    unit_avail=${inode_avail: -1}
2041    unit_quota=${inode_quota: -1}
2042
2043    temp_avail=${inode_avail%%${inode_avail: -1}*}
2044    temp_quota=${inode_quota%%${inode_quota: -1}*}
2045
2046    if [ ! ${unit_avail} = ${unit_quota} ] ; then
2047
2048    # Convertion
2049      if [ ${inode_avail: -1} = "T" ] ; then
2050        (( temp_avail = temp_avail * 1000000000000 ))
2051      elif [ ${inode_avail: -1} = "G" ] ; then
2052        (( temp_avail = temp_avail * 1000000000 ))
2053      elif [ ${inode_avail: -1} = "M" ] ; then
2054        (( temp_avail = temp_avail * 1000000 ))
2055      elif [ ${inode_avail: -1} = "k" ] ; then
2056        (( temp_avail = temp_avail * 1000 ))
2057      else
2058        (( temp_avail = inode_avail ))
2059      fi
2060
2061      if [ ${inode_quota: -1} = "T" ] ; then
2062        (( temp_quota = temp_quota * 1000000000000 ))
2063      elif [ ${inode_quota: -1} = "G" ] ; then
2064        (( temp_quota = temp_quota * 1000000000 ))
2065      elif [ ${inode_quota: -1} = "M" ] ; then
2066        (( temp_quota = temp_quota * 1000000 ))
2067      elif [ ${inode_quota: -1} = "k" ] ; then
2068        (( temp_quota = temp_quota * 1000 ))
2069      else
2070        (( temp_quota = inode_quota ))
2071      fi
2072    fi
2073    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
2074#    echo "inode ratio is " $quota_inode
2075
2076    if [ ${quota_inode} -ge ${limit_quota} ] ; then
2077      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2078      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2079      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2080      IGCM_debug_Print 1 "You must have more than 10% available to run"
2081      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2082      IGCM_debug_Verif_Exit
2083    fi
2084
2085  fi
2086  IGCM_debug_PopStack "IGCM_sys_check_quota"
2087}
2088
2089##############################################################
2090# NCO OPERATOR
2091
2092function IGCM_sys_ncap2 {
2093  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2094  if ( $DEBUG_sys ) ; then
2095    echo "IGCM_sys_ncap2 :" $@
2096  fi
2097  ncap2 "$@"
2098  if [ $? -gt 0 ] ; then
2099    echo "IGCM_sys_ncap2 : erreur ${@}."
2100    IGCM_debug_Exit "ncap2"
2101  fi
2102
2103  IGCM_debug_PopStack "IGCM_sys_ncap2"
2104}
2105
2106function IGCM_sys_ncatted {
2107  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2108  if ( $DEBUG_sys ) ; then
2109    echo "IGCM_sys_ncatted :" $@
2110  fi
2111  ncatted "$@"
2112  if [ $? -gt 0 ] ; then
2113    echo "IGCM_sys_ncatted : erreur ${@}."
2114    IGCM_debug_Exit "ncatted"
2115  fi
2116
2117  IGCM_debug_PopStack "IGCM_sys_ncatted"
2118}
2119
2120function IGCM_sys_ncbo {
2121  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2122  if ( $DEBUG_sys ) ; then
2123    echo "IGCM_sys_ncbo :" $@
2124  fi
2125  ncbo $@
2126  if [ $? -gt 0 ] ; then
2127    echo "IGCM_sys_ncbo : erreur ${@}."
2128    IGCM_debug_Exit "ncbo"
2129  fi
2130
2131  IGCM_debug_PopStack "IGCM_sys_ncbo"
2132}
2133
2134function IGCM_sys_ncdiff {
2135  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2136  if ( $DEBUG_sys ) ; then
2137    echo "IGCM_sys_ncdiff :" $@
2138  fi
2139  ncdiff $@
2140  if [ $? -gt 0 ] ; then
2141    echo "IGCM_sys_ncdiff : erreur ${@}."
2142    IGCM_debug_Exit "ncdiff"
2143  fi
2144
2145  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2146}
2147
2148function IGCM_sys_ncea {
2149  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2150  if ( $DEBUG_sys ) ; then
2151    echo "IGCM_sys_ncea :" $@
2152  fi
2153  ncea $@
2154  if [ $? -gt 0 ] ; then
2155    echo "IGCM_sys_ncea : erreur ${@}."
2156    IGCM_debug_Exit "ncea"
2157  fi
2158
2159  IGCM_debug_PopStack "IGCM_sys_ncea"
2160}
2161
2162function IGCM_sys_ncecat {
2163  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2164  if ( $DEBUG_sys ) ; then
2165    echo "IGCM_sys_ncecat :" $@
2166  fi
2167  ncecat $@
2168  if [ $? -gt 0 ] ; then
2169    echo "IGCM_sys_ncecat : erreur ${@}."
2170    IGCM_debug_Exit "ncecat"
2171  fi
2172
2173  IGCM_debug_PopStack "IGCM_sys_ncecat"
2174}
2175
2176function IGCM_sys_ncflint {
2177  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2178  if ( $DEBUG_sys ) ; then
2179    echo "IGCM_sys_ncflint :" $@
2180  fi
2181  ncflint $@
2182  if [ $? -gt 0 ] ; then
2183    echo "IGCM_sys_ncflint : erreur ${@}."
2184    IGCM_debug_Exit "ncflint"
2185  fi
2186
2187  IGCM_debug_PopStack "IGCM_sys_ncflint"
2188}
2189
2190function IGCM_sys_ncks {
2191  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2192  if ( $DEBUG_sys ) ; then
2193    echo "IGCM_sys_ncks :" $@
2194  fi
2195  ncks $@
2196  if [ $? -gt 0 ] ; then
2197    echo "IGCM_sys_ncks : erreur ${@}."
2198    IGCM_debug_Exit "ncks"
2199  fi
2200
2201  IGCM_debug_PopStack "IGCM_sys_ncks"
2202}
2203
2204function IGCM_sys_ncpdq {
2205  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2206  if ( $DEBUG_sys ) ; then
2207    echo "IGCM_sys_ncpdq :" $@
2208  fi
2209  ncpdq $@
2210  if [ $? -gt 0 ] ; then
2211    echo "IGCM_sys_ncpdq : erreur ${@}."
2212    IGCM_debug_Exit "ncpdq"
2213  fi
2214
2215  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2216}
2217
2218function IGCM_sys_ncra {
2219  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2220  if ( $DEBUG_sys ) ; then
2221    echo "IGCM_sys_ncra :" $@
2222  fi
2223  ncra $@
2224  if [ $? -gt 0 ] ; then
2225    echo "IGCM_sys_ncra : erreur ${@}."
2226    IGCM_debug_Exit "ncra"
2227  fi
2228
2229  IGCM_debug_PopStack "IGCM_sys_ncra"
2230}
2231
2232function IGCM_sys_ncrcat {
2233  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2234  if ( $DEBUG_sys ) ; then
2235    echo "IGCM_sys_ncrcat :" $@
2236  fi
2237  ncrcat --omp_num_threads 2 $@
2238  if [ $? -gt 0 ] ; then
2239    echo "IGCM_sys_ncrcat : erreur ${@}."
2240#       IGCM_debug_Exit "ncrcat"
2241  fi
2242
2243  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2244}
2245
2246function IGCM_sys_ncrename {
2247  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2248  if ( $DEBUG_sys ) ; then
2249    echo "IGCM_sys_ncrename :" $@
2250  fi
2251  ncrename $@
2252  if [ $? -gt 0 ] ; then
2253    echo "IGCM_sys_ncrename : erreur ${@}."
2254    IGCM_debug_Exit "ncrename"
2255  fi
2256
2257  IGCM_debug_PopStack "IGCM_sys_ncrename"
2258}
2259
2260function IGCM_sys_ncwa {
2261  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2262  if ( $DEBUG_sys ) ; then
2263    echo "IGCM_sys_ncwa :" $@
2264  fi
2265  ncwa $@
2266  if [ $? -gt 0 ] ; then
2267    echo "IGCM_sys_ncwa : erreur ${@}."
2268    IGCM_debug_Exit "ncwa"
2269  fi
2270
2271  IGCM_debug_PopStack "IGCM_sys_ncwa"
2272}
2273
2274##############################################################
2275# CDO OPERATOR
2276
2277function IGCM_sys_cdo {
2278  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2279  if ( $DEBUG_sys ) ; then
2280    echo "IGCM_sys_cdo :" $@
2281  fi
2282  \cdo $@
2283  if [ $? -gt 0 ] ; then
2284    echo "IGCM_sys_cdo : erreur ${@}."
2285    IGCM_debug_PopStack "IGCM_sys_cdo"
2286    return 1
2287  else
2288    IGCM_debug_PopStack "IGCM_sys_cdo"
2289    return 0
2290  fi
2291
2292  IGCM_debug_PopStack "IGCM_sys_cdo"
2293}
Note: See TracBrowser for help on using the repository browser.