source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1148

#!/bin/ksh

#**************************************************************
# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2006)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#=========================================================
# The documentation of this file can be automatically generated
# if you use the prefix #D- for comments to be extracted.
# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
#=========================================================

#D-#==================================================
#D-LibIGCM_sys for Curie
#D-#==================================================
#D-
#D- This ksh library if a layer under some usefull
#D-environment variables and shell commands.
#D-All those definitions depend on host particularities.
#D-It manages a stack mechanism and test validity of operations.
#D-All function described bellow must be prefixed by IGCM_sys.

#====================================================
# libIGCM_sys PARAMETERS
#====================================================

#====================================================
# set DEBUG_sys to true to output calls of function
typeset -r DEBUG_sys=${DEBUG_sys:=true}

#====================================================
# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
typeset -r DRYRUN=${DRYRUN:=0}

# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
# -------------------------------------------------------------------------------------
# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
# |          |  Cp/Exe/param/files |            |         |                           |
# |          |  Chmod Qsub         |            |         |                           |
# -------------------------------------------------------------------------------------
# |    0     |       yes           |    yes     |  yes    |      yes                  |
# -------------------------------------------------------------------------------------
# |    1     |       yes           |    yes     |  yes    |      no                   |
# -------------------------------------------------------------------------------------
# |    2     |       yes           |    yes     |  no     |      no                   |
# -------------------------------------------------------------------------------------
# |    3     |       yes           |    no      |  no     |      no                   |
# -------------------------------------------------------------------------------------

#=====================================================
# Global Variables :
#=====================================================
# Language : "fr" or "en"
typeset -r MYLANG="fr"

#=====================================================
# Host and user names
# $hostname ou hostname
typeset  HOST=${HOST:=$( hostname )}
# $username ou whoami
typeset  LOGIN=${LOGIN:=$( whoami )}
# $hostname of the MASTER job
typeset MASTER=curie
# add default project on curie
typeset DEFAULT_PROJECT=gen0826

#D-
#D-#==================================================
#D-Program used in libIGCM
#D-#==================================================

# Submit command
typeset SUBMIT=${SUBMIT:=ccc_msub}
# rsync with path
typeset -r RSYNC=/usr/bin/rsync
# RSYNC_opt args to rsync
typeset -r RSYNC_opt="-va"
# ie storage filesystem
typeset -r STOREHOST=${MASTER}

#====================================================
# Source default environment
#====================================================
##. /etc/profile

#====================================================
# Set environment tools (ferret, nco, cdo)
#====================================================
if [ X${TaskType} = Xcomputing ] ; then
  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4_curie_ksh > /dev/null 2>&1
# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
# uncomment 2 lines :
#  module unload netcdf
#  module load netcdf/3.6.3
  export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
  export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
else
  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4_curie_ksh > /dev/null 2>&1
fi

#====================================================
# Specific for ocean additionnal diagnostic
export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"

#====================================================
# Host specific DIRECTORIES
#====================================================

# ============ CESIUM START ============ #

#====================================================
#- Mirror libIGCM from titane to cesium if needed
#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
#if [ ! ${ROOTSYS} = "home" ] ; then
#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
#else
#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
#fi

#====================================================
#- libIGCM_POST
#if ( ${MirrorlibIGCM} ) ; then
#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
#else
#  typeset -r libIGCM_POST=${libIGCM}
#fi

# ============ CESIUM  END  ============ #

#====================================================
#- MirrorlibIGCM for frontend
typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}

#====================================================
#- libIGCM_POST for frontend
typeset -r libIGCM_POST=${libIGCM}

#====================================================
#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
typeset -r R_EXE="${MODIPSL}/bin"

#====================================================
#- SUBMIT_DIR : submission dir
typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}

#====================================================
#- IN
typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}

#====================================================
#- RUN_DIR_PATH : Temporary working directory (=> TMP)
typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}

#====================================================
#- HOST_MPIRUN_COMMAND
typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}

#====================================================
#- Max number of arguments passed to nco operator or demigration command
UNIX_MAX_LIMIT=360

#====================================================
#- set PackDefault true on curie
PackDefault=true

#====================================================
#- Number of core per node (max number of OpenMP task)
NUM_COREPERNODE=8

#====================================================
#- Default number of MPI task for IPSL coupled model
#- required for backward compatibility
#-
DEFAULT_NUM_PROC_OCE=5
DEFAULT_NUM_PROC_CPL=1
(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}

#D-#==================================================
#D-function IGCM_sys_defineArchives
#D-* Purpose:
#D-* Define ARCHIVE : Dedicated to large files
#D-* Define STORAGE : Dedicated to small/medium files
#D-* Define R_OUT   : Output tree located on ARCHIVE
#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
#D-* Examples:
#D-
function IGCM_sys_defineArchives {
  IGCM_debug_PushStack "IGCM_sys_defineArchives"

  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
    #====================================================
    #- ARCHIVE (dedicated to large files)
    ARCHIVE=${config_UserChoices_ARCHIVE}
  else
    #====================================================
    #- ARCHIVE (dedicated to large files)
    ARCHIVE=${CCCSTOREDIR}
  fi

  if [ ! X${config_UserChoices_STORAGE} = X ]; then
    #====================================================
    #- STORAGE (dedicated to small/medium files)
    STORAGE=${config_UserChoices_STORAGE}
  else
    #====================================================
    #- STORAGE (dedicated to small/medium files)
    STORAGE=${CCCWORKDIR}
  fi

  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
    #====================================================
    #- R_OUT
    R_OUT=${SCRATCHDIR}/IGCM_OUT

    #====================================================
    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
    R_FIG=${SCRATCHDIR}/IGCM_OUT

    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"

  else
    #====================================================
    #- R_OUT
    R_OUT=${ARCHIVE}/IGCM_OUT

    #====================================================
    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
    R_FIG=${STORAGE}/IGCM_OUT
  fi

  #====================================================
  #- R_BUF (ONLY FOR double copy an scratch)
  R_BUF=${SCRATCHDIR}/IGCM_OUT

  #====================================================
  #- R_TMP (point a space acting as /tmp)
  R_TMP=/tmp

  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"

  IGCM_debug_PopStack "IGCM_sys_defineArchives"
}

#D-#==================================================
#D-function IGCM_sys_RshMaster
#D-* Purpose: Connection to frontend machine.
#D-* Examples:
#D-
function IGCM_sys_RshMaster {
  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
  /bin/ksh <<-EOF
    export libIGCM=${libIGCM}
    export DEBUG_debug=${DEBUG_debug}
    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
    ${@}
EOF
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_RshMaster : erreur."
    IGCM_debug_Exit "IGCM_sys_RshMaster"
  fi
  IGCM_debug_PopStack "IGCM_sys_RshMaster"
}

#D-#==================================================
#D-function IGCM_sys_RshArchive
#D-* Purpose: Archive rsh command
#D-* Examples:
#D-
function IGCM_sys_RshArchive {
  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
  /bin/ksh <<-EOF
    ${@}
EOF
  status=$?
  if [ ${status} -gt 0 ] ; then
    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
    IGCM_debug_Exit "IGCM_sys_RshArchive"
  fi
  IGCM_debug_PopStack "IGCM_sys_RshArchive"
}

#D-#==================================================
#D-function IGCM_sys_RshArchive_NoError
#D-* Purpose: Archive rsh command, without error
#D-*          used only in monitoring.job
#D-* Examples:
#D-
function IGCM_sys_RshArchive_NoError {
  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
  /bin/ksh <<-EOF
    ${@} 2> \dev\null
EOF
  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
}

#D-#==================================================
#D-function IGCM_sys_RshPost
#D-* Purpose: Post-process rsh command
#D-* Examples:
#D-
function IGCM_sys_RshPost {
  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_RshPost :" $@
  fi
  # keep standard input (stdin) for the loop onto temporary file
  cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}

  /bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_RshPost : erreur."
    IGCM_debug_Exit "IGCM_sys_RshPost"
  fi
  # delete temporary file
  \rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}

# ============ FRONTEND  END  ============ #

# ============ CESIUM START ============ #
#  typeset NB_ESSAI DELAI status i
#  if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
#    #little hack so that rebuild submission is done on titane not an cesium
#
#    libIGCM_POST_sed=$( echo $libIGCM_POST | sed 's/\//\\\//g' )
#    POST_DIR_sed=$( echo ${POST_DIR} | sed 's/\//\\\//g' )
#    sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
#    sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
#    sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
#    sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
#    \mv tmp.txt tmp_IGCM_sys_RshPost_$$
#
#    echo cat tmp_IGCM_sys_RshPost_$$ AFTER
#    cat tmp_IGCM_sys_RshPost_$$
#
#    /bin/ksh <tmp_IGCM_sys_RshPost_$$
#    if [ $? -gt 0 ] ; then
#      echo "IGCM_sys_RshPost : erreur."
#      IGCM_debug_Exit "IGCM_sys_RshPost"
#    fi
#    # delete temporary file
#    \rm tmp_IGCM_sys_RshPost_$$
#
#  else
#    # number of tentative
#    NB_ESSAI=10
#    # time delay between tentative
#    DELAI=10
#    i=0
#    while [ $i -ne $NB_ESSAI ] ; do
#      ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
#      status=$?
#      if [ ${status} -ne 0 ];
#      then
#        sleep $DELAI
#      else
#        break
#      fi
#      let i=$i+1
#    done
#    # delete temporary file
#    \rm tmp_IGCM_sys_RshPost_$$
#
#    if [ ${status} -gt 0 ] ; then
#      echo "IGCM_sys_RshPost : erreur."
#      IGCM_debug_Exit "IGCM_sys_RshPost"
#    fi
#  fi

# ============ CESIUM  END  ============ #

  IGCM_debug_PopStack "IGCM_sys_RshPost"
}

#D-#==================================================
#D-function IGCM_sys_SendMail
#D-* Purpose: Send mail when simulation is over
#D-* Examples:
#D-
function IGCM_sys_SendMail {
  IGCM_debug_PushStack "IGCM_sys_SendMail" $@

  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_SendMail :" $@
  fi

  if [ X${1} = XAccounting ] ; then
    status=Accounting
    mailText=jobAccounting.mail
  elif ( ${ExitFlag} ) ; then
    status=failed
    mailText=jobEnd.mail
  else
    status=completed
    mailText=jobEnd.mail
  fi

  # Update selected mail template
  while read -r line; do
    eval echo $line >> mail.txt ;
  done < ${libIGCM}/libIGCM_sys/${mailText}

  if [ ! -z ${config_UserChoices_MailName} ] ; then
    mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < mail.txt
  elif [ -f ~/.forward ] ; then
    mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
  fi

  sleep 10
  rm -f mail.txt

  IGCM_debug_PopStack "IGCM_sys_SendMail"
}

#D-#==================================================
#D-function IGCM_sys_Mkdir
#D-* Purpose: Master locale mkdir command
#D-* Examples:
#D-
function IGCM_sys_Mkdir {
  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Mkdir :" $@
  fi
  if [ ! -d ${1} ]; then
    \mkdir -p $1
    if [ $? -gt 0 ] ; then
      echo "IGCM_sys_Mkdir : erreur."
      IGCM_debug_Exit "IGCM_sys_Mkdir"
    fi
  fi
  # vérification :
  if [ ! -d ${1} ] ; then
    echo "IGCM_sys_Mkdir : erreur."
    IGCM_debug_Exit "IGCM_sys_Mkdir"
  fi
  IGCM_debug_PopStack "IGCM_sys_Mkdir"
}

#D-#==================================================
#D-function IGCM_sys_MkdirArchive
#D-* Purpose: Mkdir on Archive
#D-* Examples:
#D-
function IGCM_sys_MkdirArchive {
  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_MkdirArchive :" $@
  fi
  #- creation de repertoire sur le serveur fichier
  if [ ! -d ${1} ]; then
    \mkdir -p $1
    status=$?

    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
}

#D-#==================================================
#D-function IGCM_sys_MkdirWork
#D-* Purpose: Mkdir on Work
#D-* Examples:
#D-
function IGCM_sys_MkdirWork {
  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_MkdirWork :" $@
  fi
  #- creation de repertoire sur le serveur fichier
  if [ ! -d ${1} ]; then
    \mkdir -p $1
    if [ $? -gt 0 ] ; then
      echo "IGCM_sys_MkdirWork : erreur."
      IGCM_debug_Exit "IGCM_sys_MkdirWork"
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
}

#D-#==================================================
#D-function IGCM_sys_Cd
#D-* Purpose: master cd command
#D-* Examples:
#D-
function IGCM_sys_Cd {
  IGCM_debug_PushStack "IGCM_sys_Cd" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Cd :" $@
  fi
  \cd $1
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_Cd : erreur."
    IGCM_debug_Exit "IGCM_sys_Cd"
  fi
  IGCM_debug_PopStack "IGCM_sys_Cd"
}

#D-#==================================================
#D-function IGCM_sys_Chmod
#D-* Purpose: Chmod
#D-* Examples:
#D-
function IGCM_sys_Chmod {
  IGCM_debug_PushStack "IGCM_sys_Chmod" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Chmod :" $@
  fi
  \chmod $@
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_Chmod : erreur."
    IGCM_debug_Exit "IGCM_sys_Chmod"
  fi
  IGCM_debug_PopStack "IGCM_sys_Chmod"
}

#D-#==================================================
#D-function IGCM_sys_FileSize
#D-* Purpose: Filesize
#D-* Examples:
#D-
function IGCM_sys_FileSize {
  IGCM_debug_PushStack "IGCM_sys_FileSize" $@

  typeset sizeF
  set +A sizeF -- $( ls -la ${1} )
  if [ $? -gt 0 ] ; then
    IGCM_debug_Exit "IGCM_sys_FileSize"
  fi
  eval ${2}=${sizeF[4]}

  IGCM_debug_PopStack "IGCM_sys_FileSize"
}

#D-#==================================================
#D-function IGCM_sys_TestDir
#D-* Purpose: Test Directory that must exists
#D-* Examples:
#D-
function IGCM_sys_TestDir {
  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_TestDir :" $@
  fi
  typeset ExistFlag
  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestDir"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_TestDirArchive
#D-* Purpose: Test Directory that must exists on Archive
#D-* Examples:
#D-
function IGCM_sys_TestDirArchive {
  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_TestDirArchive :" $@
  fi
  typeset ExistFlag
  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_IsFileArchived
#D-* Purpose: Test file that must NOT EXISTS on Archive
#D-* Examples:
#D-
function IGCM_sys_IsFileArchived {
  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_IsFileArchived :" $@
  fi
  typeset IsArchivedFlag
  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"

  return ${IsArchivedFlag}
}

#D-#==================================================
#D-function IGCM_sys_TestFileArchive
#D-* Purpose: Test file that must NOT EXISTS on Archive
#D-* Examples:
#D-
function IGCM_sys_TestFileArchive {
  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
  typeset ExistFlag
  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_TestFileBuffer
#D-* Purpose: Test file that must NOT EXISTS on Buffer
#D-* Examples:
#D-
function IGCM_sys_TestFileBuffer {
  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
  typeset ExistFlag
  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_CountFileArchive
#D-* Purpose: Count files on Archive filesystem
#D-* Examples:
#D-
function IGCM_sys_CountFileArchive {
  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
  ls ${@} 2>/dev/null | wc -l
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_CountFileArchive : erreur."
  fi
  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
}

#D-#==================================================
#D-function IGCM_sys_CountFileBuffer
#D-* Purpose: Count files on Scratch filesystem
#D-* Examples:
#D-
function IGCM_sys_CountFileBuffer {
  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
  ls ${@} 2>/dev/null | wc -l
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_CountFileBuffer : erreur."
  fi
  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
}

#D-#==================================================
#D-function IGCM_sys_Tree
#D-* Purpose: Tree directories with files on ${ARCHIVE}
#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
#D-
function IGCM_sys_Tree {
  IGCM_debug_PushStack "IGCM_sys_Tree" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Tree :" $@
  fi

  \ls -lR ${@}

  IGCM_debug_PopStack "IGCM_sys_Tree"
}

#D-#==================================================
#D-function IGCM_sys_Tar
#D-* Purpose: master tar command
#D-* Examples:
#D-
function IGCM_sys_Tar {
  IGCM_debug_PushStack "IGCM_sys_Tar" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Tar :" $@
  fi
  \tar cf $@
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_Tar : erreur."
    IGCM_debug_Exit "IGCM_sys_Tar"
  fi
  IGCM_debug_PopStack "IGCM_sys_Tar"
}

#D-#==================================================
#D-function IGCM_sys_UnTar
#D-* Purpose: master un-tar command
#D-* Examples:
#D-
function IGCM_sys_UnTar {
  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_UnTar :" $@
  fi
  \tar xvf $1
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_UnTar : erreur."
    IGCM_debug_Exit "IGCM_sys_UnTar"
  fi
  IGCM_debug_PopStack "IGCM_sys_UnTar"
}

#D-#==================================================
#D-function IGCM_sys_Qsub
#D-* Purpose: Qsub new job
#D-* Examples:
#D-
function IGCM_sys_Qsub {
  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Qsub :" $@
  fi
  typeset options status
  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"

  /usr/bin/ccc_msub ${options} $1 > /tmp/out_command.$$ 2>&1
  status=$?

  cat /tmp/out_command.$$
  if [ ${status} -gt 0 ] ; then
    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
    IGCM_debug_Exit "IGCM_sys_Qsub"
  else
    JobID=$( gawk {'print $4'} /tmp/out_command.$$ )
  fi
  IGCM_debug_PopStack "IGCM_sys_Qsub"
}

#D-#==================================================
#D-function IGCM_sys_QsubPost
#D-* Purpose: Qsub new job on scalaire
#D-* Examples:
#D-
function IGCM_sys_QsubPost {
  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_QsubPost :" $@
  fi
  typeset options status
  MY_PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=${DEFAULT_PROJECT}} | cut -d@ -f1 )

  # EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
  case $( hostname -s ) in
  curie*)
    options="-Q normal -A ${MY_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
  airain*)
    options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
  esac

  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > /tmp/out_command.$$ 2>&1
  status=$?

  cat /tmp/out_command.$$
  if [ ${status} -gt 0 ] ; then
    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
    IGCM_debug_Exit "IGCM_sys_QsubPost"
  else
    JobID=$( gawk {'print $4'} /tmp/out_command.$$ )
  fi
  IGCM_debug_PopStack "IGCM_sys_QsubPost"
}

#D-*************************
#D- File transfer functions
#D-*************************
#D-

#D-#==================================================
#D-function IGCM_sys_Rsync_out
#D-* Purpose: treat return val of rsync
#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
#D-  Error values and explanations can depend on your system version.
function IGCM_sys_Rsync_out {
  status=$1
  if [ ! $status ] ; then
    echo "rsync error !"
  fi

  if [ $MYLANG = "fr" ]; then
    case $status in
    0)  return ;;
    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
      echo "Erreur de syntaxe ou d'utilisation."
      return;;
    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
      echo "Incompatibilité de protocole."
      return;;
    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
      echo "répertoires"
      return;;
    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
      echo "Action demandée non supportée : une tentative de manipulation de"
      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
      echo "été faite ; ou une option qui est supportée par le  client  mais"
      echo "pas par le serveur a été spécifiée."
      return;;
    10) echo "Erreur de rsync ; RERR_SOCKETIO"
      echo "Erreur dans le socket d'entrée sortie"
      return;;
    11) echo "Erreur de rsync ; RERR_FILEIO"
      echo "Erreur d'entrée sortie fichier"
      return;;
    12) echo "Erreur de rsync ; RERR_STREAMIO"
      echo "Erreur dans flux de donnée du protocole rsync"
      return;;
    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
      echo "Erreur avec les diagnostics du programme"
      return;;
    14) echo "Erreur de rsync ; RERR_IPC"
      echo "Erreur dans le code IPC"
      return;;
    20) echo "Erreur de rsync ; RERR_SIGNAL"
      echo "SIGUSR1 ou SIGINT reçu"
      return;;
    21) echo "Erreur de rsync ; RERR_WAITCHILD"
      echo "Une erreur retournée par waitpid()"
      return;;
    22) echo "Erreur de rsync ; RERR_MALLOC"
      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
      return;;
    23) echo ""
      echo "Erreur fichier inexistant"
      return;;
    30) echo "Erreur de rsync ; RERR_TIMEOUT"
      echo "Temps d'attente écoulé dans l'envoi/réception de données"
      return;;
    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
      return;;
    esac
  elif [ $MYLANG = "en" ] ; then
    case $status in
    0)  return;;
    1)  echo "rsync error : Syntax or usage error "
      return;;
    2)  echo "rsync error : Protocol incompatibility "
      return;;
    3)  echo "rsync error : Errors selecting input/output files, dirs"
      return;;
    4)  echo "rsync error : Requested action not supported: an attempt"
      echo "was made to manipulate 64-bit files on a platform that cannot support"
      echo "them; or an option was specified that is supported by the client and"
      echo "not by the server."
      return;;
    5)  echo "rsync error : Error starting client-server protocol"
      return;;
    10) echo "rsync error : Error in socket I/O "
      return;;
    11) echo "rsync error : Error in file I/O "
      return;;
    12) echo "rsync error : Error in rsync protocol data stream "
      return;;
    13) echo "rsync error : Errors with program diagnostics "
      return;;
    14) echo "rsync error : Error in IPC code "
      return;;
    20) echo "rsync error : Received SIGUSR1 or SIGINT "
      return;;
    21) echo "rsync error : Some error returned by waitpid() "
      return;;
    22) echo "rsync error : Error allocating core memory buffers "
      return;;
    23) echo "rsync error : Partial transfer due to error"
      return;;
    24) echo "rsync error : Partial transfer due to vanished source files"
      return;;
    30) echo "rsync error : Timeout in data send/receive "
      return;;
    *)  echo "rsync error : return code of rsync unknown :" $status
      return;;
    esac
  else
    echo "unknown language $MYLANG."
    return
  fi
}

#D-#==================================================
#D-function IGCM_sys_Miror_libIGCM
#D-* Purpose: Mirror libIGCM PATH and lib to frontend
#D-* Examples:
#D-
function IGCM_sys_Mirror_libIGCM {
  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Mirror_libIGCM"
  fi

  typeset status

  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}

  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > /tmp/out_command.$$ 2>&1
  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> /tmp/out_command.$$ 2>&1
  status=$?

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
    cat /tmp/out_command.$$
  fi
  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
}

#D-#==================================================
#D-function IGCM_sys_Cp
#D-* Purpose: generic cp
#D-* Examples:
#D-
function IGCM_sys_Cp {
  IGCM_debug_PushStack "IGCM_sys_Cp" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Cp :" $@
  fi

  typeset status

  echo cp $@ > /tmp/out_command.$$ 2>&1
  \cp $@ >> /tmp/out_command.$$ 2>&1
  status=$?

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_Cp : error code ${status}"
    cat /tmp/out_command.$$
    IGCM_debug_Exit "IGCM_sys_Cp"
  else
    \rm /tmp/out_command.$$
  fi
  IGCM_debug_PopStack "IGCM_sys_Cp"
}

#D-#==================================================
#D-function IGCM_sys_Rm
#D-* Purpose: generic rm
#D-* Examples:
#D-
function IGCM_sys_Rm {
  IGCM_debug_PushStack "IGCM_sys_Rm" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Rm :" $@
  fi

  typeset status

  echo rm $@ > /tmp/out_command.$$ 2>&1
  \rm $@ >> /tmp/out_command.$$ 2>&1
  status=$?

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_Rm : error code ${status}"
    cat /tmp/out_command.$$
    IGCM_debug_Exit "IGCM_sys_Rm"
  else
    \rm /tmp/out_command.$$
  fi
  IGCM_debug_PopStack "IGCM_sys_Rm"
}

#D-#==================================================
#D-function IGCM_sys_RmRunDir
#D-* Purpose: rm tmpdir (dummy function most of the time batch
#D-                      scheduler will do the job)
#D-* Examples:
#D-
function IGCM_sys_RmRunDir {
  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_RmRunDir :" $@
  fi

  typeset status

  echo rm $@ > /tmp/out_command.$$ 2>&1
  \rm $@ >> /tmp/out_command.$$ 2>&1
  status=$?

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
    cat /tmp/out_command.$$
    IGCM_debug_Exit "IGCM_sys_RmRunDir"
  else
    \rm /tmp/out_command.$$
  fi
  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
}

#D-#==================================================
#D-function IGCM_sys_Mv
#D-* Purpose: generic move
#D-* Examples:
#D-
function IGCM_sys_Mv {
  IGCM_debug_PushStack "IGCM_sys_Mv" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Mv :" $@
  fi

  if [ $DRYRUN = 0 ]; then

    typeset status

    echo mv $@ > /tmp/out_command.$$ 2>&1
    \mv $@ >> /tmp/out_command.$$ 2>&1
    status=$?

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Mv : error code ${status}"
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_Mv"
    else
      \rm /tmp/out_command.$$
    fi
  fi

  IGCM_debug_PopStack "IGCM_sys_Mv"
}

#D-#==================================================
#D-function IGCM_sys_Put_Dir
#D-* Purpose: Copy a complete directory on $(ARCHIVE)
#D-* Examples:
#D-
function IGCM_sys_Put_Dir {
  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Dir :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if [ ! -d ${1} ] ; then
      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
      return
    fi

    typeset status

    # Only if we use rsync
    #IGCM_sys_TestDirArchive $( dirname $2 )
    #
    #USUAL WAY
    \cp -r $1 $2 > /tmp/out_command.$$ 2>&1
    status=$?

    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_Put_Dir"
    else
      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
}

#D-#==================================================
#D-function IGCM_sys_Get_Dir
#D-* Purpose: Copy a complete directory from ${ARCHIVE}
#D-* Examples:
#D-
function IGCM_sys_Get_Dir {
  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Get_Dir :" $@
  fi
  if [ $DRYRUN = 0 ]; then
#    if [ ! -d ${1} ] ; then
#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
#      return
#    fi

    typeset NB_ESSAI DELAI status i
    # number of tentative
    NB_ESSAI=3
    # time delay between tentative
    DELAI=2

    # Only if we use rsync
    #IGCM_sys_TestDirArchive $( dirname $2 )
    #
    # USUAL WAY
    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
    ccc_hsm get -r $1

    i=0
    while [ $i -lt $NB_ESSAI ] ; do
      \cp -ur $1 $2 >> /tmp/out_command.$$ 2>&1
      status=$?
      if [ ${status} -gt 0 ] ; then
        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
        sleep $DELAI
      else
        break
      fi
      (( i = i + 1 ))
    done

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Get_Dir : error."
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_Get_Dir"
    else
      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
}

#D-#==================================================
#D-function IGCM_sys_Get_Master
#D-* Purpose: Copy a complete directory from MASTER filesystem
#D-* Examples:
#D-
function IGCM_sys_Get_Master {
  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Get_Master :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_Get_Master"
      return
    fi

    typeset NB_ESSAI DELAI status i
    # number of tentative
    NB_ESSAI=3
    # time delay between tentative
    DELAI=2

    i=0
    while [ $i -lt $NB_ESSAI ] ; do
      \cp -urL $1 $2 > /tmp/out_command.$$ 2>&1
      status=$?
      if [ ${status} -gt 0 ]; then
        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
        sleep $DELAI
      else
        break
      fi
      (( i = i + 1 ))
    done

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Get_Master : error."
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_Get_Master"
    else
      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Get_Master"
}

#====================================================
#- Call IGCM_sys_Mirror_libIGCM now !
if ( $MirrorlibIGCM ) ; then
  IGCM_sys_Mirror_libIGCM
fi

#D-#==================================================
#D-function IGCM_sys_Put_Rest
#D-* Purpose: Put computied restarts on ${ARCHIVE}.
#D-           File and target directory must exist.
#D-* Examples:
#D-
function IGCM_sys_Put_Rest {
  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Rest :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
      IGCM_debug_Exit "IGCM_sys_Put_Rest"
    fi

    typeset status
    #
    # USUAL WAY
    \cp $1 $2 > /tmp/out_command.$$ 2>&1
    status=$?

#       #RSYNC WITH NETWORK SSH CALL
#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > /tmp/out_command.$$ 2>&1
#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> /tmp/out_command.$$ 2>&1

#       #RSYNC WITH NFS USE
#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > /tmp/out_command.$$ 2>&1
#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> /tmp/out_command.$$ 2>&1

#       status=$?
#       IGCM_sys_Rsync_out $status

#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
#       (( status=status+$? ))

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
      [ -f ${1} ] && ls -l ${1}
      [ -f ${2} ] && ls -l ${2}
      [ -f ${2}/${1} ] && ls -l ${2}/${1}
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_Put_Rest"
    else

      if [ X${JobType} = XRUN ] ; then
        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
      fi

      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
}

#D-#==================================================
#D-function IGCM_sys_PutBuffer_Rest
#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
#D-           File and target directory must exist.
#D-* Examples:
#D-
function IGCM_sys_PutBuffer_Rest {
  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_PutBuffer_Rest :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
    fi

    typeset status
    #
    # USUAL WAY
    \cp $1 $2 > /tmp/out_command.$$ 2>&1
    status=$?

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
      [ -f ${2} ] && ls -l ${2}
      [ -f ${2}/${1} ] && ls -l ${2}/${1}
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
    else

      if [ X${JobType} = XRUN ] ; then
        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
      fi

      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
}

#D-#==================================================
#D-function IGCM_sys_PrepareTaredRestart
#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
#D-* Examples:
#D-
function IGCM_sys_PrepareTaredRestart {
  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
  if [ $DRYRUN = 0 ]; then
    [ ! -f $( basename $1 ) ] && IGCM_sys_Get $1 .
  fi
  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
}

#D-#==================================================
#D-function IGCM_sys_Put_Out
#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
#D-* Examples:
#D-
function IGCM_sys_Put_Out {
  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Out :" $@
  fi

  typeset NB_ESSAI DELAI status i exist skip

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_Put_Out"
      return 1
    fi
    #
    IGCM_sys_MkdirArchive $( dirname $2 )
    #

    #=====================================================
    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
    #=====================================================

    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
    #if [ $? -eq 0 ] ; then
    #    typeset WORKPATH FILEPATH
    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
    #    IGCM_sys_MkdirWork ${WORKPATH}
    #    FILEPATH=${WORKPATH}/$( basename $2 )
    #    #
    #    IGCM_sys_Cp ${1} ${FILEPATH}
    #fi

    exist=false
    skip=false
    if [ -f $2 ] ; then
      IGCM_debug_Print 1 "$2 already exist"
      ccc_hsm get $2
      exist=true
      if [ "X$( diff $1 $2 )" = X ] ; then
        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
        skip=true
      else
        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
        skip=false
      fi
    fi
    #
    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
      IGCM_sys_Chmod u+w $2
    fi

    if [ X${skip} = Xfalse ] ; then
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
          # USUAL WAY
          \cp $1 $2 > /tmp/out_command.$$ 2>&1
          status=$?
        else
          # NOT SO USUAL WAY
          \mv $1 $2 > /tmp/out_command.$$ 2>&1
          status=$?
        fi
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
          [ -f ${1} ] && ls -l ${1}
          [ -f ${2} ] && ls -l ${2}
          [ -f ${2}/${1} ] && ls -l ${2}/${1}
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
    fi

#       #RSYNC WITH NETWORK SSH CALL
#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > /tmp/out_command.$$ 2>&1
#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> /tmp/out_command.$$ 2>&1

#       #RSYNC WITH NFS USE
#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > /tmp/out_command.$$ 2>&1
#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> /tmp/out_command.$$ 2>&1

#       status=$?
#       IGCM_sys_Rsync_out $status

#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
#       (( status=status+$? ))

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Out : error."
      [ -f ${1} ] && ls -l ${1}
      [ -f ${2} ] && ls -l ${2}
      [ -f ${2}/${1} ] && ls -l ${2}/${1}
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_Put_Out"
    else

      if [ X${JobType} = XRUN ] ; then
        if [ X${3} = X ] ; then
          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
        fi
      fi

      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Out"
  return 0
}

#D-#==================================================
#D-function IGCM_sys_PutBuffer_Out
#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
#D-* Examples:
#D-
function IGCM_sys_PutBuffer_Out {
  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_PutBuffer_Out :" $@
  fi

  typeset NB_ESSAI DELAI status i exist skip

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
      return 1
    fi
    #
    IGCM_sys_Mkdir $( dirname $2 )
    #

    exist=false
    skip=false
    if [ -f $2 ] ; then
      IGCM_debug_Print 1 "$2 already exist"
      exist=true
      if [ "X$( diff $1 $2 )" = X ] ; then
        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
        status=0
        skip=true
      else
        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
        skip=false
      fi
    fi
    #
    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
      IGCM_sys_Chmod u+w $2
    fi

    if [ X${skip} = Xfalse ] ; then
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
          # USUAL WAY
          \cp $1 $2 > /tmp/out_command.$$ 2>&1
          status=$?
        else
          # NOT SO USUAL WAY
          \mv $1 $2 > /tmp/out_command.$$ 2>&1
          status=$?
        fi
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
          [ -f ${2} ] && ls -l ${2}
          [ -f ${2}/${1} ] && ls -l ${2}/${1}
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
    fi

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_PutBuffer_Out : error."
      [ -f ${2} ] && ls -l ${2}
      [ -f ${2}/${1} ] && ls -l ${2}/${1}
      cat /tmp/out_command.$$
      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
    else

      if [ X${JobType} = XRUN ] ; then
        if [ X${3} = X ] ; then
          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
        fi
      fi

      \rm /tmp/out_command.$$
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
  return 0
}

#D-#==================================================
#D-function IGCM_sys_Get
#D-* Purpose: Get a file from ${ARCHIVE}
#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
function IGCM_sys_Get {
  IGCM_debug_PushStack "IGCM_sys_Get" $@

  typeset DEST dm_liste target file_work
  typeset NB_ESSAI DELAI status i

  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Get :" $@
  fi

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN -le 2 ]; then
    if [ X${1} = X'/l' ] ; then
      eval set +A dm_liste \${${2}}
    else
      eval set +A dm_liste ${1}
    fi
    eval DEST=\${${#}}

    #=====================================================
    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
    #=====================================================

    # Is it an R_OUT file (not R_IN) ?
    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
    #if [ $? -eq 0 ] ; then
    #  # Yes  ? then we try to get it in SCRATCHDIR
    #  set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
    #  if [ -f ${file_work[0]} ] ; then
    #    IGCM_sys_Cp ${file_work[*]} ${DEST}
    #    IGCM_debug_PopStack "IGCM_sys_Get"
    #    return
    #  fi
    #fi

    ccc_hsm get ${dm_liste[*]} > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      echo "WARNING IGCM_sys_Get : error code ${status}"
      cat /tmp/out_command.$$
      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
    fi

    #if [ ${status} -gt 0 ] ; then
    #  if [ ! "X$( grep "Lost dmusrcmd connection" /tmp/out_command.$$ )" = "X" ] ; then
    #    cat /tmp/out_command.$$
    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
    #    sleep 30
    #    echo "We try another time"
    ##    dmget ${dm_liste[*]} > /tmp/out_command.$$ 2>&1
    #    ccc_hsm get ${dm_liste[*]} > /tmp/out_command.$$ 2>&1
    #    status=$?
    #    if [ ${status} -gt 0 ] ; then
    #      echo "ERROR IGCM_sys_Get : again demigration error :"
    #      cat /tmp/out_command.$$
    #      IGCM_debug_Exit "IGCM_sys_Get"
    #    fi
    #  else
    #    echo "ERROR IGCM_sys_Get : demigration error :"
    #    cat /tmp/out_command.$$
    #    IGCM_debug_Exit "IGCM_sys_Get"
    #  fi
    #fi

    #   #RSYNC WITH NETWORK SSH CALL
    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > /tmp/out_command.$$ 2>&1
    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> /tmp/out_command.$$ 2>&1

    #   #RSYNC WITH NFS USE
    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > /tmp/out_command.$$ 2>&1
    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> /tmp/out_command.$$ 2>&1

    #   status=$?
    #   IGCM_sys_Rsync_out $status

    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
    #   (( status=status+$? ))

    #USUAL WAY
    if [ X${1} = X'/l' ] ; then
      for target in ${dm_liste[*]} ; do
        local_file=$( basename ${target} )
        # test if the target file is present before the loop
        IGCM_sys_TestFileArchive ${target}
        status=$?
        if [ ${status} -gt 0 ] ; then
          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
          IGCM_debug_Exit "IGCM_sys_Get"
        else
          i=0
          while [ $i -lt $NB_ESSAI ] ; do
            #if [ X${DoLink} = Xtrue ] ; then
            #  \ln -s ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
            #  status=$?
            #  else
            #  \cp ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
            #  status=$?
            #fi
            \ln -s ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
            status=$?
            if [ ${status} -gt 0 ]; then
              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
              sleep $DELAI
            else
              break
            fi
            (( i = i + 1 ))
          done
          if [ ${status} -gt 0 ] ; then
            echo "IGCM_sys_Get : error"
            cat /tmp/out_command.$$
            \rm /tmp/out_command.$$
            IGCM_debug_Exit "IGCM_sys_Get"
          else
            \rm /tmp/out_command.$$
          fi
        fi
      done
    else
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        \cp ${dm_liste} ${DEST} >> /tmp/out_command.$$ 2>&1
        status=$?
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
      if [ ${status} -gt 0 ] ; then
        echo "IGCM_sys_Get : error"
        cat /tmp/out_command.$$
        \rm /tmp/out_command.$$
        IGCM_debug_Exit "IGCM_sys_Get"
      else
        \rm /tmp/out_command.$$
      fi
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Get"
}

#D-#==================================================
#D-function IGCM_sys_GetBuffer
#D-* Purpose: Get a file from ${SCRATCHDIR}
#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
function IGCM_sys_GetBuffer {
  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@

  typeset DEST buf_liste target file_work
  typeset NB_ESSAI DELAI status i

  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetBuffer :" $@
  fi

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN -le 2 ]; then
    if [ X${1} = X'/l' ] ; then
      eval set +A buf_liste \${${2}}
    else
      eval set +A buf_liste ${1}
    fi
    eval DEST=\${${#}}

    #USUAL WAY
    if [ X${1} = X'/l' ] ; then
      for target in ${buf_liste[*]} ; do
        local_file=$( basename ${target} )
        i=0
        while [ $i -lt $NB_ESSAI ] ; do
          \cp ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
          status=$?
          if [ ${status} -gt 0 ]; then
            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
            sleep $DELAI
          else
            break
          fi
          (( i = i + 1 ))
        done
        if [ ${status} -gt 0 ] ; then
          echo "IGCM_sys_Get : error"
          cat /tmp/out_command.$$
          \rm /tmp/out_command.$$
          IGCM_debug_Exit "IGCM_sys_GetBuffer"
        else
          \rm /tmp/out_command.$$
        fi
      done
    else
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        \cp ${buf_liste} ${DEST} >> /tmp/out_command.$$ 2>&1
        status=$?
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
      if [ ${status} -gt 0 ] ; then
        echo "IGCM_sys_Get : error"
        cat /tmp/out_command.$$
        \rm /tmp/out_command.$$
        IGCM_debug_Exit "IGCM_sys_GetBuffer"
      else
        \rm /tmp/out_command.$$
      fi
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
}

#D-#==================================================
#D-function IGCM_sys_GetDate_FichWork
#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
#D-* Examples:
#D-
function IGCM_sys_GetDate_FichWork {
  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetDate_FichWork :" $@
  fi

  if [ $# -ge 3 ] ; then
    mode=$3
    TimeStyle=$4
  else
    mode="default"
    TimeStyle="%Y%m%d%H%M%S"
  fi

  typeset dateF
  set +A dateF -- $( ls -l --full-time --time-style=+"${TimeStyle}" ${1} )

  case $mode in
    "default")
      eval ${2}=${dateF[5]}
      ;;
    "SplitFields")
      eval ${2}="${dateF[5]}\ ${dateF[6]}"
      ;;
  esac


  # donne la date filesys d'un fichier sur la machine work
  IGCM_debug_PopStack "IGCM_sys_FichWork"
}

#D-#==================================================
#D-function IGCM_sys_GetDate_FichArchive
#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
#D-* Examples:
#D-
function IGCM_sys_GetDate_FichArchive {
  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetDate_FichArchive :" $@
  fi
  typeset dateF
  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
  eval ${2}=${dateF[5]}

  IGCM_debug_PopStack "IGCM_sys_FichArchive"
}

#D-#==================================================
#D-function IGCM_sys_GetDate_Monitoring
#D-* Purpose: get the last year for which the monitoring has been computed
#D-* Examples:
#D-
function IGCM_sys_GetDate_Monitoring {
  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetDate_Monitoring :" $@
  fi

  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )

  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
}

#D-#==================================================
#D-function IGCM_sys_Dods_Rm
#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
#D-* Examples:
#D-
function IGCM_sys_Dods_Rm {
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Dods_Rm :" $@
  fi
  typeset status
  if [ $DRYRUN = 0 ]; then

#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
#      echo "Nothing has been done."
#      return
#    fi

    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
    status=$?

#    if [ ${status} -gt 0 ] ; then
#      echo "IGCM_sys_Dods_Rm : error."
#      cat out_dods_rm
#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
#    else
#      rm out_dods_rm
#    fi

  fi
  return $status
}

#D-#==================================================
#D-function IGCM_sys_Dods_Cp
#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
#D-* Examples:
#D-
function IGCM_sys_Dods_Cp {
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Dods_Cp :" $@
  fi
  typeset status
  if [ $DRYRUN = 0 ]; then

#    if [ ! -d ${R_SAVE}/${1} ] ; then
#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
#      echo "Nothing has been done."
#      return
#    fi

    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
    status=$?

#       if [ ${status} -gt 0 ] ; then
#           echo "IGCM_sys_Dods_Cp : error."
#           cat out_dods_cp
#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
#       else
#           rm out_dods_cp
#       fi

  fi
  return $status
}

#D-#==================================================
#D-function IGCM_sys_Put_Dods
#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
#D-* Examples:
#D-
function IGCM_sys_Put_Dods {
  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Dods :" $@
  fi
  #set -vx
  typeset status
  if [ $DRYRUN = 0 ]; then
    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
      return
    fi

    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
      return
    fi
    #
    if [ -d ${R_SAVE}/${1} ] ; then
      cd ${R_SAVE}
    elif [ -d ${R_FIGR}/${1} ] ; then
      cd ${R_FIGR}
    fi

    IGCM_sys_Dods_Rm ${1}
    IGCM_sys_Dods_Cp ${1}
    status=0

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Dods : error."
      IGCM_debug_Exit "IGCM_sys_Put_Dods"
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
}

##############################################################
# REBUILD OPERATOR

#D-#==================================================
#D-function IGCM_sys_rebuild
#D-* Purpose: rebuild parallel files
#D-* Examples:
#D-
function IGCM_sys_rebuild {
  IGCM_debug_PushStack "IGCM_sys_rebuild" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_rebuild :" $@
  fi

  typeset NB_ESSAI DELAI status i firstArg
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ~p86ipsl/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/rebuild -f -o $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      firstArg=${1}
      \rm ${firstArg}
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
    IGCM_debug_Exit "rebuild"
  fi

  IGCM_debug_PopStack "IGCM_sys_rebuild"
}

#D-#==================================================
#D-function IGCM_sys_rebuild_station
#D-* Purpose: rebuild parallel files describing station
#D-* Examples:
#D-
function IGCM_sys_rebuild_station {
  IGCM_debug_PushStack "IGCM_sys_rebuild_station" $@
  typeset i list_opt file_in file_out prefix_invert list_invert
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_rebuild_station :" $@
  fi
  list_opt=$@

  # Invert Axis : t,x -> x,t
  #               t,pres,x -> x,t,pres
  # So that we can concatenate along x
  i=0
  for file_in in ${list_opt} ; do
    (( i = i + 1))
    [ ${i} = 1 ] && file_out=${file_in} && continue
    # detect time counter and do the job only if present
    var_unlim=$(ncdump -h ${file_in} | grep UNLIMITED | cut -d ' ' -f 1 | sed -e 's/^[ \t]*//' -e 's/[ \t]*$//')
    if [ X${var_unlim} = Xtime_counter ] ; then
      prefix_invert=$( basename ${file_in} .nc )
      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
      list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
    fi
  done

  # Concatenate
  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc

  # Re-ivert file
  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}

  # Station re-ordering is too expansive to be run within libIGCM
  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
  # This re-ordering must be done "in memory" by the cmorization process
  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
  # BEGIN reordering

  # Only LMDZ text output contains the exact ordering of the station.
  # We isolate this in the code below:
  #  0  38  -157.5000000000000  70.98591549295774
  #  0  54  27.49999999999999   67.18309859154928
  #  0  56  -62.50000000000001  82.39436619718309
  #  0  79  12.49999999999999   78.59154929577466
  #  0  116 -165.0000000000000  76.05633802816901
  #  0  117 130.0000000000000   70.98591549295774
  #  0  118 110.0000000000000   87.46478873239437
  #  1  40  4.999999999999995   51.97183098591550

#  typeset iStation iProc list_opt file_in file_out prefix_invert
#  typeset -Z4 j4
#  typeset -Z3 j3

#  unset list_opt
#  set +A list_opt $@

  # Filename after rebuild
#  file_out=${list_opt[0]}
  # Prefix of output files
#  prefix_invert=$( basename ${file_out} .nc )
  # Number of procs
#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )

#  iProc=0
#  while [ ${iProc} -lt ${num_proc} ] ; do
    # Array containing Station as a number
#    unset proc_stn
#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | gawk ' {print $2}' )
    # Number of stations produced by processor proc
#    stationLast=${#proc_stn[*]}
    # Proc number on 4 digits
#    j4=${iProc}
    # Init
#    iStation=0
#    while [ ${iStation} -lt ${stationLast} ] ; do
      # Station number on 3 digits
#      j3=${proc_stn[${iStation}]}
      # Extract station
      # Invert Axis : t,x -> x,t
      #               t,pres,x -> x,t,pres
      # So that we can concatenate along x
#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
#      (( iStation = iStation + 1 ))
#    done
#    (( iProc = iProc + 1 ))
#  done

  # Concatenate all station along x
#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc

  # Re-invert file
#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}

  # END reordering

  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
}

############################################################
# Activate Running Environnment Variables

#D-#==================================================
#D-function IGCM_sys_desactiv_variables
#D-* Purpose: set environement variables prior to execution
#D-* Examples:
#D-
function IGCM_sys_activ_variables {
  IGCM_debug_PushStack "IGCM_sys_activ_variables"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_activ_variables"
  fi

# --------------------------------------------------------------------
#D- MPI specifications
# --------------------------------------------------------------------

# --------------------------------------------------------------------
#D- Other specifications
# --------------------------------------------------------------------

  ulimit -s unlimited

  IGCM_debug_PopStack "IGCM_sys_activ_variables"
}

############################################################
# Desactivate Running Environnment Variables

#D-#==================================================
#D-function IGCM_sys_desactiv_variables
#D-* Purpose: unset environement variables after execution
#D-* Examples:
#D-
function IGCM_sys_desactiv_variables {
  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_desactiv_variables"
  fi
# --------------------------------------------------------------------
#D- MPI specifications
# --------------------------------------------------------------------

# --------------------------------------------------------------------
#D- Other specifications
# --------------------------------------------------------------------

  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
}

############################################################
# Build MPI/OMP scripts run file (dummy function)

#D-#==================================================
#D-function IGCM_sys_build_run_file
#D-* Purpose: build run file (deprecated)
#D-* Examples:
#D-
function IGCM_sys_build_run_file {

  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "

}

############################################################
# Build MPI/OMP scripts

#D-#==================================================
#D-function IGCM_sys_build_execution_scripts
#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
#D-* Examples:
#D-
function IGCM_sys_build_execution_scripts
{
  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_build_execution_scripts " $@
  fi

  EXECUTION=${HOST_MPIRUN_COMMAND}

  # MPMD mode
  if ( ${OK_PARA_MPMD} ) ; then

    # Only MPI (MPMD)
    if  ( ! ${OK_PARA_OMP} ) ; then

      if [ -f run_file ] ; then
        IGCM_sys_Rm -f run_file
      fi
      touch run_file

      # Build run_file

      # First loop on the components for the coupler ie oasis (only if oasis3)
      # the coupler ie oasis3 must be the first one
      for comp in ${config_ListOfComponents[*]} ; do

        eval ExeNameIn=\${config_Executable_${comp}[0]}
        eval ExeNameOut=\${config_Executable_${comp}[1]}

        # for CPL component only
        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then

          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
        fi
      done

      # Then second loop on the components
      for comp in ${config_ListOfComponents[*]} ; do

        eval ExeNameIn=\${config_Executable_${comp}[0]}
        eval ExeNameOut=\${config_Executable_${comp}[1]}

        # Only if we really have an executable for the component and not the coupler ie oasis:
        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then

          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
        fi
      done

      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"

      IGCM_sys_Chmod u+x run_file
      if ( $DEBUG_sys ) ; then
        echo "run_file contains : "
        cat run_file
      fi

    # MPI-OpenMP (MPMD)
    else

      # Use of mpirun instead of ccc_mprun
      EXECUTION="time mpirun"

      #  Hosts treatment
      ${EXECUTION} hostname | sort | uniq > hosts.tmp

      i=0
      rm -f hosts rankfile
      IGCM_debug_Print 1 "sys Curie, Hosts available :"
      for nodes in `cat hosts.tmp`
      do
        host[$i]=$nodes
        echo "${host[$i]}" >> hosts
        IGCM_debug_Print 1 ${host[$i]}
        i=$((i+1))
      done
      rm -f hosts.tmp

      listnodes=${host[*]}

      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"

      # Initialisation
      rank=0
      current_core=0
      core_per_node=16
      init_exec=n

      # Loop on the components
      for comp in ${config_ListOfComponents[*]} ; do

        eval ExeNameIn=\${config_Executable_${comp}[0]}
        eval ExeNameOut=\${config_Executable_${comp}[1]}

        # Not possible if oasis has an executable (i.e old version of oasis3)
        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
          IGCM_debug_Verif_Exit
        fi

        # Only if we really have an executable for the component :
        if [ "X${ExeNameOut}" != X\"\" ] ; then

          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}

          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
          echo ""  >> script_${ExeNameOut}.ksh
          if [ ${comp_proc_omp_loc} -gt 1 ] ; then

            # Check if the number of threads is correct
            case ${comp_proc_omp_loc} in
            2|4|8|16)
              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
              ;;
            *)
              IGCM_debug_Exit "ERROR with OMP parameters !"
              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
              IGCM_debug_Verif_Exit
              ;;
            esac
            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
          fi
          #echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
          #echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh

          if [ ${init_exec} = y ] ; then
            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
          else
            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
            init_exec=y
          fi

# Build rankfile : method used to assign cores and nodes for the MPI process
# Ex :
#rank 0=curie5296 slot=0,1,2,3
#rank 1=curie5296 slot=4,5,6,7
# Example of final command :
# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
# with script_lmdz.x.ksh :
# #!/bin/ksh
#export KMP_STACKSIZE=3g
#export KMP_LIBRARY=turnaround
#export MKL_SERIAL=YES
#OMP_NUM_THREADS=4
#./lmdz.x
#

          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
            (( index_host = current_core / core_per_node ))
            host_value=${host[${index_host}]}
            (( slot =  current_core % core_per_node ))
            virg=","
            string_final=""
            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
              string=$index$virg
              string_final=$string_final$string
            done
            string_final=$( echo $string_final | sed "s/.$//" )
            echo "rank $rank=$host_value slot=$string_final" >> rankfile
            (( rank = rank + 1 ))
            (( current_core = current_core + comp_proc_omp_loc ))
          done
        fi

      done
    fi

  # Only one executable (SPMD mode).
  else

    for comp in ${config_ListOfComponents[*]} ; do

      # Only if we really have an executable for the component :
      eval ExeNameOut=\${config_Executable_${comp}[1]}
      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then

        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
        echo ""  >> script_${ExeNameOut}.ksh
        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh

        if ( ${OK_PARA_OMP} ) ; then
          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
          echo ""  >> script_${ExeNameOut}.ksh
          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
        fi

        if  ( ${OK_PARA_MPI} ) ; then
          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
          # Default : ccc_mprun used if nb_proc gt 1
          # to have out/err per process on different files
          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
        else
          # Default : ccc_mprun is NOT used if nb_proc eq 1
          # to have out/err per process on different files
          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
        fi

        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
        cat script_${ExeNameOut}.ksh

      fi

    done

  fi

  IGCM_debug_Print 1 "sys Curie : execution command is "
  IGCM_debug_Print 1 "$EXECUTION"

  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
}

#D-#==================================================
#D-function IGCM_sys_check_path
#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
#D-* do not point to an important use directory. Stop immediately in that case.
#D-* Examples:
#D-
function IGCM_sys_check_path {
  IGCM_debug_PushStack "IGCM_sys_check_path"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_check_path"
  fi

  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
    IGCM_debug_Exit "This will stop the job"
  fi
  IGCM_debug_PopStack "IGCM_sys_check_path"
}

#D-#==================================================
#D-function IGCM_sys_check_quota
#D-* Purpose: check user quota. Stop the simulation if quota above 90%
#D-* Examples:
#D-
function IGCM_sys_check_quota {
  IGCM_debug_PushStack "IGCM_sys_check_quota"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_check_quota"
  fi
  # Limit of quota (in %)
  limit_quota=90

  # Check of the volume
  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')

  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then

    unit_avail=${volume_avail: -1}
    unit_quota=${volume_quota: -1}

    if [ "${unit_quota}" = "*" ] ; then
      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
      IGCM_debug_Print 1 "More than 100% of your quota is used"
      IGCM_debug_Print 1 "Use the ccc_quota command to check"
      IGCM_debug_Print 1 "You must have more than 10% available to run"
      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
      IGCM_debug_Verif_Exit
    fi

    temp_avail=${volume_avail%%${volume_avail: -1}*}
    temp_quota=${volume_quota%%${volume_quota: -1}*}

    if [ ! ${unit_avail} = ${unit_quota} ] ; then

    # Convertion
      if [ ${volume_avail: -1} = "T" ] ; then
        (( temp_avail = temp_avail * 1000000000000 ))
      elif [ ${volume_avail: -1} = "G" ] ; then
        (( temp_avail = temp_avail * 1000000000 ))
      elif [ ${volume_avail: -1} = "M" ] ; then
        (( temp_avail = temp_avail * 1000000 ))
      elif [ ${volume_avail: -1} = "k" ] ; then
        (( temp_avail = temp_avail * 1000 ))
      else
        (( temp_avail = volume_avail ))
      fi
      if [ ${volume_quota: -1} = "T" ] ; then
        (( temp_quota = temp_quota * 1000000000000 ))
      elif [ ${volume_quota: -1} = "G" ] ; then
        (( temp_quota = temp_quota * 1000000000 ))
      elif [ ${volume_quota: -1} = "M" ] ; then
        (( temp_quota = temp_quota * 1000000 ))
      elif [ ${volume_quota: -1} = "k" ] ; then
        (( temp_quota = temp_quota * 1000 ))
      else
        (( temp_quota = volume_quota ))
      fi
    fi

    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
#    echo "volume ratio is " $quota_volume

    if [ ${quota_volume} -ge ${limit_quota} ] ; then
      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
      IGCM_debug_Print 1 "Use the ccc_quota command to check"
      IGCM_debug_Print 1 "You must have more than 10% available to run"
      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
      IGCM_debug_Verif_Exit
    fi

  fi

# Check of the number of inodes

  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')

  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then

    unit_avail=${inode_avail: -1}
    unit_quota=${inode_quota: -1}

    if [ "${unit_quota}" = "*" ] ; then
      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
      IGCM_debug_Print 1 "More than 100% of your quota is used"
      IGCM_debug_Print 1 "Use the ccc_quota command to check"
      IGCM_debug_Print 1 "You must have more than 10% available to run"
      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
      IGCM_debug_Verif_Exit
    fi

    temp_avail=${inode_avail%%${inode_avail: -1}*}
    temp_quota=${inode_quota%%${inode_quota: -1}*}

    if [ ! ${unit_avail} = ${unit_quota} ] ; then

    # Convertion
      if [ ${inode_avail: -1} = "T" ] ; then
        (( temp_avail = temp_avail * 1000000000000 ))
      elif [ ${inode_avail: -1} = "G" ] ; then
        (( temp_avail = temp_avail * 1000000000 ))
      elif [ ${inode_avail: -1} = "M" ] ; then
        (( temp_avail = temp_avail * 1000000 ))
      elif [ ${inode_avail: -1} = "k" ] ; then
        (( temp_avail = temp_avail * 1000 ))
      else
        (( temp_avail = inode_avail ))
      fi

      if [ ${inode_quota: -1} = "T" ] ; then
        (( temp_quota = temp_quota * 1000000000000 ))
      elif [ ${inode_quota: -1} = "G" ] ; then
        (( temp_quota = temp_quota * 1000000000 ))
      elif [ ${inode_quota: -1} = "M" ] ; then
        (( temp_quota = temp_quota * 1000000 ))
      elif [ ${inode_quota: -1} = "k" ] ; then
        (( temp_quota = temp_quota * 1000 ))
      else
        (( temp_quota = inode_quota ))
      fi
    fi
    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
#    echo "inode ratio is " $quota_inode

    if [ ${quota_inode} -ge ${limit_quota} ] ; then
      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
      IGCM_debug_Print 1 "Use the ccc_quota command to check"
      IGCM_debug_Print 1 "You must have more than 10% available to run"
      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
      IGCM_debug_Verif_Exit
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_check_quota"
}

#D-#==================================================
#D-function IGCM_sys_GetJobID
#D-* Purpose: Check if job_name is currently
#D-  running or in queue
#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
#D-
function IGCM_sys_GetJobID {
  IGCM_debug_PushStack "IGCM_sys_GetJobID"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetJobID"
  fi

  # With -f option, the full job name is given in the last column
  ID=$( ccc_mstat -f -u $2 | \
        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )

  eval ${3}=${ID}

  IGCM_debug_PopStack "IGCM_sys_GetJobID"
}

#D-#==================================================
#D-function IGCM_sys_CountJobInQueue
#D-* Purpose: Check if job_name is currently
#D-  running or in queue
#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
#D-
function IGCM_sys_CountJobInQueue {
  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_CountJobInQueue"
  fi

  #NbRun=$( ccc_mstat -f | grep -c ${JobName} )

  # With -f option, the full job name is given in the last column
  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )

  eval ${2}=${NbRun}

  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
}

#D-#==================================================
#D-function IGCM_sys_ListJobInQueue
#D-* Purpose: Check if job_name is currently
#D-  running or in queue
#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
#D-
function IGCM_sys_ListJobInQueue {
  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ListJobInQueue"
  fi

  # With -f option, the full job name is given in the last column
  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
                                        '( $2  == User      && \
                                           $NF != /TS/      && \
                                           $NF !~ /PACK/    && \
                                           $NF !~ /REBUILD/ && \
                                           $NF !~ /pack/ )     \
                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )

  eval set -A ${2} ${JobList[*]}

  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
}

##############################################################
# NCO OPERATOR

#D-#==================================================
#D-function IGCM_sys_ncap2
#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncap2 {
  IGCM_debug_PushStack "IGCM_sys_ncap2" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncap2 :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncap2 -C "$@" > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncap2 : ncap2 error"
    IGCM_debug_Exit "ncap2"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncap2"
}

#D-#==================================================
#D-function IGCM_sys_ncatted
#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncatted {
  IGCM_debug_PushStack "IGCM_sys_ncatted" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncatted :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncatted "$@" > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncatted : ncatted error"
    IGCM_debug_Exit "ncatted"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncatted"
}

#D-#==================================================
#D-function IGCM_sys_ncbo
#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncbo {
  IGCM_debug_PushStack "IGCM_sys_ncbo" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncbo :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncbo -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncbo : ncbo error"
    IGCM_debug_Exit "ncbo"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncbo"
}

#D-#==================================================
#D-function IGCM_sys_ncdif
#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncdiff {
  IGCM_debug_PushStack "IGCM_sys_ncdiff" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncdiff :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncdiff -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncdiff : ncdiff error"
    IGCM_debug_Exit "ncdiff"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncdiff"
}

#D-#==================================================
#D-function IGCM_sys_ncea
#D-* Purpose: encapsulate ncea call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncea {
  IGCM_debug_PushStack "IGCM_sys_ncea" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncea :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncea -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncea : ncea error"
    IGCM_debug_Exit "ncea"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncea"
}

#D-#==================================================
#D-function IGCM_sys_ncecat
#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncecat {
  IGCM_debug_PushStack "IGCM_sys_ncecat" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncecat :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncecat -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncecat : ncecat error"
    IGCM_debug_Exit "ncecat"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncecat"
}

#D-#==================================================
#D-function IGCM_sys_ncflint
#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncflint {
  IGCM_debug_PushStack "IGCM_sys_ncflint" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncflint :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncflint -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncflint : ncflint error"
    IGCM_debug_Exit "ncflint"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncflint"
}

#D-#==================================================
#D-function IGCM_sys_ncks
#D-* Purpose: encapsulate ncks call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncks {
  IGCM_debug_PushStack "IGCM_sys_ncks" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncks :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncks -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncks : ncks error"
    IGCM_debug_Exit "ncks"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncks"
}

#D-#==================================================
#D-function IGCM_sys_ncpdq
#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncpdq {
  IGCM_debug_PushStack "IGCM_sys_ncpdq" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncpdq :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncpdq -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncpdq : ncpdq error"
    IGCM_debug_Exit "ncpdq"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncpdq"
}

#D-#==================================================
#D-function IGCM_sys_ncra
#D-* Purpose: encapsulate ncra call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncra {
  IGCM_debug_PushStack "IGCM_sys_ncra" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncra :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncra -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncra : ncra error"
    IGCM_debug_Exit "ncra"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncra"
}

#D-#==================================================
#D-function IGCM_sys_ncrcat
#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncrcat {
  IGCM_debug_PushStack "IGCM_sys_ncrcat" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncrcat :" $@
  fi

  typeset NB_ESSAI DELAI status i lastArg
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncrcat -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" /tmp/out_command.$$ )" = "X" ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
      cat /tmp/out_command.$$
      # remove files having corrupted time axis
      eval lastArg=\${$#}
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
      \rm ${lastArg}
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncrcat : ncrcat error"
    #IGCM_debug_Exit "ncrcat"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncrcat"
}

#D-#==================================================
#D-function IGCM_sys_ncrename
#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncrename {
  IGCM_debug_PushStack "IGCM_sys_ncrename" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncrename :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncrename $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncrename : ncrename error"
    IGCM_debug_Exit "ncrename"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncrename"
}

#D-#==================================================
#D-function IGCM_sys_ncwa
#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_ncwa {
  IGCM_debug_PushStack "IGCM_sys_ncwa" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncwa :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncwa -C $@ > /tmp/out_command.$$ 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
      cat /tmp/out_command.$$
      \rm /tmp/out_command.$$
      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm /tmp/out_command.$$
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_ncwa : ncwa error"
    IGCM_debug_Exit "ncwa"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncwa"
}

##############################################################
# CDO OPERATOR

#D-#==================================================
#D-function IGCM_sys_cdo
#D-* Purpose: encapsulate cdo call so as to manage error code and retry
#D-* Examples:
#D-
function IGCM_sys_cdo {
  IGCM_debug_PushStack "IGCM_sys_cdo" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_cdo :" $@
  fi

  typeset status

  \cdo $@ > /tmp/out_command.$$ 2>&1
  status=$?
  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_cdo : error code ${status}"
    cat /tmp/out_command.$$
    \rm /tmp/out_command.$$
    IGCM_debug_PopStack "IGCM_sys_cdo"
    return 1
  else
    IGCM_debug_PopStack "IGCM_sys_cdo"
    return 0
  fi

  IGCM_debug_PopStack "IGCM_sys_cdo"
}