source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_mercure.ksh @ 804

#!/bin/ksh

#**************************************************************
# Author: Sebastien Denvil, Martial Mancip
# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2006)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#=========================================================
# The documentation of this file can be automatically generated
# if you use the prefix #D- for comments to be extracted.
# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
#=========================================================

#D-#==================================================
#D-LibIGCM_sys for Mercure X64
#D-#==================================================
#D-
#D- This ksh library if a layer under some usefull
#D-environment variables and shell commands.
#D-All those definitions depend on host particularities.
#D-It manages a stack mechanism and test validity of operations.
#D-All function described bellow must be prefixed by IGCM_sys.

#====================================================
# libIGCM_sys PARAMETERS
#====================================================

#====================================================
# set DEBUG_sys to true to output calls of function
typeset -r DEBUG_sys=${DEBUG_sys:=true}

#====================================================
# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
typeset -r DRYRUN=${DRYRUN:=0}

# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
# -------------------------------------------------------------------------------------
# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
# |          |  Cp/Exe param files |            |  Chmod  |                           |
# |          |      Qsub           |            |         |                           |
# -------------------------------------------------------------------------------------
# |    0     |       yes           |    yes     |  yes    |      yes                  |
# -------------------------------------------------------------------------------------
# |    1     |       yes           |    yes     |  yes    |      no                   |
# -------------------------------------------------------------------------------------
# |    2     |       yes           |    yes     |  no     |      no                   |
# -------------------------------------------------------------------------------------
# |    3     |       yes           |    no      |  no     |      no                   |
# -------------------------------------------------------------------------------------

#=====================================================
# Global Variables :
#=====================================================
# Language : "fr" or "en"
typeset -r MYLANG="fr"

#=====================================================
# Host and user names
# $hostname ou hostname
typeset  HOST=${HOST:=$( hostname )}
# $username ou whoami
typeset  LOGIN=${LOGIN:=$( whoami )}

#D-
#D-#==================================================
#D-Program used in libIGCM
#D-#==================================================

# Submit command
typeset SUBMIT=${SUBMIT:=qsub}
# rsync with path
typeset -r RSYNC=/usr/bin/rsync
# RSYNC_opt args to rsync
typeset -r RSYNC_opt="-va"
# ie storage filesystem
typeset -r RHOST=mercure

#====================================================
# Set environment tools (ferret, nco, cdo)
#====================================================
. /home/cont003/p86ipsl/.atlas_env_mercure01_ksh

#====================================================
# AddNoise PATH
#====================================================
export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X86_64/bin

#====================================================
# Host specific DIRECTORIES
#====================================================

#====================================================
#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
typeset -r R_EXE="${MODIPSL}/bin"

#====================================================
# For interactive jobs on mercure
PBS_O_WORKDIR=${PBS_O_WORKDIR:=$(pwd)}

#====================================================
#- SUBMIT_DIR : submission dir
typeset SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}

#====================================================
#- IN
typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}

#====================================================
#- ARCHIVE (dedicated to large files)
typeset ARCHIVE=${CCCSTOREDIR}

#====================================================
#- STORAGE (dedicated to small/medium files)
typeset STORAGE=${CCCWORKDIR}

#====================================================
#- R_OUT
typeset R_OUT=${ARCHIVE}/IGCM_OUT

#====================================================
#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
typeset R_FIG=${STORAGE}/IGCM_OUT

#====================================================
#- R_BUF (ONLY FOR double copy an scratch)
typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT

#====================================================
#- RUN_DIR_PATH : Temporary working directory (=> TMP)
typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}

#====================================================
#- Max number of arguments passed to nco operator or demigration command
UNIX_MAX_LIMIT=360

#- set PackDefault true on NEC front-end
PackDefault=true

#D-#==================================================
#D-function IGCM_sys_ChangeArchive
#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
#D-* Examples:
#D-
function IGCM_sys_ChangeArchive {
  IGCM_debug_PushStack "IGCM_sys_ChangeArchive"

  R_OUT=${SCRATCHDIR}/IGCM_OUT
  R_FIG=${SCRATCHDIR}/IGCM_OUT
  IGCM_debug_Print 1 "R_OUT   has been redefined = ${R_OUT}"
  IGCM_debug_Print 1 "R_FIG   has been redefined = ${R_FIG}"

  IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
}

#D-#==================================================
#D-function IGCM_sys_RshMaster
#D-* Purpose: Just a fake command to wrapp
#D-           IGCM_card call in post-treatment
#D-           Ulam do not see brodie filesystem
#D-           Cesium do not see all mercure filesystem
#D-           That's why we need this hack.
#D-* Examples:
#D-
function IGCM_sys_RshMaster {
  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
  /bin/ksh <<-EOF
    export libIGCM=${libIGCM}
    export DEBUG_debug=${DEBUG_debug}
    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
    ${@}
EOF
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_RshMaster : erreur."
    IGCM_debug_Exit "IGCM_sys_RshMaster"
  fi
  IGCM_debug_PopStack "IGCM_sys_RshMaster"
}

#D-#==================================================
#D-function IGCM_sys_RshArchive
#D-* Purpose: Archive rsh command
#D-* Examples:
#D-
function IGCM_sys_RshArchive {
  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
  /bin/ksh <<-EOF
    ${@}
EOF
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_RshArchive : erreur."
    IGCM_debug_Exit "IGCM_sys_RshArchive"
  fi
  IGCM_debug_PopStack "IGCM_sys_RshArchive"
}

#D-#==================================================
#D-function IGCM_sys_RshPost
#D-* Purpose: Post-process rsh command
#D-* Examples:
#D-
function IGCM_sys_RshPost {
  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_RshPost :" $@
  fi
  /bin/ksh ${@}
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_RshPost : erreur."
    IGCM_debug_Exit "IGCM_sys_RshPost"
  fi
  IGCM_debug_PopStack "IGCM_sys_RshPost"
}

#D-#==================================================
#D-function IGCM_sys_SendMail
#D-* Purpose: Send mail when simulation is over
#D-* Examples:
#D-
function IGCM_sys_SendMail {
  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_SendMail :" $@
  fi

  if ( ${ExitFlag} ) ; then
    status=failed
  else
    status=completed
  fi

  cat  << END_MAIL > job_end.mail
Dear ${LOGIN},

  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
  Job started : ${DateBegin}
  Job ended   : ${DateEnd}
  Output files are available in ${R_SAVE}
  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
END_MAIL

  if  [ X"${config_UserChoices_MailName}" != X ] ; then
    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
  elif [ -f ~/.forward ] ; then
    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
  else
    mailx -s "${config_UserChoices_JobName} ${status}" ${LOGIN} < job_end.mail
  fi

  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_SendMail : erreur."
    IGCM_debug_Exit "IGCM_sys_SendMail"
  fi
  IGCM_debug_PopStack "IGCM_sys_SendMail"
}

#D-#==================================================
#D-function IGCM_sys_Mkdir
#D-* Purpose: Master locale mkdir command
#D-* Examples:
#D-
function IGCM_sys_Mkdir {
  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Mkdir :" $@
  fi
  if [ ! -d ${1} ]; then
    \mkdir -p $1
    if [ $? -gt 0 ] ; then
      echo "IGCM_sys_Mkdir : erreur."
      IGCM_debug_Exit "IGCM_sys_Mkdir"
    fi
  fi
  # vérification :
  if [ ! -d ${1} ] ; then
    echo "IGCM_sys_Mkdir : erreur."
    IGCM_debug_Exit "IGCM_sys_Mkdir"
  fi
  IGCM_debug_PopStack "IGCM_sys_Mkdir"
}

#D-#==================================================
#D-function IGCM_sys_MkdirArchive
#D-* Purpose: Mkdir on Archive
#D-* Examples:
#D-
function IGCM_sys_MkdirArchive {
  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_MkdirArchive :" $@
  fi
  #- creation de repertoire sur le serveur fichier
  if [ ! -d ${1} ]; then
    \mkdir -p $1
    if [ $? -gt 0 ] ; then
      echo "IGCM_sys_MkdirArchive : erreur."
      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
}

#D-#==================================================
#D-function IGCM_sys_MkdirWork
#D-* Purpose: Mkdir on Work
#D-* Examples:
#D-
function IGCM_sys_MkdirWork {
  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_MkdirWork :" $@
  fi
  #- creation de repertoire sur le serveur fichier
  if [ ! -d ${1} ]; then
    \mkdir -p $1
    if [ $? -gt 0 ] ; then
      echo "IGCM_sys_MkdirWork : erreur."
      IGCM_debug_Exit "IGCM_sys_MkdirWork"
    fi
  fi
  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
}

#D-#==================================================
#D-function IGCM_sys_Cd
#D-* Purpose: master cd command
#D-* Examples:
#D-
function IGCM_sys_Cd {
  IGCM_debug_PushStack "IGCM_sys_Cd" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Cd :" $@
  fi
  \cd $1
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_Cd : erreur."
    IGCM_debug_Exit "IGCM_sys_Cd"
  fi
  IGCM_debug_PopStack "IGCM_sys_Cd"
}

#D-#==================================================
#D-function IGCM_sys_Chmod
#D-* Purpose: Chmod
#D-* Examples:
#D-
function IGCM_sys_Chmod {
  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Chmod :" $@
  fi
  if [ $DRYRUN -le 1 ]; then
    \chmod $@
    if [ $? -gt 0 ] ; then
      echo "IGCM_sys_Chmod : erreur."
      IGCM_debug_Exit "IGCM_sys_Chmod"
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Chmod"
}

#D-#==================================================
#D-function IGCM_sys_FileSize
#D-* Purpose: Filesize
#D-* Examples:
#D-
function IGCM_sys_FileSize {
  IGCM_debug_PushStack "IGCM_sys_FileSize" $@

  typeset sizeF
  set +A sizeF -- $( ls -la ${1} )
  if [ $? -gt 0 ] ; then
    IGCM_debug_Exit "IGCM_sys_FileSize"
  fi
  eval ${2}=${sizeF[4]}

  IGCM_debug_PopStack "IGCM_sys_FileSize"
}

#D-#==================================================
#D-function IGCM_sys_TestDir
#D-* Purpose: Test Directory that must exists
#D-* Examples:
#D-
function IGCM_sys_TestDir {
  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_TestDir :" $@
  fi
  typeset ExistFlag
  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestDir"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_TestDirArchive
#D-* Purpose: Test Directory that must exists on Archive
#D-* Examples:
#D-
function IGCM_sys_TestDirArchive {
  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_TestDirArchive :" $@
  fi
  typeset ExistFlag
  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_IsFileArchived
#D-* Purpose: Test file that must NOT EXISTS on Archive
#D-* Examples:
#D-
function IGCM_sys_IsFileArchived {
  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_IsFileArchived :" $@
  fi
  typeset IsArchivedFlag
  IsArchivedFlag=$( [ X$( echo $1 | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"

  return ${IsArchivedFlag}
}

#D-#==================================================
#D-function IGCM_sys_TestFileArchive
#D-* Purpose: Test file that must NOT EXISTS on Archive
#D-* Examples:
#D-
function IGCM_sys_TestFileArchive {
  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
  typeset ExistFlag
  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_TestFileBuffer
#D-* Purpose: Test file that must NOT EXISTS on Buffer
#D-* Examples:
#D-
function IGCM_sys_TestFileBuffer {
  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
  typeset ExistFlag
  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"

  return ${ExistFlag}
}

#D-#==================================================
#D-function IGCM_sys_CountFileArchive
#D-* Purpose: Count files on Archive filesystem
#D-* Examples:
#D-
function IGCM_sys_CountFileArchive {
  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
  ls ${@} 2>/dev/null | wc -l
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_CountFileArchive : erreur."
  fi
  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
}

#D-#==================================================
#D-function IGCM_sys_CountFileBuffer
#D-* Purpose: Count files on Scratch filesystem
#D-* Examples:
#D-
function IGCM_sys_CountFileBuffer {
  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
  ls ${@} 2>/dev/null | wc -l
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_CountFileBuffer : erreur."
  fi
  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
}

#D-#==================================================
#D-function IGCM_sys_Tree
#D-* Purpose: Tree directories with files on ${ARCHIVE}
#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
#D-
function IGCM_sys_Tree {
  IGCM_debug_PushStack "IGCM_sys_Tree" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Tree :" $@
  fi

  \tree -f $@

  IGCM_debug_PopStack "IGCM_sys_Tree"
}

#D-#==================================================
#D-function IGCM_sys_Tar
#D-* Purpose: master tar command
#D-* Examples:
#D-
function IGCM_sys_Tar {
  IGCM_debug_PushStack "IGCM_sys_Tar" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Tar :" $@
  fi
  \tar cf $@
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_Tar : erreur."
    IGCM_debug_Exit "IGCM_sys_Tar"
  fi
  IGCM_debug_PopStack "IGCM_sys_Tar"
}

#D-#==================================================
#D-function IGCM_sys_UnTar
#D-* Purpose: master un-tar command
#D-* Examples:
#D-
function IGCM_sys_UnTar {
  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_UnTar :" $@
  fi
  \tar xvf $1
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_UnTar : erreur."
    IGCM_debug_Exit "IGCM_sys_UnTar"
  fi
  IGCM_debug_PopStack "IGCM_sys_UnTar"
}

#D-#==================================================
#D-function IGCM_sys_QsubPost
#D-* Purpose: Qsub new job on scalaire
#D-* Examples:
#D-
function IGCM_sys_QsubPost {
  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_QsubPost :" $@
  fi
  /usr/bin/nqsII/qsub -q scalaire -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM}/$1.job -v ${listVarEnv}
  if [ $? -gt 0 ] ; then
    echo "IGCM_sys_QsubPost : erreur " $@
    IGCM_debug_Exit "IGCM_sys_QsubPost"
  fi
  IGCM_debug_PopStack "IGCM_sys_QsubPost"
}

#D-*************************
#D- File transfer functions
#D-*************************
#D-

#D-#==================================================
#D-function IGCM_sys_Rsync_out
#D-* Purpose: treat return val of rsync
#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
#D-  Error values and explanations can depend on your system version.
function IGCM_sys_Rsync_out {
  status=$1
  if [ ! $status ] ; then
    echo "rsync error !"
  fi

  if [ $MYLANG = "fr" ]; then
    case $status in
    0)  return ;;
    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
      echo "Erreur de syntaxe ou d'utilisation."
      return;;
    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
      echo "Incompatibilité de protocole."
      return;;
    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
      echo "répertoires"
      return;;
    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
      echo "Action demandée non supportée : une tentative de manipulation de"
      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
      echo "été faite ; ou une option qui est supportée par le  client  mais"
      echo "pas par le serveur a été spécifiée."
      return;;
    10) echo "Erreur de rsync ; RERR_SOCKETIO"
      echo "Erreur dans le socket d'entrée sortie"
      return;;
    11) echo "Erreur de rsync ; RERR_FILEIO"
      echo "Erreur d'entrée sortie fichier"
      return;;
    12) echo "Erreur de rsync ; RERR_STREAMIO"
      echo "Erreur dans flux de donnée du protocole rsync"
      return;;
    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
      echo "Erreur avec les diagnostics du programme"
      return;;
    14) echo "Erreur de rsync ; RERR_IPC"
      echo "Erreur dans le code IPC"
      return;;
    20) echo "Erreur de rsync ; RERR_SIGNAL"
      echo "SIGUSR1 ou SIGINT reçu"
      return;;
    21) echo "Erreur de rsync ; RERR_WAITCHILD"
      echo "Une erreur retournée par waitpid()"
      return;;
    22) echo "Erreur de rsync ; RERR_MALLOC"
      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
      return;;
    23) echo ""
      echo "Erreur fichier inexistant"
      return;;
    30) echo "Erreur de rsync ; RERR_TIMEOUT"
      echo "Temps d'attente écoulé dans l'envoi/réception de données"
      return;;
    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
      return;;
    esac
  elif [ $MYLANG = "en" ] ; then
    case $status in
    0)  return;;
    1)  echo "rsync error : Syntax or usage error "
      return;;
    2)  echo "rsync error : Protocol incompatibility "
      return;;
    3)  echo "rsync error : Errors selecting input/output files, dirs"
      return;;
    4)  echo "rsync error : Requested action not supported: an attempt"
      echo "was made to manipulate 64-bit files on a platform that cannot support"
      echo "them; or an option was specified that is supported by the client and"
      echo "not by the server."
      return;;
    5)  echo "rsync error : Error starting client-server protocol"
      return;;
    10) echo "rsync error : Error in socket I/O "
      return;;
    11) echo "rsync error : Error in file I/O "
      return;;
    12) echo "rsync error : Error in rsync protocol data stream "
      return;;
    13) echo "rsync error : Errors with program diagnostics "
      return;;
    14) echo "rsync error : Error in IPC code "
      return;;
    20) echo "rsync error : Received SIGUSR1 or SIGINT "
      return;;
    21) echo "rsync error : Some error returned by waitpid() "
      return;;
    22) echo "rsync error : Error allocating core memory buffers "
      return;;
    23) echo "rsync error : Partial transfer due to error"
      return;;
    24) echo "rsync error : Partial transfer due to vanished source files"
      return;;
    30) echo "rsync error : Timeout in data send/receive "
      return;;
    *)  echo "rsync error : return code of rsync unknown :" $status
      return;;
    esac
  else
    echo "unknown language $MYLANG."
    return
  fi
}

#D-#==================================================
#D-function IGCM_sys_Cp
#D-* Purpose: generic cp
#D-* Examples:
#D-
function IGCM_sys_Cp {
  IGCM_debug_PushStack "IGCM_sys_Cp" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Cp :" $@
  fi

  typeset status

  echo cp $@ > out_rsync 2>&1
  \cp $@ >> out_rsync 2>&1
  status=$?

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_Cp : error code ${status}"
    cat out_rsync
    IGCM_debug_Exit "IGCM_sys_Cp"
  else
    \rm out_rsync
  fi
  IGCM_debug_PopStack "IGCM_sys_Cp"
}

#D-#==================================================
#D-function IGCM_sys_Rm
#D-* Purpose: generic rm
#D-* Examples:
#D-
function IGCM_sys_Rm {
  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Rm :" $@
  fi

  typeset status

  echo rm $@ > out_rsync 2>&1
  \rm $@ >> out_rsync 2>&1
  status=$?

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_Rm : error code ${status}"
    cat out_rsync
    IGCM_debug_Exit "IGCM_sys_Rm"
  else
    \rm out_rsync
  fi
  IGCM_debug_PopStack "IGCM_sys_Rm"
}

#D-#==================================================
#D-function IGCM_sys_RmRunDir
#D-* Purpose: rm tmpdir (dummy function most of the time batch
#D-                      scheduler will do the job)
#D-           Dummy function on mercure front-end
#D-* Examples:
#D-
function IGCM_sys_RmRunDir {
    IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
    if ( $DEBUG_sys ) ; then
        echo "IGCM_sys_RmRunDir :" $@
        echo "Dummy call, let the scheduler do that."
    fi
    IGCM_debug_PopStack "IGCM_sys_RmRunDir"
}

#D-#==================================================
#D-function IGCM_sys_Mv
#D-* Purpose: generic move
#D-* Examples:
#D-
function IGCM_sys_Mv {
  IGCM_debug_PushStack "IGCM_sys_Mv" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Mv :" $@
  fi

  if [ $DRYRUN = 0 ]; then

    typeset status

    echo mv $@ > out_rsync 2>&1
    \mv $@ >> out_rsync 2>&1
    status=$?

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Mv : error code ${status}"
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_Mv"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi

  IGCM_debug_PopStack "IGCM_sys_Mv"
}

#D-#==================================================
#D-function IGCM_sys_Put_Dir
#D-* Purpose: Copy a complete directory on $(ARCHIVE)
#D-* Examples:
#D-
function IGCM_sys_Put_Dir {
  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Dir :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if [ ! -d ${1} ] ; then
      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
      return
    fi

    typeset status

    # Only if we use rsync
    #IGCM_sys_TestDirArchive $( dirname $2 )
    #
    #USUAL WAY
    \cp -r $1 $2 > out_rsync 2>&1
    status=$?

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Dir : error code ${status}"
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_Put_Dir"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
}

#D-#==================================================
#D-function IGCM_sys_Get_Dir
#D-* Purpose: Copy a complete directory from ${ARCHIVE}
#D-* Examples:
#D-
function IGCM_sys_Get_Dir {
  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Get_Dir :" $@
  fi
  if [ $DRYRUN = 0 ]; then
#    if [ ! -d ${1} ] ; then
#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
#      return
#    fi

    typeset NB_ESSAI DELAI status i
    # number of tentative
    NB_ESSAI=3
    # time delay between tentative
    DELAI=2

    # Only if we use rsync
    #IGCM_sys_TestDirArchive $( dirname $2 )
    #
    # USUAL WAY
    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
    ccc_hsm get -r $1

    i=0
    while [ $i -lt $NB_ESSAI ] ; do
      \cp -ur $1 $2 >> out_rsync 2>&1
      status=$?
      if [ ${status} -gt 0 ]; then
        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
        sleep $DELAI
      else
        break
      fi
      (( i = i + 1 ))
    done

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Get_Dir : error."
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_Get_Dir"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
}

#D-#==================================================
#D-function IGCM_sys_Get_Master
#D-* Purpose: Copy a complete directory from MASTER filesystem
#D-* Examples:
#D-
function IGCM_sys_Get_Master {
  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Get_Master :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_Get_Master"
      return
    fi

    typeset NB_ESSAI DELAI status i
    # number of tentative
    NB_ESSAI=3
    # time delay between tentative
    DELAI=2

    i=0
    while [ $i -lt $NB_ESSAI ] ; do
      \cp -urL $1 $2 > out_rsync 2>&1
      status=$?
      if [ ${status} -gt 0 ]; then
        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
        sleep $DELAI
      else
        break
      fi
      (( i = i + 1 ))
    done

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Get_Master : error."
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_Get_Master"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Get_Master"
}

#D-#==================================================
#D-function IGCM_sys_Put_Rest
#D-* Purpose: Put computied restarts on ${ARCHIVE}.
#D-           File and target directory must exist.
#D-* Examples:
#D-
function IGCM_sys_Put_Rest {
  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Rest :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
      IGCM_debug_Exit "IGCM_sys_Put_Rest"
    fi

    typeset status
    #
    if [ X${JobType} = XRUN ] ; then
      IGCM_sys_Chmod 444 ${1}
    fi

    #
    # USUAL WAY
    \cp $1 $2 > out_rsync 2>&1
    status=$?

#       #RSYNC WITH NETWORK SSH CALL
#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1

#       #RSYNC WITH NFS USE
#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1

#       status=$?
#       IGCM_sys_Rsync_out $status

#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
#       (( status=status+$? ))

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_Put_Rest"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
}

#D-#==================================================
#D-function IGCM_sys_PutBuffer_Rest
#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
#D-           File and target directory must exist.
#D-* Examples:
#D-
function IGCM_sys_PutBuffer_Rest {
  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_PutBuffer_Rest :" $@
  fi
  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
    fi

    typeset status
    #
    if [ X${JobType} = XRUN ] ; then
      IGCM_sys_Chmod 444 ${1}
    fi

    #
    # USUAL WAY
    \cp $1 $2 > out_rsync 2>&1
    status=$?

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
}

#D-#==================================================
#D-function IGCM_sys_PrepareTaredRestart
#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
#D-* Examples:
#D-
function IGCM_sys_PrepareTaredRestart {
  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
  echo $1
  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
}

#D-#==================================================
#D-function IGCM_sys_Put_Out
#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
#D-* Examples:
#D-
function IGCM_sys_Put_Out {
  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Out :" $@
  fi

  typeset NB_ESSAI DELAI status i exist skip

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_Put_Out"
      return 1
    fi
    #
    IGCM_sys_MkdirArchive $( dirname $2 )
    #

    #=====================================================
    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
    #=====================================================

    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
    #if [ $? -eq 0 ] ; then
    #    typeset WORKPATH FILEPATH
    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
    #    IGCM_sys_MkdirWork ${WORKPATH}
    #    FILEPATH=${WORKPATH}/$( basename $2 )
    #    #
    #    IGCM_sys_Cp ${1} ${FILEPATH}
    #fi

    if [ X${JobType} = XRUN ] ; then
      if [ X${3} = X ] ; then
        IGCM_sys_Chmod 444 ${1}
      fi
    fi
    #

    exist=false
    skip=false
    if [ -f $2 ] ; then
      IGCM_debug_Print 1 "$2 already exist"
      #dmget $2
      ccc_hsm get $2
      exist=true
      if [ "X$( diff $1 $2 )" = X ] ; then
        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
        skip=true
      else
        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
        skip=false
      fi
    fi
    #
    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
      IGCM_sys_Chmod u+w $2
    fi

    if [ X${skip} = Xfalse ] ; then
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        # USUAL WAY
        \cp $1 $2 > out_rsync 2>&1
        status=$?
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
    fi

#       #RSYNC WITH NETWORK SSH CALL
#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1

#       #RSYNC WITH NFS USE
#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1

#       status=$?
#       IGCM_sys_Rsync_out $status

#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
#       (( status=status+$? ))

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Out : error."
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_Put_Out"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Out"
  return 0
}

#D-#==================================================
#D-function IGCM_sys_PutBuffer_Out
#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
#D-* Examples:
#D-
function IGCM_sys_PutBuffer_Out {
  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_PutBuffer_Out :" $@
  fi

  typeset NB_ESSAI DELAI status i exist skip 

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN = 0 ]; then
    if [ ! -f ${1} ] ; then
      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
      return 1
    fi
    #
    IGCM_sys_Mkdir $( dirname $2 )
    #

    if [ X${JobType} = XRUN ] ; then
      if [ X${3} = X ] ; then
        IGCM_sys_Chmod 444 ${1}
      fi
    fi

    exist=false
    skip=false
    if [ -f $2 ] ; then
      IGCM_debug_Print 1 "$2 already exist"
      exist=true
      if [ "X$( diff $1 $2 )" = X ] ; then
        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
        skip=true
      else
        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
        skip=false
      fi
    fi
    #
    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
      IGCM_sys_Chmod u+w $2
    fi

    if [ X${skip} = Xfalse ] ; then
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        # USUAL WAY
        #\cp $1 $2 > out_rsync 2>&1
        \mv $1 $2 > out_rsync 2>&1
        status=$?
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
    fi

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_PutBuffer_Out : error."
      cat out_rsync
      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
    else
      \rm out_rsync
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
  return 0
}

#D-#==================================================
#D-function IGCM_sys_Get
#D-* Purpose: Get a file from ${ARCHIVE}
#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
function IGCM_sys_Get {
  IGCM_debug_PushStack "IGCM_sys_Get" $@

  typeset DEST dm_liste ifile target file_work
  typeset NB_ESSAI DELAI status i

  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Get :" $@
  fi

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN -le 2 ]; then
    if [ X${1} = X'/l' ] ; then
      # test if the first file is present in the old computation :
      eval set +A dm_liste \${${2}}
    else
      eval set +A dm_liste ${1}
    fi
    eval DEST=\${${#}}

    #=====================================================
    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
    #=====================================================

    # Is it an R_OUT file (not R_IN) ?
    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
    #if [ $? -eq 0 ] ; then
    #    # Yes  ? then we try to get it in SCRATCHDIR
    #    set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
    #    if [ -f ${file_work[0]} ] ; then
    #   IGCM_sys_Cp ${file_work[*]} ${DEST}
    #   IGCM_debug_PopStack "IGCM_sys_Get"
    #   return
    #    fi
    #fi

    ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      echo "WARNING IGCM_sys_Get : error code ${status}"
      cat out_rsync
      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
    fi

    #if [ ${status} -gt 0 ] ; then
    #    if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
    #   cat out_rsync
    #   echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
    #   sleep 30
    #   echo "We try another time"
    ##  dmget ${dm_liste[*]} > out_rsync 2>&1
    #   ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
    #   status=$?
    #   if [ ${status} -gt 0 ] ; then
    #       echo "ERROR IGCM_sys_Get : again demigration error :"
    #       cat out_rsync
    #       IGCM_debug_Exit "IGCM_sys_Get"
    #   fi
    #    else
    #   echo "ERROR IGCM_sys_Get : demigration error :"
    #   cat out_rsync
    #   IGCM_debug_Exit "IGCM_sys_Get"
    #    fi
    #fi

    #   #RSYNC WITH NETWORK SSH CALL
    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1

    #   #RSYNC WITH NFS USE
    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1

    #   status=$?
    #   IGCM_sys_Rsync_out $status

    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
    #   (( status=status+$? ))

    #USUAL WAY
    if [ X${1} = X'/l' ] ; then
      for target in ${dm_liste[*]} ; do
        local_file=$( basename ${target} )
        # test if the target file is present before the loop
        IGCM_sys_TestFileArchive ${target}
        status=$?
        if [ ${status} -gt 0 ] ; then
          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
          IGCM_debug_Exit "IGCM_sys_Get"
        else
          i=0
          while [ $i -lt $NB_ESSAI ] ; do
            \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
            status=$?
            if [ ${status} -gt 0 ]; then
              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
              sleep $DELAI
            else
              break
            fi
            (( i = i + 1 ))
          done
          if [ ${status} -gt 0 ] ; then
            echo "IGCM_sys_Get : error"
            cat out_rsync
            \rm out_rsync
            IGCM_debug_Exit "IGCM_sys_Get"
          else
            \rm out_rsync
          fi
        fi
      done
    else
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        \cp ${dm_liste} ${DEST} >> out_rsync 2>&1
        status=$?
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
      if [ ${status} -gt 0 ] ; then
        echo "IGCM_sys_Get : error"
        cat out_rsync
        \rm out_rsync
        IGCM_debug_Exit "IGCM_sys_Get"
      else
        \rm out_rsync
      fi
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Get"
}

#D-#==================================================
#D-function IGCM_sys_GetBuffer
#D-* Purpose: Get a file from ${SCRATCHDIR}
#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
function IGCM_sys_GetBuffer {
  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@

  typeset DEST buf_liste ifile target file_work
  typeset NB_ESSAI DELAI status i

  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetBuffer :" $@
  fi

  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  if [ $DRYRUN -le 2 ]; then
    if [ X${1} = X'/l' ] ; then
      # test if the first file is present in the old computation :
      eval set +A buf_liste \${${2}}
    else
      eval set +A buf_liste ${1}
    fi
    eval DEST=\${${#}}

    #USUAL WAY
    if [ X${1} = X'/l' ] ; then
      for target in ${buf_liste[*]} ; do
        local_file=$( basename ${target} )
        i=0
        while [ $i -lt $NB_ESSAI ] ; do
          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
          status=$?
          if [ ${status} -gt 0 ]; then
            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
            sleep $DELAI
          else
            break
          fi
          (( i = i + 1 ))
        done
        if [ ${status} -gt 0 ] ; then
          echo "IGCM_sys_Get : error"
          cat out_rsync
          \rm out_rsync
          IGCM_debug_Exit "IGCM_sys_GetBuffer"
        else
          \rm out_rsync
        fi
      done
    else
      i=0
      while [ $i -lt $NB_ESSAI ] ; do
        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
        status=$?
        if [ ${status} -gt 0 ]; then
          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
          sleep $DELAI
        else
          break
        fi
        (( i = i + 1 ))
      done
      if [ ${status} -gt 0 ] ; then
        echo "IGCM_sys_Get : error"
        cat out_rsync
        \rm out_rsync
        IGCM_debug_Exit "IGCM_sys_GetBuffer"
      else
        \rm out_rsync
      fi
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
}

#D-#==================================================
#D-function IGCM_sys_GetDate_FichWork
#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
#D-* Examples:
#D-
function IGCM_sys_GetDate_FichWork {
  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetDate_FichWork :" $@
  fi
  # donne la date filesys d'un fichier sur la machine work
  IGCM_debug_PopStack "IGCM_sys_FichWork"
}

#D-#==================================================
#D-function IGCM_sys_GetDate_FichArchive
#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
#D-* Examples:
#D-
function IGCM_sys_GetDate_FichArchive {
  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_GetDate_FichArchive :" $@
  fi
  IGCM_debug_PopStack "IGCM_sys_FichArchive"
}

#D-#==================================================
#D-function IGCM_sys_Dods_Rm
#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
#D-* Examples:
#D-
function IGCM_sys_Dods_Rm {
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Dods_Rm :" $@
  fi
  typeset status
  if [ $DRYRUN = 0 ]; then

#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
#      echo "Nothing has been done."
#      return
#    fi

    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
    status=$?

#       if [ ${status} -gt 0 ] ; then
#           echo "IGCM_sys_Dods_Rm : error."
#           cat out_dods_rm
#           IGCM_debug_Exit "IGCM_sys_Dods_Rm"
#       else
#           rm out_dods_rm
#       fi

  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  return $status
}

#D-#==================================================
#D-function IGCM_sys_Dods_Cp
#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
#D-* Examples:
#D-
function IGCM_sys_Dods_Cp {
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Dods_Cp :" $@
  fi
  typeset status
  if [ $DRYRUN = 0 ]; then

#    if [ ! -d ${R_SAVE}/${1} ] ; then
#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
#      echo "Nothing has been done."
#      return
#    fi

    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
    status=$?

#       if [ ${status} -gt 0 ] ; then
#           echo "IGCM_sys_Dods_Cp : error."
#           cat out_dods_cp
#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
#       else
#           rm out_dods_cp
#       fi

  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  return $status
}

#D-#==================================================
#D-function IGCM_sys_Put_Dods
#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
#D-* Examples:
#D-
function IGCM_sys_Put_Dods {
  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_Put_Dods :" $@
  fi
  #set -vx
  typeset status
  if [ $DRYRUN = 0 ]; then
    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
      return
    fi

    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
      return
    fi
    #
    if [ -d ${R_SAVE}/${1} ] ; then
      cd ${R_SAVE}
    elif [ -d ${R_FIGR}/${1} ] ; then
      cd ${R_FIGR}
    fi

    IGCM_sys_Dods_Rm ${1}
    IGCM_sys_Dods_Cp ${1}
    status=0

    if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_Put_Dods : error."
      IGCM_debug_Exit "IGCM_sys_Put_Dods"
    fi
  else
    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
  fi
  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
}

##############################################################
# REBUILD OPERATOR

function IGCM_sys_rebuild {
  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_rebuild :" $@
  fi

  typeset NB_ESSAI DELAI status i firstArg
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    /home/cont003/p86ipsl/X86_64/bin/rebuild -f -o $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      firstArg=${1}
      \rm ${firstArg}
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_rebuild : rebuild error"
    IGCM_debug_Exit "rebuild"
  fi

  IGCM_debug_PopStack "IGCM_sys_rebuild"
}

function IGCM_sys_rebuild_station {
  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
  typeset i list_opt file_in file_out prefix_invert list_invert
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_rebuild_station :" $@
  fi
  list_opt=$@

  # Invert Axis : t,x -> x,t 
  #               t,pres,x -> x,t,pres
  # So that we can concatenate along x
  i=0
  for file_in in ${list_opt} ; do
    (( i = i + 1))
    [ ${i} = 1 ] && file_out=${file_in} && continue
    prefix_invert=$( basename ${file_in} .nc )
    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
  done

  # Concatenate
  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc

  # Re-ivert file
  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}

  # Station re-ordering is too expansive to be run within libICGM
  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
  # This re-ordering must be done "in memory" by the cmorization process
  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
  # BEGIN reordering

  # Only LMDZ text output contains the exact ordering of the station.
  # We isolate this in the code below:
  #  0  38  -157.5000000000000  70.98591549295774
  #  0  54  27.49999999999999   67.18309859154928
  #  0  56  -62.50000000000001  82.39436619718309
  #  0  79  12.49999999999999   78.59154929577466
  #  0  116 -165.0000000000000  76.05633802816901
  #  0  117 130.0000000000000   70.98591549295774
  #  0  118 110.0000000000000   87.46478873239437
  #  1  40  4.999999999999995   51.97183098591550
#  typeset iStation iProc list_opt file_in file_out prefix_invert
#  typeset -Z4 j4
#  typeset -Z3 j3

#  unset list_opt
#  set +A list_opt $@

  # Filename after rebuild
#  file_out=${list_opt[0]}
  # Prefix of output files
#  prefix_invert=$( basename ${file_out} .nc )
  # Number of procs
#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )

#  iProc=0
#  while [ ${iProc} -lt ${num_proc} ] ; do
    # Array containing Station as a number
#    unset proc_stn
#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | awk ' {print $2}' )
    # Number of stations produced by processor proc
#    stationLast=${#proc_stn[*]}
    # Proc number on 4 digits
#    j4=${iProc}
    # Init
#    iStation=0
#    while [ ${iStation} -lt ${stationLast} ] ; do
      # Station number on 3 digits
#      j3=${proc_stn[${iStation}]}
      # Extract station
      # Invert Axis : t,x -> x,t 
      #               t,pres,x -> x,t,pres
      # So that we can concatenate along x
#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
#      (( iStation = iStation + 1 ))
#    done
#    (( iProc = iProc + 1 ))
#  done

  # Concatenate all station along x
#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc

  # Re-invert file
#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}

  # END reordering

  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
}

############################################################
# Check of space available on temporary filesytems
function IGCM_sys_check_quota {
    IGCM_debug_PushStack "IGCM_sys_check_quota"
    if ( $DEBUG_sys ) ; then
        echo "IGCM_sys_check_quota"
    fi
    IGCM_debug_PopStack "IGCM_sys_check_quota"
}

##############################################################
# NCO OPERATOR

function IGCM_sys_ncap2 {
  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncap2 :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncap2 "$@" > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncap2 : ncap2 error"
      IGCM_debug_Exit "ncap2"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncap2"
}

function IGCM_sys_ncatted {
  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncatted :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncatted "$@" > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncatted : ncatted error"
      IGCM_debug_Exit "ncatted"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncatted"
}

function IGCM_sys_ncbo {
  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncbo :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncbo $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncbo : ncbo error"
      IGCM_debug_Exit "ncbo"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncbo"
}

function IGCM_sys_ncdiff {
  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncdiff :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncdiff $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncdiff : ncdiff error"
      IGCM_debug_Exit "ncdiff"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncdiff"
}

function IGCM_sys_ncea {
  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncea :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncea $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncea : ncea error"
      IGCM_debug_Exit "ncea"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncea"
}

function IGCM_sys_ncecat {
  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncecat :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncecat $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncecat : ncecat error"
      IGCM_debug_Exit "ncecat"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncecat"
}

function IGCM_sys_ncflint {
  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncflint :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncflint $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncflint : ncflint error"
      IGCM_debug_Exit "ncflint"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncflint"
}

function IGCM_sys_ncks {
  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncks :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncks $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncks : ncks error"
      IGCM_debug_Exit "ncks"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncks"
}

function IGCM_sys_ncpdq {
  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncpdq :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncpdq $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncpdq : ncpdq error"
      IGCM_debug_Exit "ncpdq"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncpdq"
}

function IGCM_sys_ncra {
  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncra :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncra $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncra : ncra error"
      IGCM_debug_Exit "ncra"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncra"
}

function IGCM_sys_ncrcat {
  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncrcat :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncrcat $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncrcat : ncrcat error"
      #IGCM_debug_Exit "ncrcat"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncrcat"
}

function IGCM_sys_ncrename {
  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncrename :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncrename $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncrename : ncrename error"
      IGCM_debug_Exit "ncrename"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncrename"
}

function IGCM_sys_ncwa {
  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_ncwa :" $@
  fi

  typeset NB_ESSAI DELAI status i
  # number of tentative
  NB_ESSAI=3
  # time delay between tentative
  DELAI=2

  i=0
  while [ $i -lt $NB_ESSAI ] ; do
    ncwa $@ > out_rsync 2>&1
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
      cat out_rsync
      \rm out_rsync
      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
      sleep $DELAI
    else
      \rm out_rsync
      break
    fi
    (( i = i + 1 ))
  done

  if [ ${status} -gt 0 ] ; then
      echo "IGCM_sys_ncwa : ncwa error"
      IGCM_debug_Exit "ncwa"
  fi

  IGCM_debug_PopStack "IGCM_sys_ncwa"
}

##############################################################
# CDO OPERATOR

function IGCM_sys_cdo {
  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_cdo :" $@
  fi

  typeset status

  \cdo $@ > out_rsync 2>&1
  status=$?
  if [ ${status} -gt 0 ] ; then
    echo "IGCM_sys_cdo : error code ${status}"
    cat out_rsync
    \rm out_rsync
    IGCM_debug_PopStack "IGCM_sys_cdo"
    return 1
  else
    IGCM_debug_PopStack "IGCM_sys_cdo"
    return 0
  fi

  IGCM_debug_PopStack "IGCM_sys_cdo"
}

############################################################
# Activate Running Environnment Variables

function IGCM_sys_activ_variables {
  IGCM_debug_PushStack "IGCM_sys_activ_variables"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_activ_variables"
  fi
  IGCM_debug_PopStack "IGCM_sys_activ_variables"
}

############################################################
# Desactivate Running Environnment Variables

function IGCM_sys_desactiv_variables {
  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_desactiv_variables"
  fi
  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
}

############################################################
# Build run file

function IGCM_sys_build_run_file {
  IGCM_debug_PushStack "IGCM_sys_build_run_file"
  if ( $DEBUG_sys ) ; then
    echo "IGCM_sys_build_run_file"
  fi
  IGCM_debug_PopStack "IGCM_sys_build_run_file"
}