source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 917

#!/bin/ksh

#**************************************************************
# Author: Patrick Brockmann, Martial Mancip
# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2006)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#==================================================
# The documentation of this file can be automatically generated
# if you use the prefix #D- for comments to be extracted.
# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
#==================================================

#==================================================
# Add high level verbosity
typeset -i Verbosity=${Verbosity:=3}

#==================================================
# DEBUG_debug
# Add low level verbosity
typeset DEBUG_debug=${DEBUG_debug:=false}

#==================================================
# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
typeset RandomError=false

# Where the stack file containing call tree will be stored.
typeset StackFileLocation=${StackFileLocation:=${PWD}}

if ( $DEBUG_debug ) ; then
  if [ -f ${StackFileLocation}/stack ] ;
  then
    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack
  else
    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/stack
  fi
fi

#==================================================
# NULL_STR
# Default null string
typeset -r NULL_STR="_0_"

#==================================================
# libIGCM_CurrentTag
# Current libIGCM tag, check compatibilty with *.card
typeset -r libIGCM_CurrentTag="1.0"

#==================================================
# Exit Flag (internal debug)
# When true, end the master loop AFTER SAVES FILES
ExitFlag=false

#==================================================
# Declare a stack of functions calls
unset IGCM_debug_Stack
unset IGCM_debug_StackArgs
unset IGCM_debug_StackTiming
IGCM_debug_Stack[0]=${NULL_STR}
IGCM_debug_StackArgs[0]=${NULL_STR}
IGCM_debug_StackTiming[0]=${NULL_STR}
IGCM_debug_LenStack=0

#D-#==================================================================
#D-function IGCM_debug_getDate_ms
#D- * Purpose: Give number of milliseconds since 01-jan-1970
function IGCM_debug_getDate_ms
{
  typeset nanosecs testnum
  # nano secondes since 01-jan-1970
  nanosecs=$( date +%s%N )

  # truncate the last 6 digits to get milliseconds since 01-jan-1970
  ms=${nanosecs:0:${#nanosecs}-6}

  echo "$ms"
}

#D-#==================================================================
#D-function IGCM_debug_sizeOfTabContent
#D- * Purpose: Give sumed size of a list of files
function IGCM_debug_sizeOfTabContent
{
  typeset dm_liste ifile sizeFile sizeKo sizeMo
  eval set +A dm_liste \${${1}}
  sizeKo=0
  for ((i = 0; i < ${#dm_liste[*]}; i += 1)) ; do
    # echo "dm_liste[$i]=${dm_liste[$i]}"
    # we want to compute the size from local path (not from archive ...)
    # remove path /home/login/../ from dm_liste elements
    if [ -f ${dm_liste[$i]##/*/} ] ; then
      ifile=${dm_liste[$i]##/*/}
    elif [ -f ${2} ] ; then
      ifile=${2}
    elif [ -f $2/${dm_liste[$i]##/*/} ] ; then
      ifile=${2}/${dm_liste[$i]##/*/}
    fi
    sizeFile=$( du --apparent-size -k ${ifile} | awk {'print $1'} )
    sizeKo=$(( $sizeKo + $sizeFile ))
  done
  sizeMo=$( echo "scale=6;${sizeKo}/1024" | bc )
  echo "${sizeKo}|${sizeMo}"
}

#D-#==================================================================
#D-function IGCM_debug_SendAMQP
#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
function IGCM_debug_sendAMQP
{
  typeset decal first additionnalOption encodedBody

  # Encode message Body
  encodedBody=$( echo "${Body}" | base64 -w 0 )

  # Send config.card ?
  if [ X${1} = Xactivate ] ; then
    # Encode config.card
    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
    # Prepare additionnal option
    additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
  else
    additionnalOption=
  fi

  # Only cosmetics : stack file
  decal=0
  while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
    printf ' ' >> ${StackFileLocation}/stack
    (( decal = decal + 1 ))
  done
  # Log to stack file
  echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b "${Body}"      >> ${StackFileLocation}/stack

  # Log separately AMQP send message command
  echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt

  # Send the message
  sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
  status=$?
  if [ ${status} -gt 0 ] ; then
    IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
    echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
    exit
  fi
}

#D-#==================================================================
#D-function IGCM_debug_CallStack
#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
#D-
function IGCM_debug_CallStack {
  if ( $DEBUG_debug ) ; then
    # Cosmetics
    typeset i decal
    i=0
    until [ $i -eq ${IGCM_debug_LenStack} ]; do
      decal=0
      until [ $decal -eq ${i} ]; do
        printf -- ' '
        (( decal = decal + 1 ))
      done
      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
      ((i = i + 1))
    done
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PushStack
#D-* Purpose: Push a function name in the stack
#D-
function IGCM_debug_PushStack {
  if ( $DEBUG_debug ) ; then
    typeset decal inputs startTime_ms
    echo >> ${StackFileLocation}/stack

    # Only cosmetics : stack file
    decal=0
    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
      printf ' ' >> ${StackFileLocation}/stack
      (( decal = decal + 1 ))
    done

    # Save input list in an indexed array
    INPUTS=( $@ )

    # Get timing information
    startTime_ms=$( IGCM_debug_getDate_ms )

    # We add function call name on beginning of the stack
    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}

    # Save timing in milliseconds in an indexed array
    set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}

    # We include the "null" Args in the beginning of the StackArgs
    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}

    # Then, we shift StackArgs tabular
    if [ $# -gt 1 ]; then
      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g" )
    fi

    # Fill the stack file
    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack

    if [ X${ActivateBigBro} = Xtrue ] ; then
      # RabbitMQ message code
      code=2000
      # RabbitMQ message body
      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
      # Fill the rabbitMQ queue
      IGCM_debug_sendAMQP
    fi

    # Increment LenStack
    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))

    #IGCM_debug_CallStack
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PopStack
#D-* Purpose: Pop a function name in the stack
#D-
function IGCM_debug_PopStack {
  if ( $DEBUG_debug ) ; then
    typeset decal command arguments startTime_ms endTime_ms
    typeset instrumentation source dest

    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
    if ( ${RandomError} ) ; then
      if [ $((RANDOM%10000)) -le 10 ] ; then
        IGCM_debug_Print 1 "Random error has been triggered"
        echo "RANDOM ERROR" >> ${StackFileLocation}/stack
        ExitFlag=true
      fi
    fi

    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
      # Everything is cool

      # Get timing information
      endTime_ms=$( IGCM_debug_getDate_ms )

      # Save Stack information before poping the stack
      command=${IGCM_debug_Stack[0]}

      # Go from comma separated (list) to space separated in an array
      set -A arguments $( echo ${IGCM_debug_StackArgs[0]} | sed -e "s/,/\ /g" )

      # Save Stack information before poping the stack
      startTime_ms=${IGCM_debug_StackTiming[0]}

      # Pop the stack
      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
      set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
    else
      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
      IGCM_debug_Exit $@
    fi

    # Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
    # We are interested in:
    #  0. Which command performs the work
    #  1. Size of entity we are working with
    #  2. Where are we reading
    #  3. Where are we writing
    #  4. How long it took

    instrumentation=false

    case ${command} in
    # Copy from archive machine or from buffer
    IGCM_sys_Get|IGCM_sys_GetBuffer)
      instrumentation=true
      if [ ${#arguments[*]} -eq 2 ] ; then
        source=${arguments[0]}
        dest=${arguments[1]}
        # Size of file whose name are stored in a variable
        entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
      elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
        # Keep the array name hosting the all list
        fileList=${arguments[0]}
        # just need the first file to get the directory
        eval source=\${${arguments[0]}[0]}
        dest=${arguments[1]}
        # Size of file whose name are stored in an array
        entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
      elif [ [ ${#arguments[*]} -ge 3 ] ; then
       # All but the latest
        fileList=${arguments[*]:0:${#arguments[*]}-1}
        # just need the first file to get the directory
        source=${arguments[0]}
        # Nothing but the latest
        dest=${arguments[${#arguments[*]}-1]}
        # Size of file whose name are stored in a list
        entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
      fi
      ;;

    # Classical copy (only files are given to IGCM_sys_Cp as options)
    IGCM_sys_Cp)
      instrumentation=true      
      # All but the latest
      fileList=${arguments[*]:0:${#arguments[*]}-1}
      # just need the first file to get the directory
      source=${arguments[0]}
      # Nothing but the latest
      dest=${arguments[${#arguments[*]}-1]}
      # Size of file whose name are stored in a list
      entitySize=$( IGCM_debug_sizeOfTabContent fileList $dest )
      ;;

    # Copy from compute node
    IGCM_sys_Get_Master|IGCM_sys_Get_Dir)
      instrumentation=true
      if [ ${#arguments[*]} -eq 2 ] ; then
        source=${arguments[0]}
        dest=${arguments[1]}
        # Size of file whose name are stored in a variable
        entitySize=$( IGCM_debug_sizeOfTabContent source )
      fi
      ;;

    # Rebuild command
    IGCM_sys_rebuild|IGCM_sys_rebuild_station)
      instrumentation=true
      # All but the first
      fileList=${arguments[*]:1:${#arguments[*]}-1}
      # just need a file to get the directory
      source=${arguments[1]}
      # Nothing but the first
      dest=${arguments[0]}
      # Size of file whose name are stored in a list
      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
      ;;
    esac

    # Print information related to instrumentation
    ( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}

    # Only cosmetics : stack file
    decal=0
    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
      printf ' ' >> ${StackFileLocation}/stack
      (( decal = decal + 1 ))
    done

    if ( ${ExitFlag} ) ; then
      # Inform the stack file
      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack

      if [ X${ActivateBigBro} = Xtrue ] ; then
        # RabbitMQ message code
        code=9000
        # RabbitMQ message body
        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
        # Fill the rabbitMQ queue
        IGCM_debug_sendAMQP
      fi
    else
      # Inform the stack file
      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
      if [ X${ActivateBigBro} = Xtrue ] ; then
        # RabbitMQ message code
        code=3000
        # RabbitMQ message body
        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
        # Fill the rabbitMQ queue
        IGCM_debug_sendAMQP
      fi
    fi

    # Reset array if necessary
    if [ ${IGCM_debug_LenStack} = 0 ]; then
      #echo
      #IGCM_debug_Print 3 "Clean stack array"
      #echo
      unset IGCM_debug_Stack
      unset IGCM_debug_StackArgs
      unset IGCM_debug_StackTiming
      IGCM_debug_Stack[0]=${NULL_STR}
      IGCM_debug_StackArgs[0]=${NULL_STR}
      IGCM_debug_StackTiming[0]=${NULL_STR}
    fi
  fi
  #IGCM_debug_CallStack
}

#D-#==================================================================
#D-function IGCM_debug_ActivateBigBro
#D-* Purpose: switch rabbitMQ on
#D-
function IGCM_debug_ActivateBigBro {
  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"

  if [ X${BigBrother} = Xtrue ] ; then
    # ID to identify a simulation
    simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE

    # ID to identify a job. Several Jobs are needed to complete a simulation
    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}

    # RabbitMQ message code
    if ( ${FirstInitialize} ) ; then
      code=0000
    else
      code=1000
    fi
    # RabbitMQ message body
    Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
    # Fill the rabbitMQ queue (specifying config.card must be send)
    IGCM_debug_sendAMQP activate

    # Turn the flag on
    ActivateBigBro=true
  fi
  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
}

#D-#==================================================================
#D-function IGCM_debug_Exit
#D-* Purpose: Print Call Stack and set ExitFlag to true
#D-
function IGCM_debug_Exit {
  IGCM_debug_PushStack "IGCM_debug_Exit"
  echo "IGCM_debug_Exit : " "${@}"
  echo
  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
  echo "!!   ERROR TRIGGERED    !!"
  echo "!!   EXIT FLAG SET      !!"
  echo "!------------------------!"
  echo
  IGCM_debug_CallStack
  ExitFlag=true
  IGCM_debug_PopStack "IGCM_debug_Exit"
}

#D-#==================================================
#D-function IGCM_debug_Verif_Exit
#D-* Purpose: exit with number 1 if ExitFlag is true
#D-
function IGCM_debug_Verif_Exit {
  if ( ${ExitFlag} ) ; then
    # Plan to send an email here with IGCM_sys_SendMail
    if [ X${TaskType} != Xchecking ] ; then
      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
      echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
      echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
      echo "                        EXIT THE JOB."
      echo
      IGCM_debug_CallStack
    fi

    if [ X${ActivateBigBro} = Xtrue ] ; then
      # RabbitMQ message code
      code=9999
      # RabbitMQ message body
      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
      # Fill the rabbitMQ queue
      IGCM_debug_sendAMQP
    fi

    # Not sure about that one ...
    if ( $DEBUG_debug ) ; then
      echo "Your files on ${R_OUT} :"
      IGCM_sys_Tree ${R_SAVE}
      echo
    fi

    # Mail notification
    IGCM_sys_SendMail

    # And Good Bye
    date
    exit 1
  fi
}

#D-#==================================================
#D-function IGCM_debug_Verif_Exit_Post
#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
#D-
function IGCM_debug_Verif_Exit_Post {
  if ( ${ExitFlag} ) ; then
    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
    # If SpaceName is PROD then we stop if post_processing fails
    # Plan to send an email here with IGCM_sys_SendMail
    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
      echo "                        EXIT THE JOB."
      echo
      # Mail notification
      #IGCM_sys_SendMailPost
      # And Good Bye
      date
      exit 1
    else
      echo "Either inside config.card the variable SpaceName is not in PROD"
      echo "or inside the main Job the variable JobType is not in RUN mode"
      echo "              SO WE DO NOT EXIT THE JOB."
      echo
      date
    fi
  fi
}

#D-#==================================================================
#D-function IGCM_debug_Print
#D-* Purpose: Print arguments according to a level of verbosity.
#D-
function IGCM_debug_Print
{
  typeset level=$1
  shift

  if [ X"${1}" = X"-e" ]; then
    typeset cmd_echo="echo -e"
    shift
  else
    typeset cmd_echo="echo"
  fi

  if [ ${level} -le ${Verbosity} ] ; then
    typeset i
    case "${level}" in
    1) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
      done ;;
    2) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
      done ;;
    3) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
      done ;;
    esac
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PrintVariables
#D-* Purpose: Print arguments when match a pattern
#D-           according to a level of verbosity.
function IGCM_debug_PrintVariables
{
  typeset level=$1
  shift

  list=$( set | grep ^$1 | sed -e "s/'//g" )

  if [ "X${list}" != X ]  ; then
    IGCM_debug_Print ${level} ${list}
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PrintInfosActions
#D-* Purpose: Print information related to instrumentation
function IGCM_debug_PrintInfosActions
{
  typeset actionType=$1
  typeset entitySize=$2
  typeset start_ms=$3
  typeset end_ms=$4

  typeset dest=$5
  typeset source=$6

  typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
  typeset name dirOut dirSource

  name="doNotKnow"

  diff_ms=$(( $end_ms - $start_ms ))
  # echo "diff_ms=$diff_ms"

  entitySizeKo=$( echo ${entitySize} | gawk -F"|" '{print $1}' )
  # echo "entitySizeKo=$entitySizeKo"
  entitySizeMo=$( echo ${entitySize} | gawk -F"|" '{print $2}' )

  # flux en Ko / ms
  flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" | bc )
  # echo "flux_Ko_ms=$flux_Ko_ms"

  # flux en Ko / s
  flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
  # echo "flux_Ko_s=$flux_Ko_s"

  # flux en Mo / s
  flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" | bc )
  # echo "flux_Mo_s=$flux_Mo_s"

  if [ -d $dest ] ; then
    dirOut=$( readlink -f ${dest} )
  else
    dirOut=$( readlink -f $( dirname ${dest} ) )
  fi

  if [ -d $source ] ; then
    dirSource=$( readlink -f ${source} )
  else
    dirSource=$( readlink -f $( dirname ${source} ) )
  fi

  echo "==>act:${actionType}|sz:${entitySizeMo}|ms:${diff_ms}|fx(ko):${flux_Ko_s}|fx(mo):${flux_Mo_s}|nm:${name}|dirSource:${dirSource}|dirOut:${dirOut}"
}

#D-#==================================================================
#D-function IGCM_debug_Check
#D- * Purpose: Check the present file by comparison with a reference file
function IGCM_debug_Check
{
  #---------------------
  if [ ! -n "${libIGCM}" ] ; then
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> libIGCM variable is not defined"
    exit 2
  fi

  #---------------------
  if [ ! -n "${Verbosity}" ] ; then
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> Verbosity variable is not defined"
    exit 3
  fi

  #---------------------
  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1
  sleep 2

  # Remove date stamp.
  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate
  mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed

  if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
    echo "Check libIGCM_debug ..............................................[ OK ]"
    rm -f IGCM_debug_Test.ref.failed
  else
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
    echo "           has produced the file IGCM_debug_Test.ref.failed"
    echo "           Please analyse differences with the reference file by typing:"
    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
    exit 4
  fi
  #---------------------
}