#!/bin/ksh #************************************************************** # Author: Patrick Brockmann, Martial Mancip # Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr # $Revision:: $ Revision of last commit # $Author:: $ Author of last commit # $Date:: $ Date of last commit # IPSL (2006) # This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC # #************************************************************** #================================================== # The documentation of this file can be automatically generated # if you use the prefix #D- for comments to be extracted. # Extract with command: cat lib* | grep "^#D-" | cut -c "4-" #================================================== #================================================== # Add high level verbosity typeset -i Verbosity=${Verbosity:=3} #================================================== # DEBUG_debug # Add low level verbosity typeset DEBUG_debug=${DEBUG_debug:=false} #================================================== # GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} ) typeset RandomError=false # Where the stack file containing call tree will be stored. typeset StackFileLocation=${StackFileLocation:=${PWD}} if ( $DEBUG_debug ) ; then if [ -f ${StackFileLocation}/stack ] ; then echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack else echo "Stack of an libIGCM job :" > ${StackFileLocation}/stack fi fi #================================================== # NULL_STR # Default null string typeset -r NULL_STR="_0_" #================================================== # libIGCM_CurrentTag # Current libIGCM tag, check compatibilty with *.card typeset -r libIGCM_CurrentTag="1.0" #================================================== # Exit Flag (internal debug) # When true, end the master loop AFTER SAVES FILES ExitFlag=false #================================================== # Declare a stack of functions calls # insert last argument of the Stack #set -A IGCM_debug_Stack ${NULL_STR} #set -A IGCM_debug_StackArgs ${NULL_STR} unset IGCM_debug_Stack unset IGCM_debug_StackArgs IGCM_debug_Stack[0]=${NULL_STR} IGCM_debug_StackArgs[0]=${NULL_STR} IGCM_debug_LenStack=0 #D-#================================================================== #D-function IGCM_debug_CallStack #D-* Purpose: Echo the Stack #D- function IGCM_debug_CallStack { if ( $DEBUG_debug ) ; then # La pile d'appels est affichée de la plus vieille à la plus récente # (c'est donc l'inverse de la norme d'affichage). typeset i decal i=0 until [ $i -eq ${IGCM_debug_LenStack} ]; do decal=0 until [ $decal -eq ${i} ]; do printf -- ' ' (( decal = decal + 1 )) done echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})" ((i = i + 1)) done #echo "!------------------------!" fi } #D-#================================================================== #D-function IGCM_debug_PushStack #D-* Purpose: Push a function name in the stack #D- function IGCM_debug_PushStack { if ( $DEBUG_debug ) ; then typeset decal inputs echo >> ${StackFileLocation}/stack decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # STORE input list in an indexed array INPUTS=( $@ ) # We add function call name on beginning of the stack set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]} # We include the "null" Args in the beginning of the StackArgs set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]} # Then, we shift StackArgs tabular if [ $# -gt 1 ]; then IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g") fi # Fill the stack file echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack # Fill the rabbitMQ queue # TO BE A FUNCTION BEGIN # if [ X${ActivateBigBro} = Xtrue ] ; then # Only cosmetics decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # RabbitMQ message code=2000 # Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) encodedBody=$( echo "${Body}" | base64 -w 0 ) # #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} status=$? if [ ${status} -gt 0 ] ; then IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}" echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" exit fi fi # TO BE A FUNCTION END # # Increment LenStack (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 )) # If you want to print CallStack each time : #IGCM_debug_CallStack fi } #D-#================================================================== #D-function IGCM_debug_PopStack #D-* Purpose: Pop a function name in the stack #D- function IGCM_debug_PopStack { if ( $DEBUG_debug ) ; then typeset decal if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 )) set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1} set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1} else echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack} IGCM_debug_Exit $@ fi decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD # if ( ${RandomError} ) ; then if [ $((RANDOM%10000)) -le 10 ] ; then IGCM_debug_Print 1 "A random error has been triggered" echo "RANDOM ERROR" >> ${StackFileLocation}/stack ExitFlag=true fi fi if ( ${ExitFlag} ) ; then # Inform the stack file echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack # Inform the rabbitMQ queue # TO BE A FUNCTION BEGIN # if [ X${ActivateBigBro} = Xtrue ] ; then # Only cosmetics decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # RabbitMQ message code=9000 # Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) encodedBody=$( echo "${Body}" | base64 -w 0 ) # #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody} echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} status=$? if [ ${status} -gt 0 ] ; then IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}" echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" exit fi fi # TO BE A FUNCTION END # else # Inform the stack file echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack # Inform the rabbitMQ queue # TO BE A FUNCTION BEGIN # if [ X${ActivateBigBro} = Xtrue ] ; then # Only cosmetics decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # RabbitMQ message code=3000 # Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) encodedBody=$( echo "${Body}" | base64 -w 0 ) # echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} status=$? if [ ${status} -gt 0 ] ; then IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}" echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" exit fi fi # TO BE A FUNCTION END # fi if [ ${IGCM_debug_LenStack} = 0 ]; then # Reset array only when necessary #echo #IGCM_debug_Print 3 "Clean stack array" #echo #set -A IGCM_debug_Stack ${NULL_STR} #set -A IGCM_debug_StackArgs ${NULL_STR} unset IGCM_debug_Stack unset IGCM_debug_StackArgs IGCM_debug_Stack[0]=${NULL_STR} IGCM_debug_StackArgs[0]=${NULL_STR} fi fi #IGCM_debug_CallStack } #D-#================================================================== #D-function IGCM_debug_ActivateBigBro #D-* Purpose: switch rabbitMQ on #D- function IGCM_debug_ActivateBigBro { IGCM_debug_PushStack "IGCM_debug_ActivateBigBro" # Fill the rabbitMQ queue if [ X${BigBrother} = Xtrue ] ; then # ID to identify a simulation simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE # ID to identify a job. Several Jobs are needed to complete a simulation jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod} # Only cosmetics decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # RabbitMQ message if ( ${FirstInitialize} ) ; then code=0000 else code=1000 fi Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) encodedBody=$( echo "${Body}" | base64 -w 0 ) # cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64 # echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b "${Body}" >> ${StackFileLocation}/stack echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} status=$? if [ ${status} -gt 0 ] ; then IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}" echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" IGCM_debug_Exit "IGCM_debug_ActivateBigBro" fi ActivateBigBro=true fi IGCM_debug_PopStack "IGCM_debug_ActivateBigBro" } #D-#================================================================== #D-function IGCM_debug_Exit #D-* Purpose: Print Call Stack and set ExitFlag to true #D- function IGCM_debug_Exit { IGCM_debug_PushStack "IGCM_debug_Exit" echo "IGCM_debug_Exit : " "${@}" echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "!! ERROR TRIGGERED !!" echo "!! EXIT FLAG SET !!" echo "!------------------------!" echo IGCM_debug_CallStack ExitFlag=true IGCM_debug_PopStack "IGCM_debug_Exit" } #D-#================================================== #D-function IGCM_debug_Verif_Exit #D-* Purpose: exit with number 1 if ExitFlag is true #D- function IGCM_debug_Verif_Exit { if ( ${ExitFlag} ) ; then # Plan to send an email here with IGCM_sys_SendMail if [ X${TaskType} != Xchecking ] ; then IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal" echo "IGCM_debug_Verif_Exit : Something wrong happened previously." echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where." echo " EXIT THE JOB." echo IGCM_debug_CallStack fi if ( $DEBUG_debug ) ; then echo "Your files on ${R_OUT} :" IGCM_sys_Tree ${R_SAVE} echo fi # TO BE A FUNCTION BEGIN # if [ X${ActivateBigBro} = Xtrue ] ; then # Only cosmetics decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/stack (( decal = decal + 1 )) done # RabbitMQ message code=9999 # Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) encodedBody=$( echo "${Body}" | base64 -w 0 ) # echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} status=$? if [ ${status} -gt 0 ] ; then IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}" echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" exit fi fi # TO BE A FUNCTION END # # Mail notification IGCM_sys_SendMail # And Good Bye date exit 1 fi } #D-#================================================== #D-function IGCM_debug_Verif_Exit_Post #D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment #D- function IGCM_debug_Verif_Exit_Post { if ( ${ExitFlag} ) ; then echo "IGCM_debug_Verif_Exit_Post : Something wrong happened." # If SpaceName is PROD then we stop if post_processing fails # Plan to send an email here with IGCM_sys_SendMail if [ X${config_UserChoices_SpaceName} = XPROD ] ; then echo " EXIT THE JOB." echo # Mail notification #IGCM_sys_SendMailPost # And Good Bye date exit 1 else echo "Either inside config.card the variable SpaceName is not in PROD" echo "or inside the main Job the variable JobType is not in RUN mode" echo " SO WE DO NOT EXIT THE JOB." echo date fi fi } #D-#================================================================== #D-function IGCM_debug_Print #D-* Purpose: Print arguments according to a level of verbosity. #D- function IGCM_debug_Print { typeset level=$1 shift if [ X"${1}" = X"-e" ]; then typeset cmd_echo="echo -e" shift else typeset cmd_echo="echo" fi if [ ${level} -le ${Verbosity} ] ; then typeset i case "${level}" in 1) for i in "$@" ; do ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i} done ;; 2) for i in "$@" ; do ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i} done ;; 3) for i in "$@" ; do ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i} done ;; esac fi } #D-#================================================================== #D-function IGCM_debug_PrintVariables #D-* Purpose: Print arguments when match a pattern #D- according to a level of verbosity. function IGCM_debug_PrintVariables { typeset level=$1 shift list=$( set | grep ^$1 | sed -e "s/'//g" ) if [ "X${list}" != X ] ; then IGCM_debug_Print ${level} ${list} fi } #D-#================================================================== #D-function IGCM_debug_Check #D- * Purpose: Check the present file by comparison with a reference file function IGCM_debug_Check { #--------------------- if [ ! -n "${libIGCM}" ] ; then echo "Check libIGCM_debug ..........................................[ FAILED ]" echo "--Error--> libIGCM variable is not defined" exit 2 fi #--------------------- if [ ! -n "${Verbosity}" ] ; then echo "Check libIGCM_debug ..........................................[ FAILED ]" echo "--Error--> Verbosity variable is not defined" exit 3 fi #--------------------- ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1 sleep 2 # Remove date stamp. sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then echo "Check libIGCM_debug ..............................................[ OK ]" rm -f IGCM_debug_Test.ref.failed else echo "Check libIGCM_debug ..........................................[ FAILED ]" echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh" echo " has produced the file IGCM_debug_Test.ref.failed" echo " Please analyse differences with the reference file by typing:" echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref" echo " Report errors to the author: Patrick.Brockmann@cea.fr" exit 4 fi #--------------------- }