source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 936

Last change on this file since 936 was 936, checked in by sdipsl, 11 years ago

bugfix again with instrumentation. Should be fine now...
syntax error in libIGCM_sys
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 24.7 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Patrick Brockmann, Martial Mancip
5# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#==================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#==================================================
19
20#==================================================
21# Add high level verbosity
22typeset -i Verbosity=${Verbosity:=3}
23
24#==================================================
25# DEBUG_debug
26# Add low level verbosity
27typeset DEBUG_debug=${DEBUG_debug:=false}
28
29#==================================================
30# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31typeset RandomError=false
32
33# Stack file containing call tree will be stored there.
34if ( $DEBUG_debug ) ; then
35  if [ X${TaskType} = Xcomputing ]; then
36    typeset StackFileLocation=${StackFileLocation:=${PWD}}
37    typeset StackFileName=computing.stack.$$
38  elif [ X${TaskType} = Xpost-processing ]; then
39    typeset StackFileLocation=${POST_DIR:=${StackFileLocation}}
40    typeset StackFileName=post-processing.stack.$$
41  elif [ X${TaskType} = Xchecking ]; then
42    typeset StackFileLocation=${POST_DIR:=${PWD}}
43    typeset StackFileName=checking.stack.$$
44  else
45    typeset StackFileLocation=${POST_DIR:=${PWD}}
46    typeset StackFileName=checking.stack.$$
47  fi
48
49  if [ -f ${StackFileLocation}/${StackFileName} ] ;
50  then
51    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/${StackFileName}
52  else
53    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/${StackFileName}
54  fi
55fi
56
57#==================================================
58# NULL_STR
59# Default null string
60typeset -r NULL_STR="_0_"
61
62#==================================================
63# libIGCM_CurrentTag
64# Current libIGCM tag, check compatibilty with *.card
65typeset -r libIGCM_CurrentTag="1.0"
66
67#==================================================
68# Exit Flag (internal debug)
69# When true, end the master loop AFTER SAVES FILES
70ExitFlag=false
71
72#==================================================
73# Declare a stack of functions calls
74unset IGCM_debug_Stack
75unset IGCM_debug_StackArgs
76unset IGCM_debug_StackTiming
77IGCM_debug_Stack[0]=${NULL_STR}
78IGCM_debug_StackArgs[0]=${NULL_STR}
79IGCM_debug_StackTiming[0]=${NULL_STR}
80IGCM_debug_LenStack=0
81
82#D-#==================================================================
83#D-function IGCM_debug_getDate_ms
84#D- * Purpose: Give number of milliseconds since 01-jan-1970
85function IGCM_debug_getDate_ms
86{
87  typeset nanosecs ms
88  # nano secondes since 01-jan-1970
89  nanosecs=$( date +%s%N )
90
91  # truncate the last 6 digits to get milliseconds since 01-jan-1970
92  ms=${nanosecs:0:${#nanosecs}-6}
93
94  echo "$ms"
95}
96
97#D-#==================================================================
98#D-function IGCM_debug_sizeOfTabContent
99#D- * Purpose: Give sumed size of a list of files
100#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
101#D- *        where entityList is a list of files or directory
102#D- *        where dest is either a directory or a file name
103function IGCM_debug_sizeOfTabContent
104{
105  typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
106
107  eval set +A entityListe \${${1}}
108  destination=${2} 
109  sumSizeKo=0
110
111  # Here we will try to compute size (file or directory size) from local path and not from archive.
112  for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do 
113    if [ -f ${entityListe[$i]##/*/} ] ; then
114      # One file or a bunch of files has been copied without renaming
115      # remove path /home/login/../ from entityListe elements
116      iEntity=${entityListe[$i]##/*/}
117    elif [ -f ${destination} ] ; then
118      # a file has been copied and rename
119      iEntity=${destination}
120    elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
121      # a copy in a directory but not in ${PWD}
122      iEntity=${destination}/${entityListe[$i]##/*/}
123    elif [ -d ${entityListe[$i]} ] ; then
124      # directory has been copied from non remote place
125      iEntity=${entityListe[$i]}
126    elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
127      # directory has been copied from remote archive and not rename
128      iEntity=${destination}/${entityListe[$i]##/*/}
129    elif [ -d ${destination} ] ; then
130      # directory has been copied from remote archive and rename
131      iEntity=${destination}
132    fi
133    sizeKo=$( du --apparent-size -sk ${iEntity} | awk {'print $1'} )
134    sumSizeKo=$(( $sumSizeKo + $sizeKo ))
135  done
136  sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" | bc )
137  echo "${sumSizeKo}|${sumSizeMo}"
138
139}
140
141#D-#==================================================================
142#D-function IGCM_debug_SendAMQP
143#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
144function IGCM_debug_sendAMQP
145{
146  typeset decal first additionnalOption encodedBody
147
148  # Encode message Body
149  encodedBody=$( echo "${Body}" | base64 -w 0 )
150
151  # Send config.card ?
152  if [ X${1} = Xactivate ] ; then
153    # Encode config.card
154    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
155    # Prepare additionnal option
156    additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
157  else
158    additionnalOption=
159  fi
160
161  # Only cosmetics : stack file
162  decal=0
163  while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
164    printf ' ' >> ${StackFileLocation}/${StackFileName}
165    (( decal = decal + 1 ))
166  done
167  # Log to stack file
168  echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b "${Body}"      >> ${StackFileLocation}/${StackFileName}
169
170  # Log separately AMQP send message command
171  echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
172
173  # Send the message
174  sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
175  status=$?
176  if [ ${status} -gt 0 ] ; then
177    IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
178    echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
179    exit
180  fi
181}
182
183#D-#==================================================================
184#D-function IGCM_debug_CallStack
185#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
186#D-
187function IGCM_debug_CallStack {
188  if ( $DEBUG_debug ) ; then
189    # Cosmetics
190    typeset i decal
191    i=0
192    until [ $i -eq ${IGCM_debug_LenStack} ]; do
193      decal=0
194      until [ $decal -eq ${i} ]; do
195        printf -- ' '
196        (( decal = decal + 1 ))
197      done
198      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
199      ((i = i + 1))
200    done
201  fi
202}
203
204#D-#==================================================================
205#D-function IGCM_debug_PushStack
206#D-* Purpose: Push a function name in the stack
207#D-
208function IGCM_debug_PushStack {
209  if ( $DEBUG_debug ) ; then
210    typeset decal inputs startTime_ms
211
212    # Only cosmetics : stack file
213    echo >> ${StackFileLocation}/${StackFileName}
214    decal=0
215    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
216      printf ' ' >> ${StackFileLocation}/${StackFileName}
217      (( decal = decal + 1 ))
218    done
219
220    # Fill the stack file
221    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
222
223    # Save input list in an indexed array
224    INPUTS=( $@ )
225
226    # Get timing information
227    startTime_ms=$( IGCM_debug_getDate_ms )
228
229    # We add function call name on beginning of the stack
230    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
231
232    # Save timing in milliseconds in an indexed array
233    set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
234
235    # We include the "null" Args in the beginning of the StackArgs
236    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
237
238    # Then, we shift StackArgs tabular
239    if [ $# -gt 1 ]; then
240      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g" )
241    fi
242
243    if [ X${ActivateBigBro} = Xtrue ] ; then
244      # RabbitMQ message code
245      code=2000
246      # RabbitMQ message body
247      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
248      # Fill the rabbitMQ queue
249      IGCM_debug_sendAMQP
250    fi
251
252    # Increment LenStack
253    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
254
255    #IGCM_debug_CallStack
256  fi
257}
258
259#D-#==================================================================
260#D-function IGCM_debug_PopStack
261#D-* Purpose: Pop a function name in the stack
262#D-
263function IGCM_debug_PopStack {
264  if ( $DEBUG_debug ) ; then
265    typeset i decal command arguments startTime_ms endTime_ms
266    typeset instrumentation dest
267    # they are not typeset because they are send "by adress" to son functions
268    # we unset them to avoid "memory effect"
269    unset fileList source
270
271    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
272    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
273    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
274    if ( ${RandomError} ) ; then
275      if [ $((RANDOM%10000)) -le 10 ] ; then
276        IGCM_debug_Print 1 "Random error has been triggered"
277        echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
278        ExitFlag=true
279      fi
280    fi
281
282    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
283      # Everything is cool
284
285      # Get timing information
286      endTime_ms=$( IGCM_debug_getDate_ms )
287
288      # Save Stack information before poping the stack
289      command=${IGCM_debug_Stack[0]}
290
291      # Go from comma separated (list) to space separated in an array
292      set -A arguments $( echo ${IGCM_debug_StackArgs[0]} | sed -e "s/,/\ /g" )
293
294      # Save Stack information before poping the stack
295      startTime_ms=${IGCM_debug_StackTiming[0]}
296
297      # Pop the stack
298      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
299      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
300      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
301      set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
302    else
303      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
304      IGCM_debug_Exit $@
305    fi
306
307    # Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
308    # We are interested in:
309    #  0. Which command performs the work
310    #  1. Size of entity we are working with
311    #  2. Where are we reading
312    #  3. Where are we writing
313    #  4. How long it took
314
315    instrumentation=false
316
317    case ${command} in
318    # Classical copy (only files are given to IGCM_sys_Cp as options)
319    IGCM_sys_Cp)
320      instrumentation=true
321      # All but the latest
322      fileList=${arguments[*]:0:${#arguments[*]}-1}
323      # just need the first file to get the directory
324      source=${arguments[0]}
325      # Nothing but the latest
326      dest=${arguments[${#arguments[*]}-1]}
327      # Size of file whose name are stored in a list
328      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
329      ;;
330
331    # Copy from archive machine or from buffer
332    IGCM_sys_Get|IGCM_sys_GetBuffer)
333      instrumentation=true
334      if [ ${#arguments[*]} -eq 2 ] ; then
335        source=${arguments[0]}
336        dest=${arguments[1]}
337        # Size of file whose name are stored in a variable
338        entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
339      elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
340        # IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
341        # Keep the array name hosting the all list
342        eval set +A fileList \${${arguments[1]}}
343        # just need the first file to get the directory
344        source=${fileList[0]}
345        dest=${arguments[2]}
346        # Size of file whose name are stored in a list
347        entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
348      elif [ [ ${#arguments[*]} -ge 3 ] ; then
349       # All but the latest
350        fileList=${arguments[*]:0:${#arguments[*]}-1}
351        # just need the first file to get the directory
352        source=${arguments[0]}
353        # Nothing but the latest
354        dest=${arguments[${#arguments[*]}-1]}
355        # Size of file whose name are stored in a list
356        entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
357      fi
358      ;;
359
360    # Copy from compute node or copy to archive/buffer
361    IGCM_sys_Get_Master|IGCM_sys_Get_Dir|IGCM_sys_Put_Out|IGCM_sys_PutBuffer_Out)
362      instrumentation=true
363      source=${arguments[0]}
364      dest=${arguments[1]}
365      # Size of file whose name are stored in a variable
366      entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
367      ;;
368
369    # Rebuild command
370    IGCM_sys_rebuild|IGCM_sys_rebuild_station)
371      instrumentation=true
372      # All but the first
373      fileList=${arguments[*]:1:${#arguments[*]}-1}
374      # just need a file to get the directory
375      source=${arguments[1]}
376      # Nothing but the first
377      dest=${arguments[0]}
378      # Size of file whose name are stored in a list
379      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
380      ;;
381   
382    # NCO commands
383    IGCM_sys_ncrcat|IGCM_sys_ncecat|IGCM_sys_ncra|IGCM_sys_ncks|IGCM_sys_cdo)
384      # Example of what we want to catch : only filenames in those command lines
385      # IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
386      # IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
387      # IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
388      # IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
389      instrumentation=true
390      # All files that are not the latest argument (only input files)
391      i=0
392      keepGoing=true
393      while ( ${keepGoing} ) ; do
394        if [ -f ${arguments[${#arguments[*]}-2-${i}]} ] ; then
395          fileList="${fileList} ${arguments[${#arguments[*]}-2-${i}]}"
396          ((i = i + 1))
397        elif [ ${i} -ge 1 ] ; then
398          keepGoing=false
399        else
400          ((i = i + 1))
401        fi
402      done
403      # i value is at least 1
404      # just need one file to get the directory
405      source=${arguments[${#arguments[*]}-1-${i}]}
406      # Nothing but the latest
407      dest=${arguments[${#arguments[*]}-1]}
408      # Size of file whose name are stored in a list
409      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
410      ;;
411    esac
412
413    # Print information related to instrumentation
414    ( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
415
416    # Only cosmetics : stack file
417    decal=0
418    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
419      printf ' ' >> ${StackFileLocation}/${StackFileName}
420      (( decal = decal + 1 ))
421    done
422
423    if ( ${ExitFlag} ) ; then
424      # Inform the stack file
425      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
426
427      if [ X${ActivateBigBro} = Xtrue ] ; then
428        # RabbitMQ message code
429        code=9000
430        # RabbitMQ message body
431        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
432        # Fill the rabbitMQ queue
433        IGCM_debug_sendAMQP
434      fi
435    else
436      # Inform the stack file
437      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
438      if [ X${ActivateBigBro} = Xtrue ] ; then
439        # RabbitMQ message code
440        code=3000
441        # RabbitMQ message body
442        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
443        # Fill the rabbitMQ queue
444        IGCM_debug_sendAMQP
445      fi
446    fi
447
448    # Reset array if necessary
449    if [ ${IGCM_debug_LenStack} = 0 ]; then
450      #echo
451      #IGCM_debug_Print 3 "Clean stack array"
452      #echo
453      unset IGCM_debug_Stack
454      unset IGCM_debug_StackArgs
455      unset IGCM_debug_StackTiming
456      IGCM_debug_Stack[0]=${NULL_STR}
457      IGCM_debug_StackArgs[0]=${NULL_STR}
458      IGCM_debug_StackTiming[0]=${NULL_STR}
459    fi
460  fi
461  #IGCM_debug_CallStack
462}
463
464#D-#==================================================================
465#D-function IGCM_debug_ActivateBigBro
466#D-* Purpose: switch rabbitMQ on
467#D-
468function IGCM_debug_ActivateBigBro {
469  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
470
471  if [ X${BigBrother} = Xtrue ] ; then
472    # ID to identify a simulation
473    simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
474
475    # ID to identify a job. Several Jobs are needed to complete a simulation
476    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}
477
478    # RabbitMQ message code
479    if ( ${FirstInitialize} ) ; then
480      code=0000
481    else
482      code=1000
483    fi
484    # RabbitMQ message body
485    Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
486    # Fill the rabbitMQ queue (specifying config.card must be send)
487    IGCM_debug_sendAMQP activate
488
489    # Turn the flag on
490    ActivateBigBro=true
491  fi
492  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
493}
494
495#D-#==================================================================
496#D-function IGCM_debug_Exit
497#D-* Purpose: Print Call Stack and set ExitFlag to true
498#D-
499function IGCM_debug_Exit {
500  IGCM_debug_PushStack "IGCM_debug_Exit"
501  echo "IGCM_debug_Exit : " "${@}"
502  echo
503  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
504  echo "!!   ERROR TRIGGERED    !!"
505  echo "!!   EXIT FLAG SET      !!"
506  echo "!------------------------!"
507  echo
508  IGCM_debug_CallStack
509  ExitFlag=true
510  IGCM_debug_PopStack "IGCM_debug_Exit"
511}
512
513#D-#==================================================
514#D-function IGCM_debug_Verif_Exit
515#D-* Purpose: exit with number 1 if ExitFlag is true
516#D-
517function IGCM_debug_Verif_Exit {
518  if ( ${ExitFlag} ) ; then
519    # Plan to send an email here with IGCM_sys_SendMail
520    if [ X${TaskType} != Xchecking ] ; then
521      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
522      echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
523      echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
524      echo "                        EXIT THE JOB."
525      echo
526      IGCM_debug_CallStack
527    fi
528
529    if [ X${ActivateBigBro} = Xtrue ] ; then
530      # RabbitMQ message code
531      code=9999
532      # RabbitMQ message body
533      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
534      # Fill the rabbitMQ queue
535      IGCM_debug_sendAMQP
536    fi
537
538    # Not sure about that one ...
539    if ( $DEBUG_debug ) ; then
540      echo "Your files on ${R_OUT} :"
541      IGCM_sys_Tree ${R_SAVE}
542      echo
543    fi
544
545    # Mail notification
546    IGCM_sys_SendMail
547
548    # And Good Bye
549    date
550    exit 1
551  fi
552}
553
554#D-#==================================================
555#D-function IGCM_debug_Verif_Exit_Post
556#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
557#D-
558function IGCM_debug_Verif_Exit_Post {
559  if ( ${ExitFlag} ) ; then
560    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
561    # If SpaceName is PROD then we stop if post_processing fails
562    # Plan to send an email here with IGCM_sys_SendMail
563    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
564      echo "                        EXIT THE JOB."
565      echo
566      # Mail notification
567      #IGCM_sys_SendMailPost
568      # And Good Bye
569      date
570      exit 1
571    else
572      echo "Either inside config.card the variable SpaceName is not in PROD"
573      echo "or inside the main Job the variable JobType is not in RUN mode"
574      echo "              SO WE DO NOT EXIT THE JOB."
575      echo
576      date
577    fi
578  fi
579}
580
581#D-#==================================================================
582#D-function IGCM_debug_Print
583#D-* Purpose: Print arguments according to a level of verbosity.
584#D-
585function IGCM_debug_Print
586{
587  typeset level=$1
588  shift
589
590  if [ X"${1}" = X"-e" ]; then
591    typeset cmd_echo="echo -e"
592    shift
593  else
594    typeset cmd_echo="echo"
595  fi
596
597  if [ ${level} -le ${Verbosity} ] ; then
598    typeset i
599    case "${level}" in
600    1) for i in "$@" ; do
601      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
602      done ;;
603    2) for i in "$@" ; do
604      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
605      done ;;
606    3) for i in "$@" ; do
607      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
608      done ;;
609    esac
610  fi
611}
612
613#D-#==================================================================
614#D-function IGCM_debug_PrintVariables
615#D-* Purpose: Print arguments when match a pattern
616#D-           according to a level of verbosity.
617function IGCM_debug_PrintVariables
618{
619  typeset level=$1
620  shift
621
622  list=$( set | grep ^$1 | sed -e "s/'//g" )
623
624  if [ "X${list}" != X ]  ; then
625    IGCM_debug_Print ${level} ${list}
626  fi
627}
628
629#D-#==================================================================
630#D-function IGCM_debug_PrintInfosActions
631#D-* Purpose: Print information related to instrumentation
632function IGCM_debug_PrintInfosActions
633{
634  typeset actionType=$1
635  typeset entitySize=$2
636  typeset start_ms=$3
637  typeset end_ms=$4
638
639  typeset dest=$5
640  typeset source=$6
641
642  typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
643  typeset name dirOut dirSource
644
645  name="doNotKnow"
646
647  diff_ms=$(( $end_ms - $start_ms ))
648  # echo "diff_ms=$diff_ms"
649
650  entitySizeKo=$( echo ${entitySize} | gawk -F"|" '{print $1}' )
651  # echo "entitySizeKo=$entitySizeKo"
652  entitySizeMo=$( echo ${entitySize} | gawk -F"|" '{print $2}' )
653
654  # flux en Ko / ms
655  flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" | bc )
656  # echo "flux_Ko_ms=$flux_Ko_ms"
657
658  # flux en Ko / s
659  flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
660  # echo "flux_Ko_s=$flux_Ko_s"
661
662  # flux en Mo / s
663  flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" | bc )
664  # echo "flux_Mo_s=$flux_Mo_s"
665
666  if [ -d $dest ] ; then
667    dirOut=$( readlink -f ${dest} )
668  else
669    dirOut=$( readlink -f $( dirname ${dest} ) )
670  fi
671
672  if [ -d $source ] ; then
673    dirSource=$( readlink -f ${source} )
674  else
675    dirSource=$( readlink -f $( dirname ${source} ) )
676  fi
677
678  echo "==>act:${actionType}|sz:${entitySizeMo}|ms:${diff_ms}|fx(ko):${flux_Ko_s}|fx(mo):${flux_Mo_s}|nm:${name}|dirSource:${dirSource}|dirOut:${dirOut}"
679}
680
681#D-#==================================================================
682#D-function IGCM_debug_Check
683#D- * Purpose: Check the present file by comparison with a reference file
684function IGCM_debug_Check
685{
686  #---------------------
687  if [ ! -n "${libIGCM}" ] ; then
688    echo "Check libIGCM_debug ..........................................[ FAILED ]"
689    echo "--Error--> libIGCM variable is not defined"
690    exit 2
691  fi
692
693  #---------------------
694  if [ ! -n "${Verbosity}" ] ; then
695    echo "Check libIGCM_debug ..........................................[ FAILED ]"
696    echo "--Error--> Verbosity variable is not defined"
697    exit 3
698  fi
699
700  #---------------------
701  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > /tmp/IGCM_debug_Test.$$.ref.failed 2>&1
702  sleep 2
703
704  # Remove date stamp.
705  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" /tmp/IGCM_debug_Test.$$.ref.failed > /tmp/IGCM_debug_Test.$$.ref.failed.nodate
706  mv /tmp/IGCM_debug_Test.$$.ref.failed.nodate /tmp/IGCM_debug_Test.$$.ref.failed
707
708  if diff /tmp/IGCM_debug_Test.$$.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
709    echo "Check libIGCM_debug ..............................................[ OK ]"
710    rm -f /tmp/IGCM_debug_Test.$$.ref.failed
711  else
712    echo "Check libIGCM_debug ..........................................[ FAILED ]"
713    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
714    echo "           has produced the file IGCM_debug_Test.ref.failed"
715    echo "           Please analyse differences with the reference file by typing:"
716    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
717    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
718    cat /tmp/IGCM_debug_Test.$$.ref.failed
719    exit 4
720  fi
721  #---------------------
722}
Note: See TracBrowser for help on using the repository browser.