source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 894

Last change on this file since 894 was 894, checked in by sdipsl, 11 years ago
  • clearer message when error
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 17.8 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Patrick Brockmann, Martial Mancip
5# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#==================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#==================================================
19
20#==================================================
21# Add high level verbosity
22typeset -i Verbosity=${Verbosity:=3}
23
24#==================================================
25# DEBUG_debug
26# Add low level verbosity
27typeset DEBUG_debug=${DEBUG_debug:=false}
28
29#==================================================
30# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31typeset RandomError=false
32
33# Where the stack file containing call tree will be stored.
34typeset StackFileLocation=${StackFileLocation:=${PWD}}
35
36if ( $DEBUG_debug ) ; then
37  if [ -f ${StackFileLocation}/stack ] ;
38  then
39    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack
40  else
41    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/stack
42  fi
43fi
44
45#==================================================
46# NULL_STR
47# Default null string
48typeset -r NULL_STR="_0_" 
49
50#==================================================
51# libIGCM_CurrentTag
52# Current libIGCM tag, check compatibilty with *.card
53typeset -r libIGCM_CurrentTag="1.0" 
54
55#==================================================
56# Exit Flag (internal debug)
57# When true, end the master loop AFTER SAVES FILES
58ExitFlag=false
59
60#==================================================
61# Declare a stack of functions calls
62
63# insert last argument of the Stack
64#set -A IGCM_debug_Stack ${NULL_STR}
65#set -A IGCM_debug_StackArgs ${NULL_STR}
66unset IGCM_debug_Stack
67unset IGCM_debug_StackArgs
68IGCM_debug_Stack[0]=${NULL_STR}
69IGCM_debug_StackArgs[0]=${NULL_STR}
70IGCM_debug_LenStack=0
71
72#D-#==================================================================
73#D-function IGCM_debug_CallStack
74#D-* Purpose: Echo the Stack
75#D-
76function IGCM_debug_CallStack {
77  if ( $DEBUG_debug ) ; then
78    # La pile d'appels est affichée de la plus vieille à la plus récente
79    # (c'est donc l'inverse de la norme d'affichage).
80    typeset i decal
81    i=0
82    until [ $i -eq ${IGCM_debug_LenStack} ]; do
83      decal=0
84      until [ $decal -eq ${i} ]; do
85        printf -- ' '
86        (( decal = decal + 1 ))
87      done
88      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
89      ((i = i + 1))
90    done
91    #echo "!------------------------!"
92  fi
93}
94
95#D-#==================================================================
96#D-function IGCM_debug_PushStack
97#D-* Purpose: Push a function name in the stack
98#D-
99function IGCM_debug_PushStack {
100
101  if ( $DEBUG_debug ) ; then
102    typeset decal inputs
103    echo >> ${StackFileLocation}/stack
104    decal=0
105    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
106      printf ' ' >> ${StackFileLocation}/stack
107      (( decal = decal + 1 ))
108    done
109
110    # STORE input list in an indexed array
111    INPUTS=( $@ )
112    # We add function call name on beginning of the stack
113    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
114
115    # We include the "null" Args in the beginning of the StackArgs
116    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]} 
117    # Then, we shift StackArgs tabular
118    if [ $# -gt 1 ]; then
119      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g")
120    fi
121
122    # Fill the stack file
123    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
124
125    # Fill the rabbitMQ queue
126
127    # TO BE A FUNCTION BEGIN #
128
129    if [ X${ActivateBigBro} = Xtrue ] ; then
130      # Only cosmetics
131      decal=0
132      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
133        printf ' ' >> ${StackFileLocation}/stack
134        (( decal = decal + 1 ))
135      done
136      # RabbitMQ message
137      code=2000
138      #
139      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
140      encodedBody=$( echo "${Body}" | base64 -w 0 )
141      #
142      #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
143      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
144      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
145      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
146      status=$?
147      if [ ${status} -gt 0 ] ; then
148        IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}"
149        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
150        exit
151      fi
152    fi
153
154    # TO BE A FUNCTION END #
155
156    # Increment LenStack
157    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
158
159    # If you want to print CallStack each time :
160    #IGCM_debug_CallStack
161  fi
162}
163
164#D-#==================================================================
165#D-function IGCM_debug_PopStack
166#D-* Purpose: Pop a function name in the stack
167#D-
168function IGCM_debug_PopStack {
169  if ( $DEBUG_debug ) ; then
170    typeset decal
171    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
172      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
173      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
174      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
175    else
176      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
177      IGCM_debug_Exit $@
178    fi
179    decal=0
180    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
181      printf ' ' >> ${StackFileLocation}/stack
182      (( decal = decal + 1 ))
183    done
184
185    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
186    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
187    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
188    #
189    if ( ${RandomError} ) ; then
190      if [ $((RANDOM%10000)) -le 10 ] ; then
191        IGCM_debug_Print 1 "A random error has been triggered"
192        echo "RANDOM ERROR" >> ${StackFileLocation}/stack
193        ExitFlag=true
194      fi
195    fi
196
197    if ( ${ExitFlag} ) ; then
198      # Inform the stack file
199      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack
200      # Inform the rabbitMQ queue
201
202      # TO BE A FUNCTION BEGIN #
203
204      if [ X${ActivateBigBro} = Xtrue ] ; then
205        # Only cosmetics
206        decal=0
207        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
208          printf ' ' >> ${StackFileLocation}/stack
209          (( decal = decal + 1 ))
210        done
211        # RabbitMQ message
212        code=9000
213        #
214        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
215        encodedBody=$( echo "${Body}" | base64 -w 0 )
216        #
217        #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody}
218        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
219        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
220        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
221        status=$?
222        if [ ${status} -gt 0 ] ; then
223          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
224          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
225          exit
226        fi
227      fi
228
229      # TO BE A FUNCTION END #
230
231    else
232      # Inform the stack file
233      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
234
235      # Inform the rabbitMQ queue
236
237      # TO BE A FUNCTION BEGIN #
238
239      if [ X${ActivateBigBro} = Xtrue ] ; then
240        # Only cosmetics
241        decal=0
242        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
243          printf ' ' >> ${StackFileLocation}/stack
244          (( decal = decal + 1 ))
245        done
246        # RabbitMQ message
247        code=3000
248        #
249        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
250        encodedBody=$( echo "${Body}" | base64 -w 0 )
251        #
252        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
253        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
254        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
255        status=$?
256        if [ ${status} -gt 0 ] ; then
257          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
258          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
259          exit
260        fi
261      fi
262
263      # TO BE A FUNCTION END #
264
265    fi
266
267    if [ ${IGCM_debug_LenStack} = 0 ]; then
268      # Reset array only when necessary
269      #echo
270      #IGCM_debug_Print 3 "Clean stack array"
271      #echo
272      #set -A IGCM_debug_Stack ${NULL_STR}
273      #set -A IGCM_debug_StackArgs ${NULL_STR}
274      unset IGCM_debug_Stack
275      unset IGCM_debug_StackArgs
276      IGCM_debug_Stack[0]=${NULL_STR}
277      IGCM_debug_StackArgs[0]=${NULL_STR}
278    fi
279  fi
280  #IGCM_debug_CallStack
281}
282
283#D-#==================================================================
284#D-function IGCM_debug_ActivateBigBro
285#D-* Purpose: switch rabbitMQ on
286#D-
287function IGCM_debug_ActivateBigBro {
288  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
289
290  # Fill the rabbitMQ queue
291  if [ X${BigBrother} = Xtrue ] ; then
292    # ID to identify a simulation
293    simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
294    # ID to identify a job. Several Jobs are needed to complete a simulation
295    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}
296    # Only cosmetics
297    decal=0
298    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
299      printf ' ' >> ${StackFileLocation}/stack
300      (( decal = decal + 1 ))
301    done
302    # RabbitMQ message
303    if ( ${FirstInitialize} ) ; then
304      code=0000
305    else
306      code=1000
307    fi
308    Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
309    encodedBody=$( echo "${Body}" | base64 -w 0 )
310    #
311    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
312    #
313    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b "${Body}"      >> ${StackFileLocation}/stack
314    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
315    sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
316    status=$?
317    if [ ${status} -gt 0 ] ; then
318      IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}"
319      echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
320      IGCM_debug_Exit "IGCM_debug_ActivateBigBro"
321    fi
322    ActivateBigBro=true
323  fi
324  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
325}
326
327#D-#==================================================================
328#D-function IGCM_debug_Exit
329#D-* Purpose: Print Call Stack and set ExitFlag to true
330#D-
331function IGCM_debug_Exit {
332  IGCM_debug_PushStack "IGCM_debug_Exit"
333  echo "IGCM_debug_Exit : " "${@}"
334  echo
335  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
336  echo "!!   ERROR TRIGGERED    !!" 
337  echo "!!   EXIT FLAG SET      !!" 
338  echo "!------------------------!" 
339  echo
340  IGCM_debug_CallStack
341  ExitFlag=true
342  IGCM_debug_PopStack "IGCM_debug_Exit"
343}
344
345#D-#==================================================
346#D-function IGCM_debug_Verif_Exit
347#D-* Purpose: exit with number 1 if ExitFlag is true
348#D-
349function IGCM_debug_Verif_Exit {
350  if ( ${ExitFlag} ) ; then
351    # Plan to send an email here with IGCM_sys_SendMail
352    if [ X${TaskType} != Xchecking ] ; then
353      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
354      echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
355      echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
356      echo "                        EXIT THE JOB."
357      echo
358      IGCM_debug_CallStack
359    fi
360    if ( $DEBUG_debug ) ; then
361      echo "Your files on ${R_OUT} :"
362      IGCM_sys_Tree ${R_SAVE}
363      echo
364    fi
365
366    # TO BE A FUNCTION BEGIN #
367
368    if [ X${ActivateBigBro} = Xtrue ] ; then
369      # Only cosmetics
370      decal=0
371      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
372        printf ' ' >> ${StackFileLocation}/stack
373        (( decal = decal + 1 ))
374      done
375      # RabbitMQ message
376      code=9999
377      #
378      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
379      encodedBody=$( echo "${Body}" | base64 -w 0 )
380      #
381      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
382      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
383      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
384      status=$?
385      if [ ${status} -gt 0 ] ; then
386        IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
387        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
388        exit
389      fi
390    fi
391
392    # TO BE A FUNCTION END #
393
394    # Mail notification
395    IGCM_sys_SendMail
396    # And Good Bye
397    date
398    exit 1
399  fi
400}
401
402#D-#==================================================
403#D-function IGCM_debug_Verif_Exit_Post
404#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
405#D-
406function IGCM_debug_Verif_Exit_Post {
407  if ( ${ExitFlag} ) ; then
408    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
409    # If SpaceName is PROD then we stop if post_processing fails
410    # Plan to send an email here with IGCM_sys_SendMail
411    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
412      echo "                        EXIT THE JOB."
413      echo
414      # Mail notification
415      #IGCM_sys_SendMailPost
416      # And Good Bye
417      date
418      exit 1
419    else
420      echo "Either inside config.card the variable SpaceName is not in PROD"
421      echo "or inside the main Job the variable JobType is not in RUN mode"
422      echo "              SO WE DO NOT EXIT THE JOB."
423      echo
424      date
425    fi
426  fi
427}
428
429#D-#==================================================================
430#D-function IGCM_debug_Print
431#D-* Purpose: Print arguments according to a level of verbosity.
432#D-
433function IGCM_debug_Print
434{
435  typeset level=$1
436  shift
437
438  if [ X"${1}" = X"-e" ]; then
439    typeset cmd_echo="echo -e"
440    shift
441  else
442    typeset cmd_echo="echo"
443  fi
444
445  if [ ${level} -le ${Verbosity} ] ; then
446    typeset i
447    case "${level}" in
448    1) for i in "$@" ; do
449      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
450      done ;;
451    2) for i in "$@" ; do
452      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
453      done ;;
454    3) for i in "$@" ; do
455      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
456      done ;;
457    esac
458  fi
459}
460
461#D-#==================================================================
462#D-function IGCM_debug_PrintVariables
463#D-* Purpose: Print arguments when match a pattern
464#D-           according to a level of verbosity.
465function IGCM_debug_PrintVariables
466{
467  typeset level=$1
468  shift
469
470  list=$( set | grep ^$1 | sed -e "s/'//g" )
471
472  if [ "X${list}" != X ]  ; then
473    IGCM_debug_Print ${level} ${list}
474  fi
475}
476
477#D-#==================================================================
478#D-function IGCM_debug_Check
479#D- * Purpose: Check the present file by comparison with a reference file
480function IGCM_debug_Check
481{
482  #---------------------
483  if [ ! -n "${libIGCM}" ] ; then
484    echo "Check libIGCM_debug ..........................................[ FAILED ]"
485    echo "--Error--> libIGCM variable is not defined"
486    exit 2
487  fi
488
489  #---------------------
490  if [ ! -n "${Verbosity}" ] ; then
491    echo "Check libIGCM_debug ..........................................[ FAILED ]"
492    echo "--Error--> Verbosity variable is not defined"
493    exit 3
494  fi
495
496  #---------------------
497  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1
498  sleep 2
499
500  # Remove date stamp.
501  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate
502  mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed
503
504  if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
505    echo "Check libIGCM_debug ..............................................[ OK ]"
506    rm -f IGCM_debug_Test.ref.failed
507  else
508    echo "Check libIGCM_debug ..........................................[ FAILED ]"
509    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
510    echo "           has produced the file IGCM_debug_Test.ref.failed"
511    echo "           Please analyse differences with the reference file by typing:"
512    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
513    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
514    exit 4
515  fi
516  #---------------------
517}
Note: See TracBrowser for help on using the repository browser.