Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 955

Last change on this file since 955 was 941, checked in by sdipsl, 11 years ago
Adapt instrumentation for this kind of syntax: # IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output} # IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 25.3 KB

Rev	Line
[913]	1	#!/bin/ksh
[2]	2
	3	#**************************************************************
	4	# Author: Patrick Brockmann, Martial Mancip
[373]	5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
[2]	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#==================================================
	15	# The documentation of this file can be automatically generated
[913]	16	# if you use the prefix #D- for comments to be extracted.
[2]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#==================================================
	19
	20	#==================================================
	21	# Add high level verbosity
	22	typeset -i Verbosity=${Verbosity:=3}
	23
	24	#==================================================
	25	# DEBUG_debug
	26	# Add low level verbosity
[98]	27	typeset DEBUG_debug=${DEBUG_debug:=false}
[2]	28
[872]	29	#==================================================
	30	# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
	31	typeset RandomError=false
	32
[926]	33	# Stack file containing call tree will be stored there.
[2]	34	if ( $DEBUG_debug ) ; then
[926]	35	if [ X${TaskType} = Xcomputing ]; then
	36	typeset StackFileLocation=${StackFileLocation:=${PWD}}
	37	typeset StackFileName=computing.stack.$$
	38	elif [ X${TaskType} = Xpost-processing ]; then
	39	typeset StackFileLocation=${POST_DIR:=${StackFileLocation}}
	40	typeset StackFileName=post-processing.stack.$$
	41	elif [ X${TaskType} = Xchecking ]; then
	42	typeset StackFileLocation=${POST_DIR:=${PWD}}
	43	typeset StackFileName=checking.stack.$$
[936]	44	else
	45	typeset StackFileLocation=${POST_DIR:=${PWD}}
	46	typeset StackFileName=checking.stack.$$
[926]	47	fi
[936]	48
[926]	49	if [ -f ${StackFileLocation}/${StackFileName} ] ;
[544]	50	then
[926]	51	echo "Stack of an libIGCM job :" >> ${StackFileLocation}/${StackFileName}
[544]	52	else
[926]	53	echo "Stack of an libIGCM job :" > ${StackFileLocation}/${StackFileName}
[544]	54	fi
[2]	55	fi
	56
	57	#==================================================
	58	# NULL_STR
	59	# Default null string
[913]	60	typeset -r NULL_STR="_0_"
[2]	61
	62	#==================================================
	63	# libIGCM_CurrentTag
	64	# Current libIGCM tag, check compatibilty with *.card
[915]	65	typeset -r libIGCM_CurrentTag="1.0"
[2]	66
	67	#==================================================
	68	# Exit Flag (internal debug)
	69	# When true, end the master loop AFTER SAVES FILES
	70	ExitFlag=false
	71
	72	#==================================================
	73	# Declare a stack of functions calls
[59]	74	unset IGCM_debug_Stack
	75	unset IGCM_debug_StackArgs
[913]	76	unset IGCM_debug_StackTiming
[54]	77	IGCM_debug_Stack[0]=${NULL_STR}
	78	IGCM_debug_StackArgs[0]=${NULL_STR}
[913]	79	IGCM_debug_StackTiming[0]=${NULL_STR}
[2]	80	IGCM_debug_LenStack=0
	81
	82	#D-#==================================================================
[913]	83	#D-function IGCM_debug_getDate_ms
	84	#D- * Purpose: Give number of milliseconds since 01-jan-1970
	85	function IGCM_debug_getDate_ms
	86	{
[926]	87	typeset nanosecs ms
[913]	88	# nano secondes since 01-jan-1970
	89	nanosecs=$( date +%s%N )
	90
	91	# truncate the last 6 digits to get milliseconds since 01-jan-1970
	92	ms=${nanosecs:0:${#nanosecs}-6}
	93
	94	echo "$ms"
	95	}
	96
	97	#D-#==================================================================
	98	#D-function IGCM_debug_sizeOfTabContent
	99	#D- * Purpose: Give sumed size of a list of files
[924]	100	#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
	101	#D- * where entityList is a list of files or directory
	102	#D- * where dest is either a directory or a file name
[913]	103	function IGCM_debug_sizeOfTabContent
	104	{
[924]	105	typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
	106
	107	eval set +A entityListe \${${1}}
[941]	108	destination=${2}
[924]	109	sumSizeKo=0
	110
	111	# Here we will try to compute size (file or directory size) from local path and not from archive.
[941]	112	for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
	113	if [ -f ${entityListe[$i]} ] ; then
	114	# One file or a bunch of files has been copied without renaming from a visible filesystem
	115	iEntity=${entityListe[$i]}
	116	elif [ -f ${entityListe[$i]##/*/} ] ; then
	117	# One file or a bunch of files has been copied without renaming from an non visible filesystem
[924]	118	# remove path /home/login/../ from entityListe elements
	119	iEntity=${entityListe[$i]##/*/}
	120	elif [ -f ${destination} ] ; then
[941]	121	# a file has been copied and renamed
[924]	122	iEntity=${destination}
	123	elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
	124	# a copy in a directory but not in ${PWD}
	125	iEntity=${destination}/${entityListe[$i]##/*/}
	126	elif [ -d ${entityListe[$i]} ] ; then
[941]	127	# a directory has been copied from a non remote place
[924]	128	iEntity=${entityListe[$i]}
	129	elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
[941]	130	# a directory has been copied from a remote archive and not renamed
[924]	131	iEntity=${destination}/${entityListe[$i]##/*/}
	132	elif [ -d ${destination} ] ; then
[941]	133	# a directory has been copied from a remote archive and renamed
[924]	134	iEntity=${destination}
[917]	135	fi
[941]	136	sizeKo=$( du --apparent-size -sk ${iEntity} \| gawk '{print $1}' )
[924]	137	sumSizeKo=$(( $sumSizeKo + $sizeKo ))
[913]	138	done
[924]	139	sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" \| bc )
	140	echo "${sumSizeKo}\|${sumSizeMo}"
[913]	141	}
	142
	143	#D-#==================================================================
	144	#D-function IGCM_debug_SendAMQP
	145	#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
	146	function IGCM_debug_sendAMQP
	147	{
	148	typeset decal first additionnalOption encodedBody
	149
	150	# Encode message Body
	151	encodedBody=$( echo "${Body}" \| base64 -w 0 )
	152
	153	# Send config.card ?
	154	if [ X${1} = Xactivate ] ; then
	155	# Encode config.card
	156	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
	157	# Prepare additionnal option
	158	additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
	159	else
	160	additionnalOption=
	161	fi
	162
	163	# Only cosmetics : stack file
	164	decal=0
	165	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
[926]	166	printf ' ' >> ${StackFileLocation}/${StackFileName}
[913]	167	(( decal = decal + 1 ))
	168	done
	169	# Log to stack file
[926]	170	echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b "${Body}" >> ${StackFileLocation}/${StackFileName}
[913]	171
	172	# Log separately AMQP send message command
	173	echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
	174
	175	# Send the message
	176	sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
	177	status=$?
	178	if [ ${status} -gt 0 ] ; then
	179	IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
	180	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
	181	exit
	182	fi
	183	}
	184
	185	#D-#==================================================================
[2]	186	#D-function IGCM_debug_CallStack
[913]	187	#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
[2]	188	#D-
	189	function IGCM_debug_CallStack {
[544]	190	if ( $DEBUG_debug ) ; then
[913]	191	# Cosmetics
[544]	192	typeset i decal
[823]	193	i=0
[544]	194	until [ $i -eq ${IGCM_debug_LenStack} ]; do
	195	decal=0
	196	until [ $decal -eq ${i} ]; do
[869]	197	printf -- ' '
[823]	198	(( decal = decal + 1 ))
[544]	199	done
[869]	200	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
[823]	201	((i = i + 1))
[544]	202	done
	203	fi
[2]	204	}
	205
	206	#D-#==================================================================
	207	#D-function IGCM_debug_PushStack
	208	#D-* Purpose: Push a function name in the stack
	209	#D-
	210	function IGCM_debug_PushStack {
[544]	211	if ( $DEBUG_debug ) ; then
[913]	212	typeset decal inputs startTime_ms
	213
	214	# Only cosmetics : stack file
[926]	215	echo >> ${StackFileLocation}/${StackFileName}
[544]	216	decal=0
[823]	217	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
[926]	218	printf ' ' >> ${StackFileLocation}/${StackFileName}
[823]	219	(( decal = decal + 1 ))
[544]	220	done
[2]	221
[926]	222	# Fill the stack file
	223	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
	224
[913]	225	# Save input list in an indexed array
[823]	226	INPUTS=( $@ )
[913]	227
	228	# Get timing information
	229	startTime_ms=$( IGCM_debug_getDate_ms )
	230
[544]	231	# We add function call name on beginning of the stack
	232	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
[2]	233
[913]	234	# Save timing in milliseconds in an indexed array
	235	set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
	236
[544]	237	# We include the "null" Args in the beginning of the StackArgs
[913]	238	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
	239
[544]	240	# Then, we shift StackArgs tabular
[913]	241	if [ $# -gt 1 ]; then
	242	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /,/g" )
[544]	243	fi
[855]	244
	245	if [ X${ActivateBigBro} = Xtrue ] ; then
[913]	246	# RabbitMQ message code
[855]	247	code=2000
[913]	248	# RabbitMQ message body
[869]	249	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
[913]	250	# Fill the rabbitMQ queue
	251	IGCM_debug_sendAMQP
[855]	252	fi
	253
	254	# Increment LenStack
[544]	255	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
[2]	256
[869]	257	#IGCM_debug_CallStack
[544]	258	fi
[2]	259	}
	260
	261	#D-#==================================================================
	262	#D-function IGCM_debug_PopStack
	263	#D-* Purpose: Pop a function name in the stack
	264	#D-
	265	function IGCM_debug_PopStack {
[544]	266	if ( $DEBUG_debug ) ; then
[926]	267	typeset i decal command arguments startTime_ms endTime_ms
[941]	268	typeset instrumentation dest prefix
[926]	269	# they are not typeset because they are send "by adress" to son functions
	270	# we unset them to avoid "memory effect"
	271	unset fileList source
[913]	272
	273	# INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
	274	# PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
	275	# THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
	276	if ( ${RandomError} ) ; then
	277	if [ $((RANDOM%10000)) -le 10 ] ; then
	278	IGCM_debug_Print 1 "Random error has been triggered"
[926]	279	echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
[913]	280	ExitFlag=true
	281	fi
	282	fi
	283
[544]	284	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
[913]	285	# Everything is cool
	286
	287	# Get timing information
	288	endTime_ms=$( IGCM_debug_getDate_ms )
	289
	290	# Save Stack information before poping the stack
	291	command=${IGCM_debug_Stack[0]}
	292
	293	# Go from comma separated (list) to space separated in an array
	294	set -A arguments $( echo ${IGCM_debug_StackArgs[0]} \| sed -e "s/,/\ /g" )
	295
	296	# Save Stack information before poping the stack
	297	startTime_ms=${IGCM_debug_StackTiming[0]}
	298
	299	# Pop the stack
[823]	300	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
	301	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
	302	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
[913]	303	set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
[544]	304	else
	305	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
	306	IGCM_debug_Exit $@
	307	fi
[913]	308
[914]	309	# Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
[913]	310	# We are interested in:
	311	# 0. Which command performs the work
	312	# 1. Size of entity we are working with
	313	# 2. Where are we reading
	314	# 3. Where are we writing
	315	# 4. How long it took
	316
[915]	317	instrumentation=false
	318
[913]	319	case ${command} in
[925]	320	# Classical copy (only files are given to IGCM_sys_Cp as options)
	321	IGCM_sys_Cp)
	322	instrumentation=true
	323	# All but the latest
	324	fileList=${arguments[]:0:${#arguments[]}-1}
	325	# just need the first file to get the directory
	326	source=${arguments[0]}
	327	# Nothing but the latest
	328	dest=${arguments[${#arguments[*]}-1]}
	329	# Size of file whose name are stored in a list
	330	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
	331	;;
	332
[913]	333	# Copy from archive machine or from buffer
	334	IGCM_sys_Get\|IGCM_sys_GetBuffer)
[915]	335	instrumentation=true
[913]	336	if [ ${#arguments[*]} -eq 2 ] ; then
	337	source=${arguments[0]}
	338	dest=${arguments[1]}
	339	# Size of file whose name are stored in a variable
[917]	340	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]	341	elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
[936]	342	# IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
[913]	343	# Keep the array name hosting the all list
[936]	344	eval set +A fileList \${${arguments[1]}}
[913]	345	# just need the first file to get the directory
[936]	346	source=${fileList[0]}
[931]	347	dest=${arguments[2]}
[934]	348	# Size of file whose name are stored in a list
[936]	349	entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
[913]	350	elif [ [ ${#arguments[*]} -ge 3 ] ; then
	351	# All but the latest
[916]	352	fileList=${arguments[]:0:${#arguments[]}-1}
[913]	353	# just need the first file to get the directory
	354	source=${arguments[0]}
	355	# Nothing but the latest
	356	dest=${arguments[${#arguments[*]}-1]}
	357	# Size of file whose name are stored in a list
[917]	358	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]	359	fi
	360	;;
	361
[925]	362	# Copy from compute node or copy to archive/buffer
	363	IGCM_sys_Get_Master\|IGCM_sys_Get_Dir\|IGCM_sys_Put_Out\|IGCM_sys_PutBuffer_Out)
[924]	364	instrumentation=true
[916]	365	source=${arguments[0]}
[924]	366	dest=${arguments[1]}
	367	# Size of file whose name are stored in a variable
	368	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]	369	;;
	370
	371	# Rebuild command
	372	IGCM_sys_rebuild\|IGCM_sys_rebuild_station)
[915]	373	instrumentation=true
[913]	374	# All but the first
	375	fileList=${arguments[]:1:${#arguments[]}-1}
	376	# just need a file to get the directory
	377	source=${arguments[1]}
	378	# Nothing but the first
	379	dest=${arguments[0]}
	380	# Size of file whose name are stored in a list
[917]	381	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]	382	;;
[941]	383
[926]	384	# NCO commands
	385	IGCM_sys_ncrcat\|IGCM_sys_ncecat\|IGCM_sys_ncra\|IGCM_sys_ncks\|IGCM_sys_cdo)
	386	# Example of what we want to catch : only filenames in those command lines
	387	# IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
	388	# IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
	389	# IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
	390	# IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
	391	instrumentation=true
[941]	392	keepGoing=true
	393	prefix=.
[926]	394	i=0
	395	while ( ${keepGoing} ) ; do
[941]	396	# the last one is not interesting
	397	if [ ${i} -eq ${#arguments[*]}-1 ] ; then
	398	keepGoing=false
	399	# look after "-p" option. Path prefix is the following arguments
	400	elif [ ${arguments[${i}]} = "-p" ] ; then
[926]	401	((i = i + 1))
[941]	402	prefix=${arguments[${i}]}
	403	((i = i + 1))
	404	elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
[926]	405	keepGoing=false
[941]	406	# looking for files
	407	elif [ -f ${prefix}/${arguments[${i}]} ] ; then
	408	fileList="${fileList} ${prefix}/${arguments[${i}]}"
	409	((i = i + 1))
	410	# other options are not interesting
[926]	411	else
	412	((i = i + 1))
	413	fi
	414	done
[941]	415
[926]	416	# i value is at least 1
	417	# just need one file to get the directory
[941]	418	source=$( echo ${fileList} \| gawk '{print $1}' )
[926]	419	# Nothing but the latest
	420	dest=${arguments[${#arguments[*]}-1]}
	421	# Size of file whose name are stored in a list
	422	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
	423	;;
[913]	424	esac
	425
	426	# Print information related to instrumentation
[915]	427	( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
[913]	428
	429	# Only cosmetics : stack file
[823]	430	decal=0
	431	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
[926]	432	printf ' ' >> ${StackFileLocation}/${StackFileName}
[823]	433	(( decal = decal + 1 ))
[544]	434	done
[2]	435
[855]	436	if ( ${ExitFlag} ) ; then
	437	# Inform the stack file
[926]	438	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
[874]	439
[855]	440	if [ X${ActivateBigBro} = Xtrue ] ; then
[913]	441	# RabbitMQ message code
[855]	442	code=9000
[913]	443	# RabbitMQ message body
[869]	444	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
[913]	445	# Fill the rabbitMQ queue
	446	IGCM_debug_sendAMQP
[855]	447	fi
	448	else
	449	# Inform the stack file
[926]	450	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
[855]	451	if [ X${ActivateBigBro} = Xtrue ] ; then
[913]	452	# RabbitMQ message code
[855]	453	code=3000
[913]	454	# RabbitMQ message body
[869]	455	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
[913]	456	# Fill the rabbitMQ queue
	457	IGCM_debug_sendAMQP
[855]	458	fi
	459	fi
	460
[913]	461	# Reset array if necessary
[544]	462	if [ ${IGCM_debug_LenStack} = 0 ]; then
	463	#echo
	464	#IGCM_debug_Print 3 "Clean stack array"
	465	#echo
	466	unset IGCM_debug_Stack
	467	unset IGCM_debug_StackArgs
[913]	468	unset IGCM_debug_StackTiming
[544]	469	IGCM_debug_Stack[0]=${NULL_STR}
	470	IGCM_debug_StackArgs[0]=${NULL_STR}
[913]	471	IGCM_debug_StackTiming[0]=${NULL_STR}
[2]	472	fi
[544]	473	fi
[869]	474	#IGCM_debug_CallStack
[2]	475	}
	476
	477	#D-#==================================================================
[855]	478	#D-function IGCM_debug_ActivateBigBro
	479	#D-* Purpose: switch rabbitMQ on
	480	#D-
	481	function IGCM_debug_ActivateBigBro {
	482	IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
	483
[868]	484	if [ X${BigBrother} = Xtrue ] ; then
[869]	485	# ID to identify a simulation
	486	simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
[913]	487
[869]	488	# ID to identify a job. Several Jobs are needed to complete a simulation
	489	jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}
[913]	490
	491	# RabbitMQ message code
[855]	492	if ( ${FirstInitialize} ) ; then
	493	code=0000
	494	else
	495	code=1000
	496	fi
[913]	497	# RabbitMQ message body
[869]	498	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
[913]	499	# Fill the rabbitMQ queue (specifying config.card must be send)
	500	IGCM_debug_sendAMQP activate
	501
	502	# Turn the flag on
[855]	503	ActivateBigBro=true
	504	fi
	505	IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
	506	}
	507
	508	#D-#==================================================================
[2]	509	#D-function IGCM_debug_Exit
	510	#D-* Purpose: Print Call Stack and set ExitFlag to true
	511	#D-
	512	function IGCM_debug_Exit {
[544]	513	IGCM_debug_PushStack "IGCM_debug_Exit"
	514	echo "IGCM_debug_Exit : " "${@}"
[913]	515	echo
[894]	516	echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
[913]	517	echo "!! ERROR TRIGGERED !!"
	518	echo "!! EXIT FLAG SET !!"
	519	echo "!------------------------!"
	520	echo
[894]	521	IGCM_debug_CallStack
[544]	522	ExitFlag=true
	523	IGCM_debug_PopStack "IGCM_debug_Exit"
[2]	524	}
	525
	526	#D-#==================================================
	527	#D-function IGCM_debug_Verif_Exit
	528	#D-* Purpose: exit with number 1 if ExitFlag is true
	529	#D-
	530	function IGCM_debug_Verif_Exit {
[544]	531	if ( ${ExitFlag} ) ; then
	532	# Plan to send an email here with IGCM_sys_SendMail
[775]	533	if [ X${TaskType} != Xchecking ] ; then
	534	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
[894]	535	echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
	536	echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
[775]	537	echo " EXIT THE JOB."
	538	echo
[869]	539	IGCM_debug_CallStack
[775]	540	fi
[874]	541
[855]	542	if [ X${ActivateBigBro} = Xtrue ] ; then
[913]	543	# RabbitMQ message code
[855]	544	code=9999
[913]	545	# RabbitMQ message body
[869]	546	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
[913]	547	# Fill the rabbitMQ queue
	548	IGCM_debug_sendAMQP
[855]	549	fi
[874]	550
[913]	551	# Not sure about that one ...
	552	if ( $DEBUG_debug ) ; then
	553	echo "Your files on ${R_OUT} :"
	554	IGCM_sys_Tree ${R_SAVE}
	555	echo
	556	fi
[874]	557
[544]	558	# Mail notification
	559	IGCM_sys_SendMail
[913]	560
[544]	561	# And Good Bye
	562	date
	563	exit 1
	564	fi
[2]	565	}
	566
	567	#D-#==================================================
	568	#D-function IGCM_debug_Verif_Exit_Post
	569	#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
	570	#D-
	571	function IGCM_debug_Verif_Exit_Post {
[544]	572	if ( ${ExitFlag} ) ; then
[765]	573	echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
[544]	574	# If SpaceName is PROD then we stop if post_processing fails
	575	# Plan to send an email here with IGCM_sys_SendMail
[706]	576	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
[544]	577	echo " EXIT THE JOB."
	578	echo
	579	# Mail notification
	580	#IGCM_sys_SendMailPost
	581	# And Good Bye
	582	date
	583	exit 1
	584	else
	585	echo "Either inside config.card the variable SpaceName is not in PROD"
[765]	586	echo "or inside the main Job the variable JobType is not in RUN mode"
[544]	587	echo " SO WE DO NOT EXIT THE JOB."
	588	echo
	589	date
[2]	590	fi
[544]	591	fi
[2]	592	}
	593
	594	#D-#==================================================================
	595	#D-function IGCM_debug_Print
	596	#D-* Purpose: Print arguments according to a level of verbosity.
	597	#D-
	598	function IGCM_debug_Print
	599	{
[544]	600	typeset level=$1
	601	shift
	602
	603	if [ X"${1}" = X"-e" ]; then
	604	typeset cmd_echo="echo -e"
[2]	605	shift
[544]	606	else
	607	typeset cmd_echo="echo"
	608	fi
[2]	609
[544]	610	if [ ${level} -le ${Verbosity} ] ; then
	611	typeset i
	612	case "${level}" in
	613	1) for i in "$@" ; do
[734]	614	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
[913]	615	done ;;
[544]	616	2) for i in "$@" ; do
[734]	617	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
[913]	618	done ;;
[544]	619	3) for i in "$@" ; do
[734]	620	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
[913]	621	done ;;
[544]	622	esac
	623	fi
[2]	624	}
	625
	626	#D-#==================================================================
	627	#D-function IGCM_debug_PrintVariables
	628	#D-* Purpose: Print arguments when match a pattern
	629	#D- according to a level of verbosity.
	630	function IGCM_debug_PrintVariables
	631	{
[544]	632	typeset level=$1
	633	shift
[2]	634
[830]	635	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
[54]	636
[544]	637	if [ "X${list}" != X ] ; then
	638	IGCM_debug_Print ${level} ${list}
	639	fi
[2]	640	}
	641
	642	#D-#==================================================================
[914]	643	#D-function IGCM_debug_PrintInfosActions
[913]	644	#D-* Purpose: Print information related to instrumentation
	645	function IGCM_debug_PrintInfosActions
	646	{
	647	typeset actionType=$1
	648	typeset entitySize=$2
	649	typeset start_ms=$3
	650	typeset end_ms=$4
	651
	652	typeset dest=$5
	653	typeset source=$6
	654
	655	typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
	656	typeset name dirOut dirSource
	657
	658	name="doNotKnow"
	659
	660	diff_ms=$(( $end_ms - $start_ms ))
	661	# echo "diff_ms=$diff_ms"
	662
	663	entitySizeKo=$( echo ${entitySize} \| gawk -F"\|" '{print $1}' )
	664	# echo "entitySizeKo=$entitySizeKo"
	665	entitySizeMo=$( echo ${entitySize} \| gawk -F"\|" '{print $2}' )
	666
	667	# flux en Ko / ms
	668	flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" \| bc )
	669	# echo "flux_Ko_ms=$flux_Ko_ms"
	670
	671	# flux en Ko / s
	672	flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
	673	# echo "flux_Ko_s=$flux_Ko_s"
	674
	675	# flux en Mo / s
	676	flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" \| bc )
	677	# echo "flux_Mo_s=$flux_Mo_s"
	678
	679	if [ -d $dest ] ; then
	680	dirOut=$( readlink -f ${dest} )
	681	else
	682	dirOut=$( readlink -f $( dirname ${dest} ) )
	683	fi
	684
	685	if [ -d $source ] ; then
	686	dirSource=$( readlink -f ${source} )
	687	else
[914]	688	dirSource=$( readlink -f $( dirname ${source} ) )
[913]	689	fi
	690
	691	echo "==>act:${actionType}\|sz:${entitySizeMo}\|ms:${diff_ms}\|fx(ko):${flux_Ko_s}\|fx(mo):${flux_Mo_s}\|nm:${name}\|dirSource:${dirSource}\|dirOut:${dirOut}"
	692	}
	693
	694	#D-#==================================================================
[2]	695	#D-function IGCM_debug_Check
	696	#D- * Purpose: Check the present file by comparison with a reference file
	697	function IGCM_debug_Check
	698	{
[544]	699	#---------------------
	700	if [ ! -n "${libIGCM}" ] ; then
	701	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	702	echo "--Error--> libIGCM variable is not defined"
	703	exit 2
	704	fi
[2]	705
[544]	706	#---------------------
	707	if [ ! -n "${Verbosity}" ] ; then
	708	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	709	echo "--Error--> Verbosity variable is not defined"
	710	exit 3
	711	fi
[2]	712
[544]	713	#---------------------
[931]	714	${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > /tmp/IGCM_debug_Test.$$.ref.failed 2>&1
[544]	715	sleep 2
[2]	716
[772]	717	# Remove date stamp.
[931]	718	sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" /tmp/IGCM_debug_Test.$$.ref.failed > /tmp/IGCM_debug_Test.$$.ref.failed.nodate
	719	mv /tmp/IGCM_debug_Test.$$.ref.failed.nodate /tmp/IGCM_debug_Test.$$.ref.failed
[772]	720
[931]	721	if diff /tmp/IGCM_debug_Test.$$.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
[544]	722	echo "Check libIGCM_debug ..............................................[ OK ]"
[931]	723	rm -f /tmp/IGCM_debug_Test.$$.ref.failed
[544]	724	else
	725	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	726	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
	727	echo " has produced the file IGCM_debug_Test.ref.failed"
	728	echo " Please analyse differences with the reference file by typing:"
	729	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
	730	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
[931]	731	cat /tmp/IGCM_debug_Test.$$.ref.failed
[544]	732	exit 4
	733	fi
	734	#---------------------
[2]	735	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: