Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1206

Last change on this file since 1206 was 1206, checked in by sdipsl, 9 years ago
Remove IGCM_debug_Verif_Exit_Post. Only IGCM_debug_Verif_Exit will manage exit cases.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 34.0 KB

Rev	Line
[913]	1	#!/bin/ksh
[2]	2
	3	#**************************************************************
	4	# Author: Patrick Brockmann, Martial Mancip
[373]	5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
[2]	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#==================================================
	15	# The documentation of this file can be automatically generated
[913]	16	# if you use the prefix #D- for comments to be extracted.
[2]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#==================================================
	19
	20	#==================================================
	21	# Add high level verbosity
	22	typeset -i Verbosity=${Verbosity:=3}
	23
	24	#==================================================
	25	# DEBUG_debug
	26	# Add low level verbosity
[1083]	27	DEBUG_debug=${DEBUG_debug:=false}
[2]	28
[872]	29	#==================================================
	30	# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
	31	typeset RandomError=false
	32
[2]	33	#==================================================
	34	# NULL_STR
	35	# Default null string
[913]	36	typeset -r NULL_STR="_0_"
[2]	37
	38	#==================================================
	39	# libIGCM_CurrentTag
	40	# Current libIGCM tag, check compatibilty with *.card
[915]	41	typeset -r libIGCM_CurrentTag="1.0"
[2]	42
	43	#==================================================
	44	# Exit Flag (internal debug)
	45	# When true, end the master loop AFTER SAVES FILES
	46	ExitFlag=false
	47
	48	#==================================================
	49	# Declare a stack of functions calls
[59]	50	unset IGCM_debug_Stack
	51	unset IGCM_debug_StackArgs
[913]	52	unset IGCM_debug_StackTiming
[54]	53	IGCM_debug_Stack[0]=${NULL_STR}
	54	IGCM_debug_StackArgs[0]=${NULL_STR}
[913]	55	IGCM_debug_StackTiming[0]=${NULL_STR}
[2]	56	IGCM_debug_LenStack=0
	57
	58	#D-#==================================================================
[913]	59	#D-function IGCM_debug_getDate_ms
	60	#D- * Purpose: Give number of milliseconds since 01-jan-1970
	61	function IGCM_debug_getDate_ms
	62	{
[926]	63	typeset nanosecs ms
[913]	64	# nano secondes since 01-jan-1970
	65	nanosecs=$( date +%s%N )
	66
	67	# truncate the last 6 digits to get milliseconds since 01-jan-1970
	68	ms=${nanosecs:0:${#nanosecs}-6}
	69
	70	echo "$ms"
	71	}
	72
	73	#D-#==================================================================
	74	#D-function IGCM_debug_sizeOfTabContent
	75	#D- * Purpose: Give sumed size of a list of files
[924]	76	#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
	77	#D- * where entityList is a list of files or directory
	78	#D- * where dest is either a directory or a file name
[913]	79	function IGCM_debug_sizeOfTabContent
	80	{
[924]	81	typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
	82
	83	eval set +A entityListe \${${1}}
[941]	84	destination=${2}
[924]	85	sumSizeKo=0
	86
	87	# Here we will try to compute size (file or directory size) from local path and not from archive.
[941]	88	for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
	89	if [ -f ${entityListe[$i]} ] ; then
	90	# One file or a bunch of files has been copied without renaming from a visible filesystem
	91	iEntity=${entityListe[$i]}
	92	elif [ -f ${entityListe[$i]##/*/} ] ; then
	93	# One file or a bunch of files has been copied without renaming from an non visible filesystem
[924]	94	# remove path /home/login/../ from entityListe elements
	95	iEntity=${entityListe[$i]##/*/}
	96	elif [ -f ${destination} ] ; then
[941]	97	# a file has been copied and renamed
[924]	98	iEntity=${destination}
	99	elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
	100	# a copy in a directory but not in ${PWD}
	101	iEntity=${destination}/${entityListe[$i]##/*/}
	102	elif [ -d ${entityListe[$i]} ] ; then
[941]	103	# a directory has been copied from a non remote place
[924]	104	iEntity=${entityListe[$i]}
	105	elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
[941]	106	# a directory has been copied from a remote archive and not renamed
[924]	107	iEntity=${destination}/${entityListe[$i]##/*/}
	108	elif [ -d ${destination} ] ; then
[941]	109	# a directory has been copied from a remote archive and renamed
[924]	110	iEntity=${destination}
[917]	111	fi
[1083]	112	sizeKo=$( du --apparent-size -skL ${iEntity} \| gawk '{print $1}' )
[924]	113	sumSizeKo=$(( $sumSizeKo + $sizeKo ))
[913]	114	done
[924]	115	sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" \| bc )
	116	echo "${sumSizeKo}\|${sumSizeMo}"
[913]	117	}
	118
	119	#D-#==================================================================
[983]	120	#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
	121	#D- * Purpose: Take over AMQP C client using mail as a message recipient
	122	#D- * One argument : base64 encoded message
[1051]	123	#D- * Attach encoded config.card when starting the simulation
	124
[983]	125	function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
	126
[987]	127	typeset b64_encoded_msg mail_recipient
[1076]	128	typeset buffer send_messages mail_frequency
[987]	129	typeset last_mail_date__file
	130
[983]	131	b64_encoded_msg=$1
	132
[1072]	133	mail_recipient="superviseur@ipsl.jussieu.fr"
[983]	134	send_messages=0
	135	mail_frequency=3600 # in seconds
	136	# use to keep track when was last mail sent (maybe to be replaced with global variable)
[1150]	137	last_mail_date__file=${R_BUF}/.stamp.${config_UserChoices_TagName}.${config_UserChoices_JobName}
	138	# use to accumulate messages before sending them
	139	buffer=${R_BUF}/.buffer.${config_UserChoices_TagName}.${config_UserChoices_JobName}
[983]	140
	141	# init
	142	if [ ! -f "${buffer}" ]; then
[1150]	143	touch ${buffer}
[983]	144	fi
	145
	146	if [ ! -f "${last_mail_date__file}" ]; then
[1150]	147	touch ${last_mail_date__file}
[983]	148	else
[1150]	149	# compute last time the file was changed (in seconds)
[983]	150	seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
[997]	151	# send message when exceeding threshold
	152	[ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
[983]	153	fi
	154
[997]	155	# queue messages in the buffer
	156	echo ${b64_encoded_msg} >> ${buffer}
	157
	158	# send mail
[1051]	159
[1053]	160	if [ X${initBigBro} = Xtrue ] ; then
[1087]	161	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt
[1051]	162	mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
	163	rm -f $buffer ; touch ${buffer} # clear buffer
	164	touch ${last_mail_date__file} # memorize last mail date
	165	initBigBro=false
[1150]	166	elif [ ${send_messages} -eq 1 ] ; then
[1087]	167	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt
[1051]	168	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
[1150]	169	rm -f ${buffer} ; touch ${buffer} # flush the buffer
[1051]	170	touch ${last_mail_date__file} # memorize last mail date
[983]	171	fi
[987]	172
[1189]	173	if ( ${FlushAMQP} ) ; then
[1150]	174	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
	175	rm -f ${buffer} # cleaning behind us
	176	rm -f ${last_mail_date__file} # cleaning behind us
	177	fi
	178
[983]	179	# Allways all good for now.
	180	return 0
	181	}
	182
	183	#D-#==================================================================
[1162]	184	#D-function IGCM_debug_sendAMQP_Metrics
	185	#D- * Purpose: Take over AMQP C client using mail as a message recipient
[1202]	186	#D- * Two arguments : - Directory where metrics.json files can be found
	187	#D- * - Metrics Group Name. metrics will be added to this group
[1162]	188	#D- * Attach encoded metrics.json files.
	189
	190	function IGCM_debug_sendAMQP_Metrics {
	191
	192	typeset mail_recipient encodedBody
	193	if [ X${ActivateBigBro} = Xtrue ] ; then
	194	mail_recipient="superviseur@ipsl.jussieu.fr"
	195	# Metrics tag on server side
	196	code=7100
	197	# Usual AMQP message to route messages on server side
[1202]	198	encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"metricsGroupName\":\"${2}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" \| base64 -w 0 )
[1162]	199	# send mail
	200	attachmentsOptions=""
	201	for metricsFile in $( ls $1/*json ) ; do
	202	attachmentsOptions="-a ${metricsFile} ${attachmentsOptions}"
	203	done
[1204]	204	IGCM_debug_Print 2 "IGCM_debug_sendAMQP_Metrics "
[1162]	205	echo ${encodedBody}\|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient}
	206	fi
	207
	208	# Allways all good for now.
	209	return 0
	210	}
	211
	212	#D-#==================================================================
[913]	213	#D-function IGCM_debug_SendAMQP
	214	#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
[1053]	215	function IGCM_debug_sendAMQP {
	216
[913]	217	typeset decal first additionnalOption encodedBody
	218
	219	# Encode message Body
	220	encodedBody=$( echo "${Body}" \| base64 -w 0 )
	221
	222	# Send config.card ?
	223	if [ X${1} = Xactivate ] ; then
	224	# Encode config.card
	225	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
	226	# Prepare additionnal option
	227	additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
[1076]	228	#
[1051]	229	initBigBro=true
[913]	230	else
	231	additionnalOption=
[1051]	232	#
	233	initBigBro=false
[913]	234	fi
	235
	236	# Only cosmetics : stack file
[1115]	237	if [ X${ActivateStackFilling} = Xtrue ] ; then
	238	decal=0
	239	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
	240	printf ' ' >> ${StackFileLocation}/${StackFileName}
	241	(( decal = decal + 1 ))
	242	done
	243	# Log to stack file using human readable format
	244	echo "${Body}" >> ${StackFileLocation}/${StackFileName}
	245	fi
[913]	246
[983]	247	# Log separately encoded AMQP message command for reuse in a mock up
[1120]	248	#echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> ${RUN_DIR_PATH}/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
[913]	249
	250	# Send the message
[983]	251	if [ X${BigBrotherChannel} = XMAIL ] ; then
	252	IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
	253	status=$?
	254	else
	255	sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
	256	status=$?
	257	fi
	258
[913]	259	if [ ${status} -gt 0 ] ; then
	260	IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
[1051]	261	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
[1090]	262	exit 1
[913]	263	fi
	264	}
	265
	266	#D-#==================================================================
[2]	267	#D-function IGCM_debug_CallStack
[913]	268	#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
[2]	269	#D-
	270	function IGCM_debug_CallStack {
[544]	271	if ( $DEBUG_debug ) ; then
[913]	272	# Cosmetics
[544]	273	typeset i decal
[823]	274	i=0
[544]	275	until [ $i -eq ${IGCM_debug_LenStack} ]; do
	276	decal=0
	277	until [ $decal -eq ${i} ]; do
[869]	278	printf -- ' '
[823]	279	(( decal = decal + 1 ))
[544]	280	done
[869]	281	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
[823]	282	((i = i + 1))
[544]	283	done
	284	fi
[2]	285	}
	286
	287	#D-#==================================================================
	288	#D-function IGCM_debug_PushStack
	289	#D-* Purpose: Push a function name in the stack
	290	#D-
	291	function IGCM_debug_PushStack {
[544]	292	if ( $DEBUG_debug ) ; then
[913]	293	typeset decal inputs startTime_ms
	294
	295	# Only cosmetics : stack file
[1115]	296	if [ X${ActivateStackFilling} = Xtrue ] ; then
	297	echo >> ${StackFileLocation}/${StackFileName}
	298	decal=0
	299	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
	300	printf ' ' >> ${StackFileLocation}/${StackFileName}
	301	(( decal = decal + 1 ))
	302	done
[2]	303
[1115]	304	# Fill the stack file
	305	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
	306	fi
[926]	307
[913]	308	# Save input list in an indexed array
[823]	309	INPUTS=( $@ )
[913]	310
	311	# Get timing information
	312	startTime_ms=$( IGCM_debug_getDate_ms )
	313
[544]	314	# We add function call name on beginning of the stack
	315	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
[2]	316
[913]	317	# Save timing in milliseconds in an indexed array
	318	set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
	319
[544]	320	# We include the "null" Args in the beginning of the StackArgs
[913]	321	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
	322
[544]	323	# Then, we shift StackArgs tabular
[1065]	324	# Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
[913]	325	if [ $# -gt 1 ]; then
[1065]	326	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /\",\"/g" )
[544]	327	fi
[855]	328
[1189]	329	# Unplugged message 4000 handling for now. To ease downstream treatment.
[1150]	330	#if [ X${ActivateBigBro} = Xtrue ] ; then
	331	# # RabbitMQ message code "PUSHSTACK"
[1189]	332	# code=4000
[1150]	333	# # RabbitMQ message body
	334	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	335	# # Fill the rabbitMQ queue
	336	# IGCM_debug_sendAMQP
	337	#fi
[855]	338
	339	# Increment LenStack
[544]	340	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
[2]	341
[869]	342	#IGCM_debug_CallStack
[544]	343	fi
[2]	344	}
	345
	346	#D-#==================================================================
	347	#D-function IGCM_debug_PopStack
	348	#D-* Purpose: Pop a function name in the stack
	349	#D-
	350	function IGCM_debug_PopStack {
[544]	351	if ( $DEBUG_debug ) ; then
[926]	352	typeset i decal command arguments startTime_ms endTime_ms
[941]	353	typeset instrumentation dest prefix
[926]	354	# they are not typeset because they are send "by adress" to son functions
	355	# we unset them to avoid "memory effect"
	356	unset fileList source
[913]	357
	358	# INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
	359	# PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
	360	# THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
	361	if ( ${RandomError} ) ; then
	362	if [ $((RANDOM%10000)) -le 10 ] ; then
	363	IGCM_debug_Print 1 "Random error has been triggered"
[1115]	364	if [ X${ActivateStackFilling} = Xtrue ] ; then
	365	echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
	366	fi
[913]	367	ExitFlag=true
	368	fi
	369	fi
	370
[544]	371	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
[913]	372	# Everything is cool
	373
	374	# Get timing information
	375	endTime_ms=$( IGCM_debug_getDate_ms )
	376
	377	# Save Stack information before poping the stack
	378	command=${IGCM_debug_Stack[0]}
	379
[1084]	380	# Go from comma separated list of quoted elements (except the first and the last element)
	381	# to unquoted space separated elements in an array
[1083]	382	set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} \| sed -e "s/\",\"/\ /g" )
[913]	383
	384	# Save Stack information before poping the stack
	385	startTime_ms=${IGCM_debug_StackTiming[0]}
	386
	387	# Pop the stack
[823]	388	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
	389	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
	390	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
[913]	391	set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
[544]	392	else
	393	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
	394	IGCM_debug_Exit $@
	395	fi
[913]	396
[914]	397	# Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
[913]	398	# We are interested in:
	399	# 0. Which command performs the work
	400	# 1. Size of entity we are working with
	401	# 2. Where are we reading
	402	# 3. Where are we writing
	403	# 4. How long it took
	404
[915]	405	instrumentation=false
	406
[913]	407	case ${command} in
[925]	408	# Classical copy (only files are given to IGCM_sys_Cp as options)
	409	IGCM_sys_Cp)
	410	instrumentation=true
	411	# All but the latest
	412	fileList=${arguments[]:0:${#arguments[]}-1}
	413	# just need the first file to get the directory
	414	source=${arguments[0]}
	415	# Nothing but the latest
	416	dest=${arguments[${#arguments[*]}-1]}
	417	# Size of file whose name are stored in a list
	418	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
	419	;;
	420
[913]	421	# Copy from archive machine or from buffer
	422	IGCM_sys_Get\|IGCM_sys_GetBuffer)
[915]	423	instrumentation=true
[913]	424	if [ ${#arguments[*]} -eq 2 ] ; then
	425	source=${arguments[0]}
	426	dest=${arguments[1]}
	427	# Size of file whose name are stored in a variable
[917]	428	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]	429	elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
[936]	430	# IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
[913]	431	# Keep the array name hosting the all list
[936]	432	eval set +A fileList \${${arguments[1]}}
[913]	433	# just need the first file to get the directory
[936]	434	source=${fileList[0]}
[931]	435	dest=${arguments[2]}
[934]	436	# Size of file whose name are stored in a list
[936]	437	entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
[913]	438	elif [ [ ${#arguments[*]} -ge 3 ] ; then
	439	# All but the latest
[916]	440	fileList=${arguments[]:0:${#arguments[]}-1}
[913]	441	# just need the first file to get the directory
	442	source=${arguments[0]}
	443	# Nothing but the latest
	444	dest=${arguments[${#arguments[*]}-1]}
	445	# Size of file whose name are stored in a list
[917]	446	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]	447	fi
	448	;;
	449
[925]	450	# Copy from compute node or copy to archive/buffer
	451	IGCM_sys_Get_Master\|IGCM_sys_Get_Dir\|IGCM_sys_Put_Out\|IGCM_sys_PutBuffer_Out)
[924]	452	instrumentation=true
[916]	453	source=${arguments[0]}
[924]	454	dest=${arguments[1]}
	455	# Size of file whose name are stored in a variable
	456	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]	457	;;
	458
	459	# Rebuild command
	460	IGCM_sys_rebuild\|IGCM_sys_rebuild_station)
[915]	461	instrumentation=true
[913]	462	# All but the first
	463	fileList=${arguments[]:1:${#arguments[]}-1}
	464	# just need a file to get the directory
	465	source=${arguments[1]}
	466	# Nothing but the first
	467	dest=${arguments[0]}
	468	# Size of file whose name are stored in a list
[917]	469	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]	470	;;
[941]	471
[926]	472	# NCO commands
	473	IGCM_sys_ncrcat\|IGCM_sys_ncecat\|IGCM_sys_ncra\|IGCM_sys_ncks\|IGCM_sys_cdo)
	474	# Example of what we want to catch : only filenames in those command lines
	475	# IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
	476	# IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
	477	# IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
	478	# IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
	479	instrumentation=true
[941]	480	keepGoing=true
	481	prefix=.
[926]	482	i=0
	483	while ( ${keepGoing} ) ; do
[941]	484	# the last one is not interesting
	485	if [ ${i} -eq ${#arguments[*]}-1 ] ; then
	486	keepGoing=false
	487	# look after "-p" option. Path prefix is the following arguments
	488	elif [ ${arguments[${i}]} = "-p" ] ; then
[926]	489	((i = i + 1))
[941]	490	prefix=${arguments[${i}]}
	491	((i = i + 1))
	492	elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
[926]	493	keepGoing=false
[941]	494	# looking for files
	495	elif [ -f ${prefix}/${arguments[${i}]} ] ; then
	496	fileList="${fileList} ${prefix}/${arguments[${i}]}"
	497	((i = i + 1))
	498	# other options are not interesting
[926]	499	else
	500	((i = i + 1))
	501	fi
	502	done
[941]	503
[926]	504	# i value is at least 1
	505	# just need one file to get the directory
[941]	506	source=$( echo ${fileList} \| gawk '{print $1}' )
[926]	507	# Nothing but the latest
	508	dest=${arguments[${#arguments[*]}-1]}
	509	# Size of file whose name are stored in a list
	510	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
	511	;;
[913]	512	esac
	513
	514	# Print information related to instrumentation
[915]	515	( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
[913]	516
	517	# Only cosmetics : stack file
[1115]	518	if [ X${ActivateStackFilling} = Xtrue ] ; then
	519	decal=0
	520	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
	521	printf ' ' >> ${StackFileLocation}/${StackFileName}
	522	(( decal = decal + 1 ))
	523	done
	524	fi
[2]	525
[855]	526	if ( ${ExitFlag} ) ; then
	527	# Inform the stack file
[1115]	528	if [ X${ActivateStackFilling} = Xtrue ] ; then
	529	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
	530	fi
[874]	531
[855]	532	if [ X${ActivateBigBro} = Xtrue ] ; then
[1076]	533	# RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
[1189]	534	code=4900
[913]	535	# RabbitMQ message body
[1065]	536	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[913]	537	# Fill the rabbitMQ queue
	538	IGCM_debug_sendAMQP
[855]	539	fi
	540	else
	541	# Inform the stack file
[1115]	542	if [ X${ActivateStackFilling} = Xtrue ] ; then
	543	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
	544	fi
	545
[1189]	546	# Unplugged message 4100 handling for now. To ease downstream treatment.
[1150]	547	#if [ X${ActivateBigBro} = Xtrue ] ; then
	548	# # RabbitMQ message code "POPSTACK"
[1189]	549	# code=4100
[1150]	550	# # RabbitMQ message body
	551	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	552	# # Fill the rabbitMQ queue
	553	# IGCM_debug_sendAMQP
	554	#fi
[855]	555	fi
	556
[913]	557	# Reset array if necessary
[544]	558	if [ ${IGCM_debug_LenStack} = 0 ]; then
	559	#echo
	560	#IGCM_debug_Print 3 "Clean stack array"
	561	#echo
	562	unset IGCM_debug_Stack
	563	unset IGCM_debug_StackArgs
[913]	564	unset IGCM_debug_StackTiming
[544]	565	IGCM_debug_Stack[0]=${NULL_STR}
	566	IGCM_debug_StackArgs[0]=${NULL_STR}
[913]	567	IGCM_debug_StackTiming[0]=${NULL_STR}
[2]	568	fi
[544]	569	fi
[869]	570	#IGCM_debug_CallStack
[2]	571	}
	572
	573	#D-#==================================================================
[1189]	574	#D-function IGCM_debug_BigBro_Initialize
[855]	575	#D-* Purpose: switch rabbitMQ on
	576	#D-
[1189]	577	function IGCM_debug_BigBro_Initialize {
	578	IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize"
[855]	579
[1051]	580	# Message type standard fields:
	581	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
	582
	583	# Message type dictionnary and custom fields:
	584	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
	585
[868]	586	if [ X${BigBrother} = Xtrue ] ; then
[1051]	587	# create a unique ID for this specific job
	588	jobuid=$(uuidgen)
[913]	589
[1189]	590	if [ X${TaskType} = Xcomputing ]; then
	591	if ( ${FirstInitialize} ) ; then
	592	# RabbitMQ message code "BEGIN A SIMULATION"
	593	code=0000
	594	# create and persist a unique id for this simulation
	595	simuid=$(uuidgen)
	596	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
	597	# Standard fields for the first message
	598	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
	599	# RabbitMQ message body with specific fields associated message codes treated here
[1202]	600	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1189]	601	# Fill the rabbitMQ queue (the config.card in use will be sent)
	602	IGCM_debug_sendAMQP activate
	603	else
	604	# RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION"
	605	code=1000
	606	# retrieve this simulation's unique id
	607	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
	608	simuid=${run_Configuration_simuid}
	609	# Using standard fields for message others than the first one. Still subject to change
	610	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
	611	# RabbitMQ message body with specific fields associated message codes treated here
[1201]	612	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1189]	613	# Fill the rabbitMQ queue
	614	IGCM_debug_sendAMQP
	615	fi
	616
	617	# NOT VERY NICE BUT ... IT WORKS
	618	# Be sure that the genericSimulationID will be small from now on
	619	# Using standard fields for messages others than the first one. Still subject to change
	620	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
	621
	622	elif [ X${TaskType} = Xpost-processing ]; then
	623	# RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION"
	624	code=2000
[1076]	625	# retrieve this simulation's unique id
[1051]	626	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
[1086]	627	simuid=${run_Configuration_simuid}
[1076]	628	# Using standard fields for message others than the first one. Still subject to change
[1051]	629	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1087]	630	# RabbitMQ message body with specific fields associated message codes treated here
[1202]	631	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1087]	632	# Fill the rabbitMQ queue
	633	IGCM_debug_sendAMQP
[855]	634	fi
[913]	635	# Turn the flag on
[855]	636	ActivateBigBro=true
[1189]	637	# Dont flush by default
	638	FlushAMQP=false
[855]	639	fi
[1189]	640	IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize"
[855]	641	}
	642
	643	#D-#==================================================================
[1189]	644	#D-function IGCM_debug_BigBro_Finalize
	645	#D-* Purpose: Finalize rabbitMQ messages exchanges
	646	#D-
	647	function IGCM_debug_BigBro_Finalize {
	648	IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize"
	649
	650	# Message type standard fields:
	651	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
	652
	653	# Message type dictionnary and custom fields:
	654	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
	655
	656	if ( $DEBUG_debug ) ; then
	657	if [ X${ActivateBigBro} = Xtrue ] ; then
	658	if [ X${TaskType} = Xcomputing ]; then
	659	if ( ${simulationIsOver} ) ; then
	660	# RabbitMQ message code "SIMULATION ENDS"
	661	code=0100
	662	FlushAMQP=true
	663	else
	664	# RabbitMQ message code "COMPUTING JOB ENDS"
	665	code=1100
	666	fi
	667	elif [ X${TaskType} = Xpost-processing ]; then
	668	# RabbitMQ message code "POST-PROCESSING JOB ENDS"
	669	code=2100
	670	FlushAMQP=true
	671	fi
	672	# RabbitMQ message body
	673	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	674	# Fill the rabbitMQ queue
	675	IGCM_debug_sendAMQP
	676	fi
	677	fi
	678
	679	IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize"
	680	}
	681
	682	#D-#==================================================================
[2]	683	#D-function IGCM_debug_Exit
	684	#D-* Purpose: Print Call Stack and set ExitFlag to true
	685	#D-
	686	function IGCM_debug_Exit {
[544]	687	IGCM_debug_PushStack "IGCM_debug_Exit"
	688	echo "IGCM_debug_Exit : " "${@}"
[913]	689	echo
[894]	690	echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
[913]	691	echo "!! ERROR TRIGGERED !!"
	692	echo "!! EXIT FLAG SET !!"
	693	echo "!------------------------!"
	694	echo
[894]	695	IGCM_debug_CallStack
[544]	696	ExitFlag=true
	697	IGCM_debug_PopStack "IGCM_debug_Exit"
[2]	698	}
	699
	700	#D-#==================================================
	701	#D-function IGCM_debug_Verif_Exit
	702	#D-* Purpose: exit with number 1 if ExitFlag is true
	703	#D-
	704	function IGCM_debug_Verif_Exit {
[544]	705	if ( ${ExitFlag} ) ; then
[1206]	706	echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
	707	# Only computing TaskType stops the job for now.
	708	if [ X${TaskType} = Xcomputing ] ; then
[775]	709	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
[894]	710	echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
[775]	711	echo " EXIT THE JOB."
	712	echo
[869]	713	IGCM_debug_CallStack
[874]	714
[1206]	715	if [ X${ActivateBigBro} = Xtrue ] ; then
	716	# RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
	717	code=9999
	718	# RabbitMQ message body
	719	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	720	# To be sure we flush message buffer and clean up behind us before the end of the simulation
	721	FlushAMQP=true
	722	# Fill the rabbitMQ queue
	723	IGCM_debug_sendAMQP
	724	fi
[874]	725
[1206]	726	# Mail notification
	727	IGCM_sys_SendMail
[913]	728
[544]	729	# And Good Bye
	730	date
	731	exit 1
[1206]	732
	733	elif [ X${TaskType} = Xpost-processing ] ; then
	734	# If SpaceName is PROD then we stop when post_processing failed
	735	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
	736	echo " EXIT THE JOB."
	737	echo
	738	# Mail notification?
	739	#IGCM_sys_SendMailPost
	740	# And Good Bye
	741	date
	742	exit 1
	743	else
	744	echo "Either inside config.card the variable SpaceName is not in PROD"
	745	echo "or inside the main Job the variable JobType is not in RUN mode"
	746	echo " SO WE DO NOT EXIT THE JOB."
	747	echo
	748	date
	749	fi
	750	elif [ X${TaskType} = Xchecking ] ; then
	751	echo "Do nothing at the moment"
[2]	752	fi
[544]	753	fi
[2]	754	}
	755
	756	#D-#==================================================================
	757	#D-function IGCM_debug_Print
	758	#D-* Purpose: Print arguments according to a level of verbosity.
	759	#D-
	760	function IGCM_debug_Print
	761	{
[544]	762	typeset level=$1
	763	shift
	764
	765	if [ X"${1}" = X"-e" ]; then
	766	typeset cmd_echo="echo -e"
[2]	767	shift
[544]	768	else
	769	typeset cmd_echo="echo"
	770	fi
[2]	771
[544]	772	if [ ${level} -le ${Verbosity} ] ; then
	773	typeset i
	774	case "${level}" in
	775	1) for i in "$@" ; do
[734]	776	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
[913]	777	done ;;
[544]	778	2) for i in "$@" ; do
[734]	779	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
[913]	780	done ;;
[544]	781	3) for i in "$@" ; do
[734]	782	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
[913]	783	done ;;
[544]	784	esac
	785	fi
[2]	786	}
	787
	788	#D-#==================================================================
	789	#D-function IGCM_debug_PrintVariables
	790	#D-* Purpose: Print arguments when match a pattern
	791	#D- according to a level of verbosity.
	792	function IGCM_debug_PrintVariables
	793	{
[544]	794	typeset level=$1
	795	shift
[2]	796
[830]	797	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
[54]	798
[544]	799	if [ "X${list}" != X ] ; then
	800	IGCM_debug_Print ${level} ${list}
	801	fi
[2]	802	}
	803
	804	#D-#==================================================================
[914]	805	#D-function IGCM_debug_PrintInfosActions
[913]	806	#D-* Purpose: Print information related to instrumentation
	807	function IGCM_debug_PrintInfosActions
	808	{
	809	typeset actionType=$1
	810	typeset entitySize=$2
	811	typeset start_ms=$3
	812	typeset end_ms=$4
	813
	814	typeset dest=$5
	815	typeset source=$6
	816
	817	typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
[1090]	818	typeset dirFrom dirTo
[913]	819
	820	diff_ms=$(( $end_ms - $start_ms ))
	821	# echo "diff_ms=$diff_ms"
	822
	823	entitySizeKo=$( echo ${entitySize} \| gawk -F"\|" '{print $1}' )
	824	# echo "entitySizeKo=$entitySizeKo"
	825	entitySizeMo=$( echo ${entitySize} \| gawk -F"\|" '{print $2}' )
	826
	827	# flux en Ko / ms
	828	flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" \| bc )
	829	# echo "flux_Ko_ms=$flux_Ko_ms"
	830
	831	# flux en Ko / s
	832	flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
	833	# echo "flux_Ko_s=$flux_Ko_s"
	834
	835	# flux en Mo / s
	836	flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" \| bc )
	837	# echo "flux_Mo_s=$flux_Mo_s"
	838
	839	if [ -d $dest ] ; then
[1090]	840	dirTo=$( readlink -f ${dest} )
[913]	841	else
[1090]	842	dirTo=$( readlink -f $( dirname ${dest} ) )
[913]	843	fi
	844
	845	if [ -d $source ] ; then
[1090]	846	dirFrom=$( readlink -f ${source} )
[913]	847	else
[1090]	848	dirFrom=$( readlink -f $( dirname ${source} ) )
[913]	849	fi
	850
[1094]	851	instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" )
[1050]	852
[1115]	853	if [ X${ActivateStackFilling} = Xtrue ] ; then
	854	echo "{${instrumentationContent}}" >> ${StackFileLocation}/${StackFileName}
	855	fi
[1050]	856
	857	# Inform the rabbitMQ queue
	858	if [ X${ActivateBigBro} = Xtrue ] ; then
	859	# RabbitMQ message body
[1065]	860	Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1050]	861	# Fill the rabbitMQ queue
	862	IGCM_debug_sendAMQP
	863	fi
[913]	864	}
	865
	866	#D-#==================================================================
[2]	867	#D-function IGCM_debug_Check
	868	#D- * Purpose: Check the present file by comparison with a reference file
	869	function IGCM_debug_Check
	870	{
[544]	871	#---------------------
	872	if [ ! -n "${libIGCM}" ] ; then
	873	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	874	echo "--Error--> libIGCM variable is not defined"
	875	exit 2
	876	fi
[2]	877
[544]	878	#---------------------
	879	if [ ! -n "${Verbosity}" ] ; then
	880	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	881	echo "--Error--> Verbosity variable is not defined"
	882	exit 3
	883	fi
[2]	884
[544]	885	#---------------------
[1118]	886	# Need to remove timestamps here
	887	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") > /dev/null 2>&1
[1106]	888	status=$?
[2]	889
[1118]	890	if [ ${status} -eq 0 ] ; then
[544]	891	echo "Check libIGCM_debug ..............................................[ OK ]"
	892	else
	893	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	894	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
	895	echo " has produced the file IGCM_debug_Test.ref.failed"
	896	echo " Please analyse differences with the reference file by typing:"
	897	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
	898	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
[1118]	899	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g")
[544]	900	exit 4
	901	fi
	902	#---------------------
[2]	903	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: