Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1225

Last change on this file since 1225 was 1220, checked in by sdipsl, 9 years ago
Any post-procesing errors will stop the computing jobs when SpaceName?=PROD. See #260 Feature was removed some years ago because postprocessing errors were legions. More robust now.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 34.2 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Patrick Brockmann, Martial Mancip
5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#==================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#==================================================
19
20	#==================================================
21	# Add high level verbosity
22	typeset -i Verbosity=${Verbosity:=3}
23
24	#==================================================
25	# DEBUG_debug
26	# Add low level verbosity
27	DEBUG_debug=${DEBUG_debug:=false}
28
29	#==================================================
30	# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31	typeset RandomError=false
32
33	#==================================================
34	# NULL_STR
35	# Default null string
36	typeset -r NULL_STR="_0_"
37
38	#==================================================
39	# libIGCM_CurrentTag
40	# Current libIGCM tag, check compatibilty with *.card
41	typeset -r libIGCM_CurrentTag="1.0"
42
43	#==================================================
44	# Exit Flag (internal debug)
45	# When true, end the master loop AFTER SAVES FILES
46	ExitFlag=false
47
48	#==================================================
49	# Declare a stack of functions calls
50	unset IGCM_debug_Stack
51	unset IGCM_debug_StackArgs
52	unset IGCM_debug_StackTiming
53	IGCM_debug_Stack[0]=${NULL_STR}
54	IGCM_debug_StackArgs[0]=${NULL_STR}
55	IGCM_debug_StackTiming[0]=${NULL_STR}
56	IGCM_debug_LenStack=0
57
58	#D-#==================================================================
59	#D-function IGCM_debug_getDate_ms
60	#D- * Purpose: Give number of milliseconds since 01-jan-1970
61	function IGCM_debug_getDate_ms
62	{
63	typeset nanosecs ms
64	# nano secondes since 01-jan-1970
65	nanosecs=$( date +%s%N )
66
67	# truncate the last 6 digits to get milliseconds since 01-jan-1970
68	ms=${nanosecs:0:${#nanosecs}-6}
69
70	echo "$ms"
71	}
72
73	#D-#==================================================================
74	#D-function IGCM_debug_sizeOfTabContent
75	#D- * Purpose: Give sumed size of a list of files
76	#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
77	#D- * where entityList is a list of files or directory
78	#D- * where dest is either a directory or a file name
79	function IGCM_debug_sizeOfTabContent
80	{
81	typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
82
83	eval set +A entityListe \${${1}}
84	destination=${2}
85	sumSizeKo=0
86
87	# Here we will try to compute size (file or directory size) from local path and not from archive.
88	for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
89	if [ -f ${entityListe[$i]} ] ; then
90	# One file or a bunch of files has been copied without renaming from a visible filesystem
91	iEntity=${entityListe[$i]}
92	elif [ -f ${entityListe[$i]##/*/} ] ; then
93	# One file or a bunch of files has been copied without renaming from an non visible filesystem
94	# remove path /home/login/../ from entityListe elements
95	iEntity=${entityListe[$i]##/*/}
96	elif [ -f ${destination} ] ; then
97	# a file has been copied and renamed
98	iEntity=${destination}
99	elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
100	# a copy in a directory but not in ${PWD}
101	iEntity=${destination}/${entityListe[$i]##/*/}
102	elif [ -d ${entityListe[$i]} ] ; then
103	# a directory has been copied from a non remote place
104	iEntity=${entityListe[$i]}
105	elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
106	# a directory has been copied from a remote archive and not renamed
107	iEntity=${destination}/${entityListe[$i]##/*/}
108	elif [ -d ${destination} ] ; then
109	# a directory has been copied from a remote archive and renamed
110	iEntity=${destination}
111	fi
112	sizeKo=$( du --apparent-size -skL ${iEntity} \| gawk '{print $1}' )
113	sumSizeKo=$(( $sumSizeKo + $sizeKo ))
114	done
115	sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" \| bc )
116	echo "${sumSizeKo}\|${sumSizeMo}"
117	}
118
119	#D-#==================================================================
120	#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
121	#D- * Purpose: Take over AMQP C client using mail as a message recipient
122	#D- * One argument : base64 encoded message
123	#D- * Attach encoded config.card when starting the simulation
124
125	function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
126
127	typeset b64_encoded_msg mail_recipient
128	typeset buffer send_messages mail_frequency
129	typeset last_mail_date__file
130
131	b64_encoded_msg=$1
132
133	mail_recipient="superviseur@ipsl.jussieu.fr"
134	send_messages=0
135	mail_frequency=3600 # in seconds
136	# use to keep track when was last mail sent (maybe to be replaced with global variable)
137	last_mail_date__file=${R_BUF}/.stamp.${config_UserChoices_TagName}.${config_UserChoices_JobName}
138	# use to accumulate messages before sending them
139	buffer=${R_BUF}/.buffer.${config_UserChoices_TagName}.${config_UserChoices_JobName}
140
141	# init
142	if [ ! -f "${buffer}" ]; then
143	touch ${buffer}
144	fi
145
146	if [ ! -f "${last_mail_date__file}" ]; then
147	touch ${last_mail_date__file}
148	else
149	# compute last time the file was changed (in seconds)
150	seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
151	# send message when exceeding threshold
152	[ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
153	fi
154
155	# queue messages in the buffer
156	echo ${b64_encoded_msg} >> ${buffer}
157
158	# send mail
159
160	if [ X${initBigBro} = Xtrue ] ; then
161	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt
162	mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
163	rm -f $buffer ; touch ${buffer} # clear buffer
164	touch ${last_mail_date__file} # memorize last mail date
165	initBigBro=false
166	elif [ ${send_messages} -eq 1 ] ; then
167	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt
168	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
169	rm -f ${buffer} ; touch ${buffer} # flush the buffer
170	touch ${last_mail_date__file} # memorize last mail date
171	fi
172
173	if ( ${FlushAMQP} ) ; then
174	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
175	rm -f ${buffer} # cleaning behind us
176	rm -f ${last_mail_date__file} # cleaning behind us
177	fi
178
179	# Allways all good for now.
180	return 0
181	}
182
183	#D-#==================================================================
184	#D-function IGCM_debug_sendAMQP_Metrics
185	#D- * Purpose: Take over AMQP C client using mail as a message recipient
186	#D- * Two arguments : - Directory where metrics.json files can be found
187	#D- * - Metrics Group Name. metrics will be added to this group
188	#D- * Attach encoded metrics.json files.
189
190	function IGCM_debug_sendAMQP_Metrics {
191
192	typeset mail_recipient encodedBody
193	if [ X${ActivateBigBro} = Xtrue ] ; then
194	mail_recipient="superviseur@ipsl.jussieu.fr"
195	# Metrics tag on server side
196	code=7100
197	# Usual AMQP message to route messages on server side
198	encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"metricsGroupName\":\"${2}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" \| base64 -w 0 )
199	# send mail
200	attachmentsOptions=""
201	for metricsFile in $( ls $1/*json ) ; do
202	attachmentsOptions="-a ${metricsFile} ${attachmentsOptions}"
203	done
204	IGCM_debug_Print 2 "IGCM_debug_sendAMQP_Metrics "
205	echo ${encodedBody}\|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient}
206	fi
207
208	# Allways all good for now.
209	return 0
210	}
211
212	#D-#==================================================================
213	#D-function IGCM_debug_SendAMQP
214	#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
215	function IGCM_debug_sendAMQP {
216
217	typeset decal first additionnalOption encodedBody
218
219	# Encode message Body
220	encodedBody=$( echo "${Body}" \| base64 -w 0 )
221
222	# Send config.card ?
223	if [ X${1} = Xactivate ] ; then
224	# Encode config.card
225	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
226	# Prepare additionnal option
227	additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
228	#
229	initBigBro=true
230	else
231	additionnalOption=
232	#
233	initBigBro=false
234	fi
235
236	# Only cosmetics : stack file
237	if [ X${ActivateStackFilling} = Xtrue ] ; then
238	decal=0
239	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
240	printf ' ' >> ${StackFileLocation}/${StackFileName}
241	(( decal = decal + 1 ))
242	done
243	# Log to stack file using human readable format
244	echo "${Body}" >> ${StackFileLocation}/${StackFileName}
245	fi
246
247	# Log separately encoded AMQP message command for reuse in a mock up
248	#echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> ${RUN_DIR_PATH}/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
249
250	# Send the message
251	if [ X${BigBrotherChannel} = XMAIL ] ; then
252	IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
253	status=$?
254	else
255	sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
256	status=$?
257	fi
258
259	if [ ${status} -gt 0 ] ; then
260	IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
261	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
262	exit 1
263	fi
264	}
265
266	#D-#==================================================================
267	#D-function IGCM_debug_CallStack
268	#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
269	#D-
270	function IGCM_debug_CallStack {
271	if ( $DEBUG_debug ) ; then
272	# Cosmetics
273	typeset i decal
274	i=0
275	until [ $i -eq ${IGCM_debug_LenStack} ]; do
276	decal=0
277	until [ $decal -eq ${i} ]; do
278	printf -- ' '
279	(( decal = decal + 1 ))
280	done
281	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
282	((i = i + 1))
283	done
284	fi
285	}
286
287	#D-#==================================================================
288	#D-function IGCM_debug_PushStack
289	#D-* Purpose: Push a function name in the stack
290	#D-
291	function IGCM_debug_PushStack {
292	if ( $DEBUG_debug ) ; then
293	typeset decal inputs startTime_ms
294
295	# Only cosmetics : stack file
296	if [ X${ActivateStackFilling} = Xtrue ] ; then
297	echo >> ${StackFileLocation}/${StackFileName}
298	decal=0
299	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
300	printf ' ' >> ${StackFileLocation}/${StackFileName}
301	(( decal = decal + 1 ))
302	done
303
304	# Fill the stack file
305	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
306	fi
307
308	# Save input list in an indexed array
309	INPUTS=( $@ )
310
311	# Get timing information
312	startTime_ms=$( IGCM_debug_getDate_ms )
313
314	# We add function call name on beginning of the stack
315	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
316
317	# Save timing in milliseconds in an indexed array
318	set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
319
320	# We include the "null" Args in the beginning of the StackArgs
321	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
322
323	# Then, we shift StackArgs tabular
324	# Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
325	if [ $# -gt 1 ]; then
326	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /\",\"/g" )
327	fi
328
329	# Unplugged message 4000 handling for now. To ease downstream treatment.
330	#if [ X${ActivateBigBro} = Xtrue ] ; then
331	# # RabbitMQ message code "PUSHSTACK"
332	# code=4000
333	# # RabbitMQ message body
334	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
335	# # Fill the rabbitMQ queue
336	# IGCM_debug_sendAMQP
337	#fi
338
339	# Increment LenStack
340	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
341
342	#IGCM_debug_CallStack
343	fi
344	}
345
346	#D-#==================================================================
347	#D-function IGCM_debug_PopStack
348	#D-* Purpose: Pop a function name in the stack
349	#D-
350	function IGCM_debug_PopStack {
351	if ( $DEBUG_debug ) ; then
352	typeset i decal command arguments startTime_ms endTime_ms
353	typeset instrumentation dest prefix
354	# they are not typeset because they are send "by adress" to son functions
355	# we unset them to avoid "memory effect"
356	unset fileList source
357
358	# INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
359	# PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
360	# THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
361	if ( ${RandomError} ) ; then
362	if [ $((RANDOM%10000)) -le 10 ] ; then
363	IGCM_debug_Print 1 "Random error has been triggered"
364	if [ X${ActivateStackFilling} = Xtrue ] ; then
365	echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
366	fi
367	ExitFlag=true
368	fi
369	fi
370
371	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
372	# Everything is cool
373
374	# Get timing information
375	endTime_ms=$( IGCM_debug_getDate_ms )
376
377	# Save Stack information before poping the stack
378	command=${IGCM_debug_Stack[0]}
379
380	# Go from comma separated list of quoted elements (except the first and the last element)
381	# to unquoted space separated elements in an array
382	set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} \| sed -e "s/\",\"/\ /g" )
383
384	# Save Stack information before poping the stack
385	startTime_ms=${IGCM_debug_StackTiming[0]}
386
387	# Pop the stack
388	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
389	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
390	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
391	set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
392	else
393	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
394	IGCM_debug_Exit $@
395	fi
396
397	# Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
398	# We are interested in:
399	# 0. Which command performs the work
400	# 1. Size of entity we are working with
401	# 2. Where are we reading
402	# 3. Where are we writing
403	# 4. How long it took
404
405	instrumentation=false
406
407	case ${command} in
408	# Classical copy (only files are given to IGCM_sys_Cp as options)
409	IGCM_sys_Cp)
410	instrumentation=true
411	# All but the latest
412	fileList=${arguments[]:0:${#arguments[]}-1}
413	# just need the first file to get the directory
414	source=${arguments[0]}
415	# Nothing but the latest
416	dest=${arguments[${#arguments[*]}-1]}
417	# Size of file whose name are stored in a list
418	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
419	;;
420
421	# Copy from archive machine or from buffer
422	IGCM_sys_Get\|IGCM_sys_GetBuffer)
423	instrumentation=true
424	if [ ${#arguments[*]} -eq 2 ] ; then
425	source=${arguments[0]}
426	dest=${arguments[1]}
427	# Size of file whose name are stored in a variable
428	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
429	elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
430	# IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
431	# Keep the array name hosting the all list
432	eval set +A fileList \${${arguments[1]}}
433	# just need the first file to get the directory
434	source=${fileList[0]}
435	dest=${arguments[2]}
436	# Size of file whose name are stored in a list
437	entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
438	elif [ [ ${#arguments[*]} -ge 3 ] ; then
439	# All but the latest
440	fileList=${arguments[]:0:${#arguments[]}-1}
441	# just need the first file to get the directory
442	source=${arguments[0]}
443	# Nothing but the latest
444	dest=${arguments[${#arguments[*]}-1]}
445	# Size of file whose name are stored in a list
446	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
447	fi
448	;;
449
450	# Copy from compute node or copy to archive/buffer
451	IGCM_sys_Get_Master\|IGCM_sys_Get_Dir\|IGCM_sys_Put_Out\|IGCM_sys_PutBuffer_Out)
452	instrumentation=true
453	source=${arguments[0]}
454	dest=${arguments[1]}
455	# Size of file whose name are stored in a variable
456	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
457	;;
458
459	# Rebuild command
460	IGCM_sys_rebuild\|IGCM_sys_rebuild_station)
461	instrumentation=true
462	# All but the first
463	fileList=${arguments[]:1:${#arguments[]}-1}
464	# just need a file to get the directory
465	source=${arguments[1]}
466	# Nothing but the first
467	dest=${arguments[0]}
468	# Size of file whose name are stored in a list
469	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
470	;;
471
472	# NCO commands
473	IGCM_sys_ncrcat\|IGCM_sys_ncecat\|IGCM_sys_ncra\|IGCM_sys_ncks\|IGCM_sys_cdo)
474	# Example of what we want to catch : only filenames in those command lines
475	# IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
476	# IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
477	# IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
478	# IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
479	instrumentation=true
480	keepGoing=true
481	prefix=.
482	i=0
483	while ( ${keepGoing} ) ; do
484	# the last one is not interesting
485	if [ ${i} -eq ${#arguments[*]}-1 ] ; then
486	keepGoing=false
487	# look after "-p" option. Path prefix is the following arguments
488	elif [ ${arguments[${i}]} = "-p" ] ; then
489	((i = i + 1))
490	prefix=${arguments[${i}]}
491	((i = i + 1))
492	elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
493	keepGoing=false
494	# looking for files
495	elif [ -f ${prefix}/${arguments[${i}]} ] ; then
496	fileList="${fileList} ${prefix}/${arguments[${i}]}"
497	((i = i + 1))
498	# other options are not interesting
499	else
500	((i = i + 1))
501	fi
502	done
503
504	# i value is at least 1
505	# just need one file to get the directory
506	source=$( echo ${fileList} \| gawk '{print $1}' )
507	# Nothing but the latest
508	dest=${arguments[${#arguments[*]}-1]}
509	# Size of file whose name are stored in a list
510	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
511	;;
512	esac
513
514	# Print information related to instrumentation
515	( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
516
517	# Only cosmetics : stack file
518	if [ X${ActivateStackFilling} = Xtrue ] ; then
519	decal=0
520	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
521	printf ' ' >> ${StackFileLocation}/${StackFileName}
522	(( decal = decal + 1 ))
523	done
524	fi
525
526	if ( ${ExitFlag} ) ; then
527	# Inform the stack file
528	if [ X${ActivateStackFilling} = Xtrue ] ; then
529	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
530	fi
531
532	# Unplugged message 4900 handling for now. To ease downstream treatment.
533	#if [ X${ActivateBigBro} = Xtrue ] ; then
534	# # RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
535	# code=4900
536	# # RabbitMQ message body
537	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
538	# # Fill the rabbitMQ queue
539	# IGCM_debug_sendAMQP
540	#fi
541	else
542	# Inform the stack file
543	if [ X${ActivateStackFilling} = Xtrue ] ; then
544	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
545	fi
546
547	# Unplugged message 4100 handling for now. To ease downstream treatment.
548	#if [ X${ActivateBigBro} = Xtrue ] ; then
549	# # RabbitMQ message code "POPSTACK"
550	# code=4100
551	# # RabbitMQ message body
552	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
553	# # Fill the rabbitMQ queue
554	# IGCM_debug_sendAMQP
555	#fi
556	fi
557
558	# Reset array if necessary
559	if [ ${IGCM_debug_LenStack} = 0 ]; then
560	#echo
561	#IGCM_debug_Print 3 "Clean stack array"
562	#echo
563	unset IGCM_debug_Stack
564	unset IGCM_debug_StackArgs
565	unset IGCM_debug_StackTiming
566	IGCM_debug_Stack[0]=${NULL_STR}
567	IGCM_debug_StackArgs[0]=${NULL_STR}
568	IGCM_debug_StackTiming[0]=${NULL_STR}
569	fi
570	fi
571	#IGCM_debug_CallStack
572	}
573
574	#D-#==================================================================
575	#D-function IGCM_debug_BigBro_Initialize
576	#D-* Purpose: switch rabbitMQ on
577	#D-
578	function IGCM_debug_BigBro_Initialize {
579	IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize"
580
581	# Message type standard fields:
582	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
583
584	# Message type dictionnary and custom fields:
585	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
586
587	if [ X${BigBrother} = Xtrue ] ; then
588	# create a unique ID for this specific job
589	jobuid=$(uuidgen)
590
591	if [ X${TaskType} = Xcomputing ]; then
592	if ( ${FirstInitialize} ) ; then
593	# RabbitMQ message code "BEGIN A SIMULATION"
594	code=0000
595	# create and persist a unique id for this simulation
596	simuid=$(uuidgen)
597	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
598	# Standard fields for the first message
599	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
600	# RabbitMQ message body with specific fields associated message codes treated here
601	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
602	# Fill the rabbitMQ queue (the config.card in use will be sent)
603	IGCM_debug_sendAMQP activate
604	else
605	# RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION"
606	code=1000
607	# retrieve this simulation's unique id
608	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
609	simuid=${run_Configuration_simuid}
610	# Using standard fields for message others than the first one. Still subject to change
611	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
612	# RabbitMQ message body with specific fields associated message codes treated here
613	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
614	# Fill the rabbitMQ queue
615	IGCM_debug_sendAMQP
616	fi
617
618	# NOT VERY NICE BUT ... IT WORKS
619	# Be sure that the genericSimulationID will be small from now on
620	# Using standard fields for messages others than the first one. Still subject to change
621	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
622
623	elif [ X${TaskType} = Xpost-processing ]; then
624	# RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION"
625	code=2000
626	# retrieve this simulation's unique id
627	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
628	simuid=${run_Configuration_simuid}
629	# Using standard fields for message others than the first one. Still subject to change
630	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
631	# RabbitMQ message body with specific fields associated message codes treated here
632	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
633	# Fill the rabbitMQ queue
634	IGCM_debug_sendAMQP
635	fi
636	# Turn the flag on
637	ActivateBigBro=true
638	# Dont flush by default
639	FlushAMQP=false
640	fi
641	IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize"
642	}
643
644	#D-#==================================================================
645	#D-function IGCM_debug_BigBro_Finalize
646	#D-* Purpose: Finalize rabbitMQ messages exchanges
647	#D-
648	function IGCM_debug_BigBro_Finalize {
649	IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize"
650
651	# Message type standard fields:
652	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
653
654	# Message type dictionnary and custom fields:
655	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
656
657	if ( $DEBUG_debug ) ; then
658	if [ X${ActivateBigBro} = Xtrue ] ; then
659	if [ X${TaskType} = Xcomputing ]; then
660	if ( ${simulationIsOver} ) ; then
661	# RabbitMQ message code "SIMULATION ENDS"
662	code=0100
663	FlushAMQP=true
664	elif ( ${ExitFlag} ) ; then
665	# RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
666	code=9999
667	FlushAMQP=true
668	else
669	# RabbitMQ message code "COMPUTING JOB ENDS"
670	code=1100
671	fi
672	elif [ X${TaskType} = Xpost-processing ]; then
673	if ( ${ExitFlag} ) ; then
674	# RabbitMQ message code "POST-PROCESSING JOB FAILS"
675	code=2900
676	FlushAMQP=true
677	else
678	# RabbitMQ message code "POST-PROCESSING JOB ENDS"
679	code=2100
680	FlushAMQP=true
681	fi
682	fi
683	# RabbitMQ message body
684	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
685	# Fill the rabbitMQ queue
686	IGCM_debug_sendAMQP
687	fi
688	fi
689
690	IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize"
691	}
692
693	#D-#==================================================================
694	#D-function IGCM_debug_Exit
695	#D-* Purpose: Print Call Stack and set ExitFlag to true
696	#D-
697	function IGCM_debug_Exit {
698	IGCM_debug_PushStack "IGCM_debug_Exit"
699	echo "IGCM_debug_Exit : " "${@}"
700	echo
701	echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
702	echo "!! ERROR TRIGGERED !!"
703	echo "!! EXIT FLAG SET !!"
704	echo "!------------------------!"
705	echo
706	IGCM_debug_CallStack
707	ExitFlag=true
708	IGCM_debug_PopStack "IGCM_debug_Exit"
709	}
710
711	#D-#==================================================
712	#D-function IGCM_debug_Verif_Exit
713	#D-* Purpose: exit with number 1 if ExitFlag is true
714	#D-
715	function IGCM_debug_Verif_Exit {
716	if ( ${ExitFlag} ) ; then
717	echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
718	echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
719	# Only computing TaskType stops the job for now.
720	if [ X${TaskType} = Xcomputing ] ; then
721	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
722	echo " EXIT THE JOB."
723	echo
724	IGCM_debug_CallStack
725
726	# Mail notification
727	IGCM_sys_SendMail
728
729	# Inform the rabbitMQ queue
730	IGCM_debug_BigBro_Finalize
731
732	# And Good Bye
733	date
734	exit 1
735
736	elif [ X${TaskType} = Xpost-processing ] ; then
737	# If SpaceName is PROD then we stop when post_processing failed
738	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
739	echo " EXIT THE POST-PROCESSING JOB."
740	echo
741	IGCM_debug_CallStack
742
743	# Notify the computing job that something wrong happened.
744	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
745
746	# Mail notification?
747	#IGCM_sys_SendMailPost
748
749	# Inform the rabbitMQ queue
750	IGCM_debug_BigBro_Finalize
751
752	# And Good Bye
753	date
754	exit 1
755	else
756	echo "In config.card the variable SpaceName is not in PROD"
757	echo " SO WE DO NOT EXIT THE JOB."
758	echo
759
760	# Inform the rabbitMQ queue
761	IGCM_debug_BigBro_Finalize
762
763	date
764	fi
765	elif [ X${TaskType} = Xchecking ] ; then
766	echo "Nothing will happen for now"
767	fi
768	fi
769	}
770
771	#D-#==================================================================
772	#D-function IGCM_debug_Print
773	#D-* Purpose: Print arguments according to a level of verbosity.
774	#D-
775	function IGCM_debug_Print
776	{
777	typeset level=$1
778	shift
779
780	if [ X"${1}" = X"-e" ]; then
781	typeset cmd_echo="echo -e"
782	shift
783	else
784	typeset cmd_echo="echo"
785	fi
786
787	if [ ${level} -le ${Verbosity} ] ; then
788	typeset i
789	case "${level}" in
790	1) for i in "$@" ; do
791	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
792	done ;;
793	2) for i in "$@" ; do
794	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
795	done ;;
796	3) for i in "$@" ; do
797	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
798	done ;;
799	esac
800	fi
801	}
802
803	#D-#==================================================================
804	#D-function IGCM_debug_PrintVariables
805	#D-* Purpose: Print arguments when match a pattern
806	#D- according to a level of verbosity.
807	function IGCM_debug_PrintVariables
808	{
809	typeset level=$1
810	shift
811
812	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
813
814	if [ "X${list}" != X ] ; then
815	IGCM_debug_Print ${level} ${list}
816	fi
817	}
818
819	#D-#==================================================================
820	#D-function IGCM_debug_PrintInfosActions
821	#D-* Purpose: Print information related to instrumentation
822	function IGCM_debug_PrintInfosActions
823	{
824	typeset actionType=$1
825	typeset entitySize=$2
826	typeset start_ms=$3
827	typeset end_ms=$4
828
829	typeset dest=$5
830	typeset source=$6
831
832	typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
833	typeset dirFrom dirTo
834
835	diff_ms=$(( $end_ms - $start_ms ))
836	# echo "diff_ms=$diff_ms"
837
838	entitySizeKo=$( echo ${entitySize} \| gawk -F"\|" '{print $1}' )
839	# echo "entitySizeKo=$entitySizeKo"
840	entitySizeMo=$( echo ${entitySize} \| gawk -F"\|" '{print $2}' )
841
842	# flux en Ko / ms
843	flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" \| bc )
844	# echo "flux_Ko_ms=$flux_Ko_ms"
845
846	# flux en Ko / s
847	flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
848	# echo "flux_Ko_s=$flux_Ko_s"
849
850	# flux en Mo / s
851	flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" \| bc )
852	# echo "flux_Mo_s=$flux_Mo_s"
853
854	if [ -d $dest ] ; then
855	dirTo=$( readlink -f ${dest} )
856	else
857	dirTo=$( readlink -f $( dirname ${dest} ) )
858	fi
859
860	if [ -d $source ] ; then
861	dirFrom=$( readlink -f ${source} )
862	else
863	dirFrom=$( readlink -f $( dirname ${source} ) )
864	fi
865
866	instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" )
867
868	if [ X${ActivateStackFilling} = Xtrue ] ; then
869	echo "{${instrumentationContent}}" >> ${StackFileLocation}/${StackFileName}
870	fi
871
872	# Inform the rabbitMQ queue
873	if [ X${ActivateBigBro} = Xtrue ] ; then
874	# RabbitMQ message body
875	Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
876	# Fill the rabbitMQ queue
877	IGCM_debug_sendAMQP
878	fi
879	}
880
881	#D-#==================================================================
882	#D-function IGCM_debug_Check
883	#D- * Purpose: Check the present file by comparison with a reference file
884	function IGCM_debug_Check
885	{
886	#---------------------
887	if [ ! -n "${libIGCM}" ] ; then
888	echo "Check libIGCM_debug ..........................................[ FAILED ]"
889	echo "--Error--> libIGCM variable is not defined"
890	exit 2
891	fi
892
893	#---------------------
894	if [ ! -n "${Verbosity}" ] ; then
895	echo "Check libIGCM_debug ..........................................[ FAILED ]"
896	echo "--Error--> Verbosity variable is not defined"
897	exit 3
898	fi
899
900	#---------------------
901	# Need to remove timestamps here
902	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") > /dev/null 2>&1
903	status=$?
904
905	if [ ${status} -eq 0 ] ; then
906	echo "Check libIGCM_debug ..............................................[ OK ]"
907	else
908	echo "Check libIGCM_debug ..........................................[ FAILED ]"
909	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
910	echo " has produced the file IGCM_debug_Test.ref.failed"
911	echo " Please analyse differences with the reference file by typing:"
912	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
913	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
914	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g")
915	exit 4
916	fi
917	#---------------------
918	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: