Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 926

Last change on this file since 926 was 926, checked in by sdipsl, 11 years ago
Add instrumentation for IGCM_sys_ncrcat IGCM_sys_ncecat IGCM_sys_ncra IGCM_sys_ncks IGCM_sys_cdo see #140 stack files handling : one stack file per TaskType? : computing.stack, post-processsing, checking.stack
Property svn:keywords set to `Revision Author Date`
File size: 82.1 KB

Rev	Line
[622]	1	#!/bin/ksh
	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[623]	16	# if you use the prefix #D- for comments to be extracted.
[622]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[623]	21	#D-LibIGCM_sys for Curie
[622]	22	#D-#==================================================
	23	#D-
[623]	24	#D- This ksh library if a layer under some usefull
[622]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[622]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
	64	# Host and user names
	65	# $hostname ou hostname
	66	typeset HOST=${HOST:=$( hostname )}
	67	# $username ou whoami
	68	typeset LOGIN=${LOGIN:=$( whoami )}
	69	# $hostname of the MASTER job
	70	typeset MASTER=curie
[880]	71	# add default project on curie
[900]	72	typeset DEFAULT_PROJECT=gen0826
[622]	73
	74	#D-
	75	#D-#==================================================
	76	#D-Program used in libIGCM
	77	#D-#==================================================
	78
[742]	79	# Submit command
[747]	80	typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]	81	# rsync with path
	82	typeset -r RSYNC=/usr/bin/rsync
	83	# RSYNC_opt args to rsync
	84	typeset -r RSYNC_opt="-va"
	85	# ie storage filesystem
[623]	86	typeset -r STOREHOST=${MASTER}
[622]	87
	88	#====================================================
	89	# Source default environment
	90	#====================================================
	91	##. /etc/profile
	92
	93	#====================================================
	94	# Set environment tools (ferret, nco, cdo)
	95	#====================================================
[712]	96	if [ X${TaskType} = Xcomputing ] ; then
	97	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_calcul_curie_ksh
[739]	98	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64/bin
[712]	99	else
	100	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_post_curie_ksh
	101	fi
[622]	102
	103	#====================================================
	104	# Specific for ocean additionnal diagnostic
	105	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	106	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
	107
	108	#====================================================
	109	# Host specific DIRECTORIES
	110	#====================================================
	111
[623]	112	# ============ CESIUM START ============ #
	113
[622]	114	#====================================================
[623]	115	#- Mirror libIGCM from titane to cesium if needed
	116	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	117	#if [ ! ${ROOTSYS} = "home" ] ; then
	118	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	119	#else
	120	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	121	#fi
	122
	123	#====================================================
	124	#- libIGCM_POST
	125	#if ( ${MirrorlibIGCM} ) ; then
	126	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	127	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	128	#else
	129	# typeset -r libIGCM_POST=${libIGCM}
	130	#fi
	131
	132	# ============ CESIUM END ============ #
	133
	134	#====================================================
	135	#- MirrorlibIGCM for frontend
	136	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	137
	138	#====================================================
	139	#- libIGCM_POST for frontend
	140	typeset -r libIGCM_POST=${libIGCM}
	141
	142	#====================================================
[622]	143	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	144	typeset -r R_EXE="${MODIPSL}/bin"
	145
	146	#====================================================
	147	#- SUBMIT_DIR : submission dir
	148	typeset SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
	149
	150	#====================================================
	151	#- IN
	152	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
	153	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
	154
	155	#====================================================
[725]	156	#- ARCHIVE (dedicated to large files)
	157	typeset ARCHIVE=${CCCSTOREDIR}
	158
	159	#====================================================
	160	#- STORAGE (dedicated to small/medium files)
	161	typeset STORAGE=${CCCWORKDIR}
	162
	163	#====================================================
[623]	164	#- R_OUT
[725]	165	typeset R_OUT=${ARCHIVE}/IGCM_OUT
[622]	166
	167	#====================================================
[623]	168	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[725]	169	typeset R_FIG=${STORAGE}/IGCM_OUT
[622]	170
	171	#====================================================
[623]	172	#- R_BUF (ONLY FOR double copy an scratch)
	173	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
	174
	175	#====================================================
[622]	176	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[850]	177	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[622]	178
	179	#====================================================
	180	#- HOST_MPIRUN_COMMAND
[808]	181	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
[622]	182
	183	#====================================================
	184	#- Max number of arguments passed to nco operator or demigration command
	185	UNIX_MAX_LIMIT=360
	186
[623]	187	#====================================================
	188	#- set PackDefault true on curie
	189	PackDefault=true
	190
[664]	191	#====================================================
	192	#- Number of core per node (max number of OpenMP task)
[622]	193	NUM_COREPERNODE=8
	194
[623]	195	#====================================================
	196	#- Default number of MPI task for IPSL coupled model
	197	#- required for backward compatibility
	198	#-
	199	DEFAULT_NUM_PROC_OCE=5
	200	DEFAULT_NUM_PROC_CPL=1
[637]	201	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
	202	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
[623]	203
[725]	204	#D-#==================================================
	205	#D-function IGCM_sys_ChangeArchive
	206	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
	207	#D-* Examples:
	208	#D-
	209	function IGCM_sys_ChangeArchive {
	210	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
[676]	211
[743]	212	R_OUT=${SCRATCHDIR}/IGCM_OUT
	213	R_FIG=${SCRATCHDIR}/IGCM_OUT
[725]	214
	215	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
	216	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
	217
	218	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
	219	}
	220
[622]	221	#D-#==================================================
	222	#D-function IGCM_sys_RshMaster
[623]	223	#D-* Purpose: Connection to frontend machine.
[622]	224	#D-* Examples:
	225	#D-
	226	function IGCM_sys_RshMaster {
[623]	227	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
	228	/bin/ksh <<-EOF
[622]	229	export libIGCM=${libIGCM}
	230	export DEBUG_debug=${DEBUG_debug}
	231	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
	232	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
	233	${@}
	234	EOF
[623]	235	if [ $? -gt 0 ] ; then
	236	echo "IGCM_sys_RshMaster : erreur."
	237	IGCM_debug_Exit "IGCM_sys_RshMaster"
	238	fi
	239	IGCM_debug_PopStack "IGCM_sys_RshMaster"
[622]	240	}
	241
	242	#D-#==================================================
	243	#D-function IGCM_sys_RshArchive
	244	#D-* Purpose: Archive rsh command
	245	#D-* Examples:
	246	#D-
	247	function IGCM_sys_RshArchive {
[623]	248	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	249	/bin/ksh <<-EOF
[622]	250	${@}
	251	EOF
[811]	252	status=$?
	253	if [ ${status} -gt 0 ] ; then
	254	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]	255	IGCM_debug_Exit "IGCM_sys_RshArchive"
	256	fi
	257	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]	258	}
	259
	260	#D-#==================================================
[845]	261	#D-function IGCM_sys_RshArchive_NoError
	262	#D-* Purpose: Archive rsh command, without error
	263	#D-* used only in monitoring.job
	264	#D-* Examples:
	265	#D-
	266	function IGCM_sys_RshArchive_NoError {
	267	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
	268	/bin/ksh <<-EOF
	269	${@} 2> \dev\null
	270	EOF
	271	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
	272	}
	273
	274	#D-#==================================================
[622]	275	#D-function IGCM_sys_RshPost
	276	#D-* Purpose: Post-process rsh command
	277	#D-* Examples:
	278	#D-
	279	function IGCM_sys_RshPost {
[623]	280	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
	281	if ( $DEBUG_sys ) ; then
	282	echo "IGCM_sys_RshPost :" $@
	283	fi
	284	# keep standard input (stdin) for the loop onto temporary file
[811]	285	cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
[623]	286
[811]	287	/bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
[623]	288	if [ $? -gt 0 ] ; then
	289	echo "IGCM_sys_RshPost : erreur."
	290	IGCM_debug_Exit "IGCM_sys_RshPost"
	291	fi
	292	# delete temporary file
[811]	293	\rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
[623]	294
	295	# ============ FRONTEND END ============ #
	296
	297	# ============ CESIUM START ============ #
	298	# typeset NB_ESSAI DELAI status i
	299	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
	300	# #little hack so that rebuild submission is done on titane not an cesium
	301	#
	302	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
	303	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
	304	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
	305	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
	306	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
	307	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
	308	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
	309	#
	310	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
	311	# cat tmp_IGCM_sys_RshPost_$$
	312	#
	313	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
	314	# if [ $? -gt 0 ] ; then
	315	# echo "IGCM_sys_RshPost : erreur."
	316	# IGCM_debug_Exit "IGCM_sys_RshPost"
	317	# fi
	318	# # delete temporary file
	319	# \rm tmp_IGCM_sys_RshPost_$$
	320	#
	321	# else
	322	# # number of tentative
	323	# NB_ESSAI=10
	324	# # time delay between tentative
	325	# DELAI=10
	326	# i=0
	327	# while [ $i -ne $NB_ESSAI ] ; do
	328	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
	329	# status=$?
	330	# if [ ${status} -ne 0 ];
	331	# then
[785]	332	# sleep $DELAI
[623]	333	# else
[785]	334	# break
[623]	335	# fi
	336	# let i=$i+1
	337	# done
	338	# # delete temporary file
	339	# \rm tmp_IGCM_sys_RshPost_$$
	340	#
	341	# if [ ${status} -gt 0 ] ; then
	342	# echo "IGCM_sys_RshPost : erreur."
	343	# IGCM_debug_Exit "IGCM_sys_RshPost"
	344	# fi
	345	# fi
	346
	347	# ============ CESIUM END ============ #
	348
	349	IGCM_debug_PopStack "IGCM_sys_RshPost"
[622]	350	}
	351
	352	#D-#==================================================
	353	#D-function IGCM_sys_SendMail
	354	#D-* Purpose: Send mail when simulation is over
	355	#D-* Examples:
	356	#D-
	357	function IGCM_sys_SendMail {
[623]	358	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
[922]	359
[623]	360	if ( $DEBUG_sys ) ; then
	361	echo "IGCM_sys_SendMail :" $@
	362	fi
[622]	363
[922]	364	if [ X${1} = XAccounting ] ; then
	365	status=Accounting
	366	mailText=jobAccounting.mail
	367	elif ( ${ExitFlag} ) ; then
[623]	368	status=failed
[922]	369	mailText=jobEnd.mail
[623]	370	else
	371	status=completed
[922]	372	mailText=jobEnd.mail
[623]	373	fi
[622]	374
[922]	375	# Update selected mail template
	376	while read -r line; do
	377	eval echo $line >> mail.txt ;
	378	done < ${libIGCM}/libIGCM_sys/${mailText}
[622]	379
[806]	380	if [ ! -z ${config_UserChoices_MailName} ] ; then
[922]	381	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < mail.txt
[806]	382	elif [ -f ~/.forward ] ; then
[922]	383	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
[806]	384	fi
[811]	385
[806]	386	sleep 10
[922]	387	rm -f mail.txt
[811]	388
[623]	389	IGCM_debug_PopStack "IGCM_sys_SendMail"
[622]	390	}
	391
	392	#D-#==================================================
	393	#D-function IGCM_sys_Mkdir
	394	#D-* Purpose: Master locale mkdir command
	395	#D-* Examples:
	396	#D-
	397	function IGCM_sys_Mkdir {
[623]	398	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
	399	if ( $DEBUG_sys ) ; then
	400	echo "IGCM_sys_Mkdir :" $@
	401	fi
	402	if [ ! -d ${1} ]; then
	403	\mkdir -p $1
	404	if [ $? -gt 0 ] ; then
	405	echo "IGCM_sys_Mkdir : erreur."
	406	IGCM_debug_Exit "IGCM_sys_Mkdir"
[622]	407	fi
[623]	408	fi
	409	# vérification :
	410	if [ ! -d ${1} ] ; then
	411	echo "IGCM_sys_Mkdir : erreur."
	412	IGCM_debug_Exit "IGCM_sys_Mkdir"
	413	fi
	414	IGCM_debug_PopStack "IGCM_sys_Mkdir"
[622]	415	}
	416
	417	#D-#==================================================
	418	#D-function IGCM_sys_MkdirArchive
	419	#D-* Purpose: Mkdir on Archive
	420	#D-* Examples:
	421	#D-
	422	function IGCM_sys_MkdirArchive {
[623]	423	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	424	if ( $DEBUG_sys ) ; then
	425	echo "IGCM_sys_MkdirArchive :" $@
	426	fi
	427	#- creation de repertoire sur le serveur fichier
	428	if [ ! -d ${1} ]; then
	429	\mkdir -p $1
[811]	430	status=$?
	431
	432	if [ ${status} -gt 0 ] ; then
	433	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]	434	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]	435	fi
[623]	436	fi
	437	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]	438	}
	439
	440	#D-#==================================================
	441	#D-function IGCM_sys_MkdirWork
	442	#D-* Purpose: Mkdir on Work
	443	#D-* Examples:
	444	#D-
	445	function IGCM_sys_MkdirWork {
[623]	446	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
	447	if ( $DEBUG_sys ) ; then
	448	echo "IGCM_sys_MkdirWork :" $@
	449	fi
	450	#- creation de repertoire sur le serveur fichier
	451	if [ ! -d ${1} ]; then
	452	\mkdir -p $1
	453	if [ $? -gt 0 ] ; then
	454	echo "IGCM_sys_MkdirWork : erreur."
	455	IGCM_debug_Exit "IGCM_sys_MkdirWork"
[622]	456	fi
[623]	457	fi
	458	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
[622]	459	}
	460
	461	#D-#==================================================
	462	#D-function IGCM_sys_Cd
	463	#D-* Purpose: master cd command
	464	#D-* Examples:
	465	#D-
	466	function IGCM_sys_Cd {
[623]	467	IGCM_debug_PushStack "IGCM_sys_Cd" $@
	468	if ( $DEBUG_sys ) ; then
	469	echo "IGCM_sys_Cd :" $@
	470	fi
	471	\cd $1
	472	if [ $? -gt 0 ] ; then
	473	echo "IGCM_sys_Cd : erreur."
	474	IGCM_debug_Exit "IGCM_sys_Cd"
	475	fi
	476	IGCM_debug_PopStack "IGCM_sys_Cd"
[622]	477	}
	478
	479	#D-#==================================================
	480	#D-function IGCM_sys_Chmod
	481	#D-* Purpose: Chmod
	482	#D-* Examples:
	483	#D-
	484	function IGCM_sys_Chmod {
[623]	485	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
	486	if ( $DEBUG_sys ) ; then
	487	echo "IGCM_sys_Chmod :" $@
	488	fi
[809]	489	\chmod $@
	490	if [ $? -gt 0 ] ; then
	491	echo "IGCM_sys_Chmod : erreur."
	492	IGCM_debug_Exit "IGCM_sys_Chmod"
[623]	493	fi
	494	IGCM_debug_PopStack "IGCM_sys_Chmod"
[622]	495	}
	496
	497	#D-#==================================================
	498	#D-function IGCM_sys_FileSize
	499	#D-* Purpose: Filesize
	500	#D-* Examples:
	501	#D-
	502	function IGCM_sys_FileSize {
[623]	503	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
[622]	504
[623]	505	typeset sizeF
	506	set +A sizeF -- $( ls -la ${1} )
	507	if [ $? -gt 0 ] ; then
	508	IGCM_debug_Exit "IGCM_sys_FileSize"
	509	fi
	510	eval ${2}=${sizeF[4]}
[622]	511
[623]	512	IGCM_debug_PopStack "IGCM_sys_FileSize"
[622]	513	}
	514
	515	#D-#==================================================
	516	#D-function IGCM_sys_TestDir
	517	#D-* Purpose: Test Directory that must exists
	518	#D-* Examples:
	519	#D-
	520	function IGCM_sys_TestDir {
[623]	521	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
	522	if ( $DEBUG_sys ) ; then
	523	echo "IGCM_sys_TestDir :" $@
	524	fi
	525	typeset ExistFlag
	526	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	527	IGCM_debug_PopStack "IGCM_sys_TestDir"
[622]	528
[623]	529	return ${ExistFlag}
[622]	530	}
	531
	532	#D-#==================================================
	533	#D-function IGCM_sys_TestDirArchive
	534	#D-* Purpose: Test Directory that must exists on Archive
	535	#D-* Examples:
	536	#D-
	537	function IGCM_sys_TestDirArchive {
[623]	538	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	539	if ( $DEBUG_sys ) ; then
	540	echo "IGCM_sys_TestDirArchive :" $@
	541	fi
	542	typeset ExistFlag
	543	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	544	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	545	return ${ExistFlag}
[622]	546	}
	547
	548	#D-#==================================================
[623]	549	#D-function IGCM_sys_IsFileArchived
	550	#D-* Purpose: Test file that must NOT EXISTS on Archive
	551	#D-* Examples:
	552	#D-
	553	function IGCM_sys_IsFileArchived {
	554	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	555	if ( $DEBUG_sys ) ; then
	556	echo "IGCM_sys_IsFileArchived :" $@
	557	fi
	558	typeset IsArchivedFlag
[818]	559	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[623]	560	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	561
	562	return ${IsArchivedFlag}
	563	}
	564
	565	#D-#==================================================
[622]	566	#D-function IGCM_sys_TestFileArchive
	567	#D-* Purpose: Test file that must NOT EXISTS on Archive
	568	#D-* Examples:
	569	#D-
	570	function IGCM_sys_TestFileArchive {
[623]	571	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	572	typeset ExistFlag
	573	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	574	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]	575
[623]	576	return ${ExistFlag}
[622]	577	}
	578
	579	#D-#==================================================
[623]	580	#D-function IGCM_sys_TestFileBuffer
	581	#D-* Purpose: Test file that must NOT EXISTS on Buffer
	582	#D-* Examples:
	583	#D-
	584	function IGCM_sys_TestFileBuffer {
	585	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
	586	typeset ExistFlag
	587	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	588	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
	589
	590	return ${ExistFlag}
	591	}
	592
	593	#D-#==================================================
[622]	594	#D-function IGCM_sys_CountFileArchive
	595	#D-* Purpose: Count files on Archive filesystem
	596	#D-* Examples:
	597	#D-
	598	function IGCM_sys_CountFileArchive {
[623]	599	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	600	ls ${@} 2>/dev/null \| wc -l
	601	if [ $? -gt 0 ] ; then
	602	echo "IGCM_sys_CountFileArchive : erreur."
	603	fi
	604	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]	605	}
	606
	607	#D-#==================================================
[623]	608	#D-function IGCM_sys_CountFileBuffer
	609	#D-* Purpose: Count files on Scratch filesystem
	610	#D-* Examples:
	611	#D-
	612	function IGCM_sys_CountFileBuffer {
	613	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
	614	ls ${@} 2>/dev/null \| wc -l
	615	if [ $? -gt 0 ] ; then
	616	echo "IGCM_sys_CountFileBuffer : erreur."
	617	fi
	618	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
	619	}
	620
	621	#D-#==================================================
[622]	622	#D-function IGCM_sys_Tree
	623	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	624	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	625	#D-
	626	function IGCM_sys_Tree {
[623]	627	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	628	if ( $DEBUG_sys ) ; then
	629	echo "IGCM_sys_Tree :" $@
	630	fi
[622]	631
[623]	632	\ls -lR ${@}
[622]	633
[623]	634	IGCM_debug_PopStack "IGCM_sys_Tree"
[622]	635	}
	636
	637	#D-#==================================================
	638	#D-function IGCM_sys_Tar
[623]	639	#D-* Purpose: master tar command
[622]	640	#D-* Examples:
	641	#D-
	642	function IGCM_sys_Tar {
[623]	643	IGCM_debug_PushStack "IGCM_sys_Tar" $@
	644	if ( $DEBUG_sys ) ; then
	645	echo "IGCM_sys_Tar :" $@
	646	fi
	647	\tar cf $@
	648	if [ $? -gt 0 ] ; then
	649	echo "IGCM_sys_Tar : erreur."
	650	IGCM_debug_Exit "IGCM_sys_Tar"
	651	fi
	652	IGCM_debug_PopStack "IGCM_sys_Tar"
[622]	653	}
	654
	655	#D-#==================================================
	656	#D-function IGCM_sys_UnTar
	657	#D-* Purpose: master un-tar command
	658	#D-* Examples:
	659	#D-
	660	function IGCM_sys_UnTar {
[623]	661	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
	662	if ( $DEBUG_sys ) ; then
	663	echo "IGCM_sys_UnTar :" $@
	664	fi
	665	\tar xvf $1
	666	if [ $? -gt 0 ] ; then
	667	echo "IGCM_sys_UnTar : erreur."
	668	IGCM_debug_Exit "IGCM_sys_UnTar"
	669	fi
	670	IGCM_debug_PopStack "IGCM_sys_UnTar"
[622]	671	}
	672
	673	#D-#==================================================
	674	#D-function IGCM_sys_Qsub
	675	#D-* Purpose: Qsub new job
	676	#D-* Examples:
	677	#D-
	678	function IGCM_sys_Qsub {
[623]	679	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	680	if ( $DEBUG_sys ) ; then
	681	echo "IGCM_sys_Qsub :" $@
	682	fi
[811]	683	typeset options status
[923]	684	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]	685
[625]	686	/usr/bin/ccc_msub ${options} < $1
[811]	687	status=$?
	688	if [ ${status} -gt 0 ] ; then
	689	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]	690	IGCM_debug_Exit "IGCM_sys_Qsub"
	691	fi
	692	IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]	693	}
	694
	695	#D-#==================================================
	696	#D-function IGCM_sys_QsubPost
	697	#D-* Purpose: Qsub new job on scalaire
	698	#D-* Examples:
	699	#D-
	700	function IGCM_sys_QsubPost {
[623]	701	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	702	if ( $DEBUG_sys ) ; then
	703	echo "IGCM_sys_QsubPost :" $@
	704	fi
[923]	705	typeset options status
[880]	706	MY_PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=${DEFAULT_PROJECT}} \| cut -d@ -f1 )
[923]	707	options="-Q normal -A ${MY_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
	708
	709	/usr/bin/ccc_msub ${options} < ${libIGCM_POST}/$1.job
[811]	710	status=$?
	711	if [ ${status} -gt 0 ] ; then
[923]	712	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]	713	IGCM_debug_Exit "IGCM_sys_QsubPost"
	714	fi
	715	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]	716	}
	717
	718	#D-*************************
[623]	719	#D- File transfer functions
[622]	720	#D-*************************
	721	#D-
	722
	723	#D-#==================================================
	724	#D-function IGCM_sys_Rsync_out
	725	#D-* Purpose: treat return val of rsync
	726	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
	727	#D- Error values and explanations can depend on your system version.
	728	function IGCM_sys_Rsync_out {
[719]	729	status=$1
	730	if [ ! $status ] ; then
[623]	731	echo "rsync error !"
	732	fi
[622]	733
[623]	734	if [ $MYLANG = "fr" ]; then
[719]	735	case $status in
[623]	736	0) return ;;
	737	1) echo "Erreur de rsync ; RERR_SYNTAX : "
	738	echo "Erreur de syntaxe ou d'utilisation."
	739	return;;
	740	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
	741	echo "Incompatibilité de protocole."
	742	return;;
	743	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
	744	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
	745	echo "répertoires"
	746	return;;
	747	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
	748	echo "Action demandée non supportée : une tentative de manipulation de"
	749	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
	750	echo "été faite ; ou une option qui est supportée par le client mais"
	751	echo "pas par le serveur a été spécifiée."
	752	return;;
	753	10) echo "Erreur de rsync ; RERR_SOCKETIO"
	754	echo "Erreur dans le socket d'entrée sortie"
	755	return;;
	756	11) echo "Erreur de rsync ; RERR_FILEIO"
	757	echo "Erreur d'entrée sortie fichier"
	758	return;;
	759	12) echo "Erreur de rsync ; RERR_STREAMIO"
	760	echo "Erreur dans flux de donnée du protocole rsync"
	761	return;;
	762	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
	763	echo "Erreur avec les diagnostics du programme"
	764	return;;
	765	14) echo "Erreur de rsync ; RERR_IPC"
	766	echo "Erreur dans le code IPC"
	767	return;;
	768	20) echo "Erreur de rsync ; RERR_SIGNAL"
	769	echo "SIGUSR1 ou SIGINT reçu"
	770	return;;
	771	21) echo "Erreur de rsync ; RERR_WAITCHILD"
	772	echo "Une erreur retournée par waitpid()"
	773	return;;
	774	22) echo "Erreur de rsync ; RERR_MALLOC"
	775	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
	776	return;;
	777	23) echo ""
	778	echo "Erreur fichier inexistant"
	779	return;;
	780	30) echo "Erreur de rsync ; RERR_TIMEOUT"
	781	echo "Temps d'attente écoulé dans l'envoi/réception de données"
	782	return;;
[719]	783	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
[623]	784	return;;
	785	esac
	786	elif [ $MYLANG = "en" ] ; then
[719]	787	case $status in
[623]	788	0) return;;
	789	1) echo "rsync error : Syntax or usage error "
	790	return;;
	791	2) echo "rsync error : Protocol incompatibility "
	792	return;;
	793	3) echo "rsync error : Errors selecting input/output files, dirs"
	794	return;;
	795	4) echo "rsync error : Requested action not supported: an attempt"
	796	echo "was made to manipulate 64-bit files on a platform that cannot support"
	797	echo "them; or an option was specified that is supported by the client and"
	798	echo "not by the server."
	799	return;;
	800	5) echo "rsync error : Error starting client-server protocol"
	801	return;;
	802	10) echo "rsync error : Error in socket I/O "
	803	return;;
	804	11) echo "rsync error : Error in file I/O "
	805	return;;
	806	12) echo "rsync error : Error in rsync protocol data stream "
	807	return;;
	808	13) echo "rsync error : Errors with program diagnostics "
	809	return;;
	810	14) echo "rsync error : Error in IPC code "
	811	return;;
	812	20) echo "rsync error : Received SIGUSR1 or SIGINT "
	813	return;;
	814	21) echo "rsync error : Some error returned by waitpid() "
	815	return;;
	816	22) echo "rsync error : Error allocating core memory buffers "
	817	return;;
	818	23) echo "rsync error : Partial transfer due to error"
	819	return;;
	820	24) echo "rsync error : Partial transfer due to vanished source files"
	821	return;;
	822	30) echo "rsync error : Timeout in data send/receive "
	823	return;;
[719]	824	*) echo "rsync error : return code of rsync unknown :" $status
[623]	825	return;;
	826	esac
	827	else
	828	echo "unknown language $MYLANG."
	829	return
	830	fi
[622]	831	}
	832
	833	#D-#==================================================
	834	#D-function IGCM_sys_Miror_libIGCM
[811]	835	#D-* Purpose: Mirror libIGCM PATH and lib to frontend
[622]	836	#D-* Examples:
	837	#D-
	838	function IGCM_sys_Mirror_libIGCM {
[623]	839	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
	840	if ( $DEBUG_sys ) ; then
	841	echo "IGCM_sys_Mirror_libIGCM"
	842	fi
[622]	843
[773]	844	typeset status
[622]	845
[623]	846	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
[622]	847
[623]	848	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
	849	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
[715]	850	status=$?
[622]	851
[715]	852	if [ ${status} -gt 0 ] ; then
[811]	853	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
[623]	854	cat out_rsync
	855	fi
	856	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
[622]	857	}
	858
	859	#D-#==================================================
	860	#D-function IGCM_sys_Cp
	861	#D-* Purpose: generic cp
	862	#D-* Examples:
	863	#D-
	864	function IGCM_sys_Cp {
[623]	865	IGCM_debug_PushStack "IGCM_sys_Cp" $@
	866	if ( $DEBUG_sys ) ; then
	867	echo "IGCM_sys_Cp :" $@
	868	fi
[622]	869
[715]	870	typeset status
[622]	871
[623]	872	echo cp $@ > out_rsync 2>&1
	873	\cp $@ >> out_rsync 2>&1
[715]	874	status=$?
[622]	875
[715]	876	if [ ${status} -gt 0 ] ; then
	877	echo "IGCM_sys_Cp : error code ${status}"
[623]	878	cat out_rsync
	879	IGCM_debug_Exit "IGCM_sys_Cp"
	880	else
	881	\rm out_rsync
	882	fi
	883	IGCM_debug_PopStack "IGCM_sys_Cp"
[622]	884	}
	885
	886	#D-#==================================================
	887	#D-function IGCM_sys_Rm
	888	#D-* Purpose: generic rm
	889	#D-* Examples:
	890	#D-
	891	function IGCM_sys_Rm {
[623]	892	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
	893	if ( $DEBUG_sys ) ; then
	894	echo "IGCM_sys_Rm :" $@
	895	fi
[622]	896
[715]	897	typeset status
[622]	898
[623]	899	echo rm $@ > out_rsync 2>&1
	900	\rm $@ >> out_rsync 2>&1
[715]	901	status=$?
[622]	902
[715]	903	if [ ${status} -gt 0 ] ; then
	904	echo "IGCM_sys_Rm : error code ${status}"
[623]	905	cat out_rsync
	906	IGCM_debug_Exit "IGCM_sys_Rm"
	907	else
	908	\rm out_rsync
	909	fi
	910	IGCM_debug_PopStack "IGCM_sys_Rm"
[622]	911	}
	912
	913	#D-#==================================================
	914	#D-function IGCM_sys_RmRunDir
	915	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	916	#D- scheduler will do the job)
	917	#D-* Examples:
	918	#D-
	919	function IGCM_sys_RmRunDir {
[623]	920	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
	921	if ( $DEBUG_sys ) ; then
	922	echo "IGCM_sys_RmRunDir :" $@
	923	fi
[622]	924
[715]	925	typeset status
[622]	926
[623]	927	echo rm $@ > out_rsync 2>&1
	928	\rm $@ >> out_rsync 2>&1
[715]	929	status=$?
[622]	930
[715]	931	if [ ${status} -gt 0 ] ; then
[811]	932	echo "IGCM_sys_RmRunDir : rm error code is ${status}."
[623]	933	cat out_rsync
	934	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	935	else
	936	\rm out_rsync
	937	fi
	938	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]	939	}
	940
	941	#D-#==================================================
	942	#D-function IGCM_sys_Mv
	943	#D-* Purpose: generic move
	944	#D-* Examples:
	945	#D-
	946	function IGCM_sys_Mv {
[623]	947	IGCM_debug_PushStack "IGCM_sys_Mv" $@
	948	if ( $DEBUG_sys ) ; then
	949	echo "IGCM_sys_Mv :" $@
	950	fi
[622]	951
[623]	952	if [ $DRYRUN = 0 ]; then
[622]	953
[715]	954	typeset status
[623]	955
	956	echo mv $@ > out_rsync 2>&1
	957	\mv $@ >> out_rsync 2>&1
[715]	958	status=$?
[623]	959
[715]	960	if [ ${status} -gt 0 ] ; then
	961	echo "IGCM_sys_Mv : error code ${status}"
[623]	962	cat out_rsync
	963	IGCM_debug_Exit "IGCM_sys_Mv"
[622]	964	else
[623]	965	\rm out_rsync
[622]	966	fi
[623]	967	fi
[622]	968
[623]	969	IGCM_debug_PopStack "IGCM_sys_Mv"
[622]	970	}
	971
	972	#D-#==================================================
	973	#D-function IGCM_sys_Put_Dir
	974	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	975	#D-* Examples:
	976	#D-
	977	function IGCM_sys_Put_Dir {
[623]	978	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	979	if ( $DEBUG_sys ) ; then
	980	echo "IGCM_sys_Put_Dir :" $@
	981	fi
	982	if [ $DRYRUN = 0 ]; then
	983	if [ ! -d ${1} ] ; then
	984	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	985	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	986	return
[622]	987	fi
	988
[715]	989	typeset status
[622]	990
[623]	991	# Only if we use rsync
	992	#IGCM_sys_TestDirArchive $( dirname $2 )
	993	#
	994	#USUAL WAY
	995	\cp -r $1 $2 > out_rsync 2>&1
[715]	996	status=$?
[622]	997
[715]	998	if [ ${status} -gt 0 ] ; then
[811]	999	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[623]	1000	cat out_rsync
	1001	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]	1002	else
[623]	1003	\rm out_rsync
[622]	1004	fi
[623]	1005	fi
	1006	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]	1007	}
	1008
	1009	#D-#==================================================
	1010	#D-function IGCM_sys_Get_Dir
[623]	1011	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]	1012	#D-* Examples:
	1013	#D-
	1014	function IGCM_sys_Get_Dir {
[623]	1015	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	1016	if ( $DEBUG_sys ) ; then
	1017	echo "IGCM_sys_Get_Dir :" $@
	1018	fi
	1019	if [ $DRYRUN = 0 ]; then
	1020	# if [ ! -d ${1} ] ; then
	1021	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
	1022	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
	1023	# return
	1024	# fi
[622]	1025
[708]	1026	typeset NB_ESSAI DELAI status i
	1027	# number of tentative
	1028	NB_ESSAI=3
	1029	# time delay between tentative
	1030	DELAI=2
[622]	1031
[676]	1032	# Only if we use rsync
	1033	#IGCM_sys_TestDirArchive $( dirname $2 )
	1034	#
[708]	1035	# USUAL WAY
	1036	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]	1037	ccc_hsm get -r $1
[622]	1038
[709]	1039	i=0
[708]	1040	while [ $i -lt $NB_ESSAI ] ; do
	1041	\cp -ur $1 $2 >> out_rsync 2>&1
	1042	status=$?
[811]	1043	if [ ${status} -gt 0 ] ; then
[713]	1044	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	1045	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	1046	sleep $DELAI
	1047	else
	1048	break
	1049	fi
	1050	(( i = i + 1 ))
	1051	done
	1052
	1053	if [ ${status} -gt 0 ] ; then
[623]	1054	echo "IGCM_sys_Get_Dir : error."
	1055	cat out_rsync
	1056	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]	1057	else
[623]	1058	\rm out_rsync
[622]	1059	fi
[623]	1060	fi
	1061	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]	1062	}
	1063
	1064	#D-#==================================================
	1065	#D-function IGCM_sys_Get_Master
	1066	#D-* Purpose: Copy a complete directory from MASTER filesystem
	1067	#D-* Examples:
	1068	#D-
	1069	function IGCM_sys_Get_Master {
[623]	1070	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
	1071	if ( $DEBUG_sys ) ; then
	1072	echo "IGCM_sys_Get_Master :" $@
	1073	fi
	1074	if [ $DRYRUN = 0 ]; then
[640]	1075	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
[623]	1076	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
	1077	IGCM_debug_PopStack "IGCM_sys_Get_Master"
	1078	return
[622]	1079	fi
	1080
[708]	1081	typeset NB_ESSAI DELAI status i
	1082	# number of tentative
	1083	NB_ESSAI=3
	1084	# time delay between tentative
	1085	DELAI=2
[622]	1086
[709]	1087	i=0
[708]	1088	while [ $i -lt $NB_ESSAI ] ; do
	1089	\cp -urL $1 $2 > out_rsync 2>&1
	1090	status=$?
	1091	if [ ${status} -gt 0 ]; then
[713]	1092	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	1093	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
	1094	sleep $DELAI
	1095	else
	1096	break
	1097	fi
	1098	(( i = i + 1 ))
	1099	done
[622]	1100
[708]	1101	if [ ${status} -gt 0 ] ; then
[623]	1102	echo "IGCM_sys_Get_Master : error."
	1103	cat out_rsync
	1104	IGCM_debug_Exit "IGCM_sys_Get_Master"
[622]	1105	else
[623]	1106	\rm out_rsync
[622]	1107	fi
[623]	1108	fi
	1109	IGCM_debug_PopStack "IGCM_sys_Get_Master"
[622]	1110	}
	1111
[689]	1112	#====================================================
	1113	#- Call IGCM_sys_Mirror_libIGCM now !
	1114	if ( $MirrorlibIGCM ) ; then
	1115	IGCM_sys_Mirror_libIGCM
	1116	fi
	1117
[622]	1118	#D-#==================================================
	1119	#D-function IGCM_sys_Put_Rest
[623]	1120	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]	1121	#D- File and target directory must exist.
	1122	#D-* Examples:
	1123	#D-
	1124	function IGCM_sys_Put_Rest {
[623]	1125	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	1126	if ( $DEBUG_sys ) ; then
	1127	echo "IGCM_sys_Put_Rest :" $@
	1128	fi
	1129	if [ $DRYRUN = 0 ]; then
	1130	if [ ! -f ${1} ] ; then
	1131	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	1132	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	1133	fi
	1134
[715]	1135	typeset status
[623]	1136	#
	1137	# USUAL WAY
	1138	\cp $1 $2 > out_rsync 2>&1
[715]	1139	status=$?
[622]	1140
[623]	1141	# #RSYNC WITH NETWORK SSH CALL
	1142	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1143	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[622]	1144
[623]	1145	# #RSYNC WITH NFS USE
	1146	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1147	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
[622]	1148
[715]	1149	# status=$?
	1150	# IGCM_sys_Rsync_out $status
[622]	1151
[623]	1152	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
[715]	1153	# (( status=status+$? ))
[623]	1154
[715]	1155	if [ ${status} -gt 0 ] ; then
	1156	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]	1157	[ -f ${1} ] && ls -l ${1}
[759]	1158	[ -f ${2} ] && ls -l ${2}
	1159	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[623]	1160	cat out_rsync
	1161	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	1162	else
[759]	1163
	1164	if [ X${JobType} = XRUN ] ; then
	1165	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	1166	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	1167	fi
	1168
[623]	1169	\rm out_rsync
[622]	1170	fi
[623]	1171	fi
	1172	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]	1173	}
	1174
	1175	#D-#==================================================
[623]	1176	#D-function IGCM_sys_PutBuffer_Rest
	1177	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
	1178	#D- File and target directory must exist.
	1179	#D-* Examples:
	1180	#D-
	1181	function IGCM_sys_PutBuffer_Rest {
	1182	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
	1183	if ( $DEBUG_sys ) ; then
	1184	echo "IGCM_sys_PutBuffer_Rest :" $@
	1185	fi
	1186	if [ $DRYRUN = 0 ]; then
	1187	if [ ! -f ${1} ] ; then
	1188	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
	1189	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1190	fi
	1191
[715]	1192	typeset status
[623]	1193	#
	1194	# USUAL WAY
	1195	\cp $1 $2 > out_rsync 2>&1
[715]	1196	status=$?
[623]	1197
[715]	1198	if [ ${status} -gt 0 ] ; then
	1199	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
[759]	1200	[ -f ${2} ] && ls -l ${2}
	1201	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[623]	1202	cat out_rsync
	1203	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1204	else
[759]	1205
	1206	if [ X${JobType} = XRUN ] ; then
	1207	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	1208	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	1209	fi
	1210
[623]	1211	\rm out_rsync
	1212	fi
	1213	fi
	1214	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
	1215	}
	1216
	1217	#D-#==================================================
[801]	1218	#D-function IGCM_sys_PrepareTaredRestart
	1219	#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
	1220	#D-* Examples:
	1221	#D-
	1222	function IGCM_sys_PrepareTaredRestart {
	1223	IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
[802]	1224	if [ $DRYRUN = 0 ]; then
[918]	1225	[ ! -f $( basename $1 ) ] && IGCM_sys_Get $1 .
[802]	1226	fi
[801]	1227	IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
	1228	}
	1229
	1230	#D-#==================================================
[622]	1231	#D-function IGCM_sys_Put_Out
[623]	1232	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]	1233	#D-* Examples:
	1234	#D-
	1235	function IGCM_sys_Put_Out {
[623]	1236	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	1237	if ( $DEBUG_sys ) ; then
	1238	echo "IGCM_sys_Put_Out :" $@
	1239	fi
[667]	1240
[711]	1241	typeset NB_ESSAI DELAI status i exist skip
[667]	1242
	1243	# number of tentative
[668]	1244	NB_ESSAI=3
[667]	1245	# time delay between tentative
[668]	1246	DELAI=2
[667]	1247
[623]	1248	if [ $DRYRUN = 0 ]; then
	1249	if [ ! -f ${1} ] ; then
	1250	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	1251	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1252	return 1
[622]	1253	fi
[623]	1254	#
	1255	IGCM_sys_MkdirArchive $( dirname $2 )
	1256	#
[622]	1257
[623]	1258	#=====================================================
	1259	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
	1260	#=====================================================
[622]	1261
[623]	1262	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
	1263	#if [ $? -eq 0 ] ; then
	1264	# typeset WORKPATH FILEPATH
	1265	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
	1266	# IGCM_sys_MkdirWork ${WORKPATH}
	1267	# FILEPATH=${WORKPATH}/$( basename $2 )
	1268	# #
	1269	# IGCM_sys_Cp ${1} ${FILEPATH}
	1270	#fi
[622]	1271
[711]	1272	exist=false
	1273	skip=false
	1274	if [ -f $2 ] ; then
	1275	IGCM_debug_Print 1 "$2 already exist"
	1276	#dmget $2
	1277	ccc_hsm get $2
	1278	exist=true
	1279	if [ "X$( diff $1 $2 )" = X ] ; then
	1280	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	1281	skip=true
[667]	1282	else
[711]	1283	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	1284	skip=false
[667]	1285	fi
[711]	1286	fi
	1287	#
	1288	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	1289	IGCM_sys_Chmod u+w $2
	1290	fi
[622]	1291
[711]	1292	if [ X${skip} = Xfalse ] ; then
[719]	1293	i=0
[711]	1294	while [ $i -lt $NB_ESSAI ] ; do
	1295	# USUAL WAY
	1296	\cp $1 $2 > out_rsync 2>&1
	1297	status=$?
	1298	if [ ${status} -gt 0 ]; then
[713]	1299	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	1300	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]	1301	[ -f ${1} ] && ls -l ${1}
[759]	1302	[ -f ${2} ] && ls -l ${2}
	1303	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	1304	sleep $DELAI
	1305	else
	1306	break
	1307	fi
	1308	(( i = i + 1 ))
	1309	done
	1310	fi
	1311
[623]	1312	# #RSYNC WITH NETWORK SSH CALL
	1313	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1314	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[622]	1315
[623]	1316	# #RSYNC WITH NFS USE
	1317	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1318	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
[622]	1319
[711]	1320	# status=$?
	1321	# IGCM_sys_Rsync_out $status
[622]	1322
[623]	1323	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
[711]	1324	# (( status=status+$? ))
[622]	1325
[711]	1326	if [ ${status} -gt 0 ] ; then
[623]	1327	echo "IGCM_sys_Put_Out : error."
[765]	1328	[ -f ${1} ] && ls -l ${1}
[759]	1329	[ -f ${2} ] && ls -l ${2}
	1330	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[623]	1331	cat out_rsync
	1332	IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]	1333	else
[759]	1334
	1335	if [ X${JobType} = XRUN ] ; then
	1336	if [ X${3} = X ] ; then
	1337	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	1338	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	1339	fi
	1340	fi
	1341
[623]	1342	\rm out_rsync
[622]	1343	fi
[623]	1344	fi
	1345	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1346	return 0
[622]	1347	}
	1348
	1349	#D-#==================================================
[623]	1350	#D-function IGCM_sys_PutBuffer_Out
	1351	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
	1352	#D-* Examples:
	1353	#D-
	1354	function IGCM_sys_PutBuffer_Out {
	1355	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
	1356	if ( $DEBUG_sys ) ; then
	1357	echo "IGCM_sys_PutBuffer_Out :" $@
	1358	fi
[711]	1359
[773]	1360	typeset NB_ESSAI DELAI status i exist skip
[711]	1361
	1362	# number of tentative
	1363	NB_ESSAI=3
	1364	# time delay between tentative
	1365	DELAI=2
	1366
[623]	1367	if [ $DRYRUN = 0 ]; then
	1368	if [ ! -f ${1} ] ; then
	1369	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
	1370	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1371	return 1
	1372	fi
	1373	#
	1374	IGCM_sys_Mkdir $( dirname $2 )
	1375	#
	1376
[711]	1377	exist=false
	1378	skip=false
	1379	if [ -f $2 ] ; then
	1380	IGCM_debug_Print 1 "$2 already exist"
	1381	exist=true
	1382	if [ "X$( diff $1 $2 )" = X ] ; then
	1383	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
[811]	1384	status=0
[711]	1385	skip=true
	1386	else
	1387	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	1388	skip=false
	1389	fi
	1390	fi
[623]	1391	#
[711]	1392	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	1393	IGCM_sys_Chmod u+w $2
	1394	fi
[623]	1395
[711]	1396	if [ X${skip} = Xfalse ] ; then
	1397	i=0
	1398	while [ $i -lt $NB_ESSAI ] ; do
	1399	# USUAL WAY
	1400	\cp $1 $2 > out_rsync 2>&1
	1401	status=$?
	1402	if [ ${status} -gt 0 ]; then
[713]	1403	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	1404	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
[759]	1405	[ -f ${2} ] && ls -l ${2}
	1406	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	1407	sleep $DELAI
	1408	else
	1409	break
	1410	fi
	1411	(( i = i + 1 ))
	1412	done
	1413	fi
	1414
	1415	if [ ${status} -gt 0 ] ; then
[623]	1416	echo "IGCM_sys_PutBuffer_Out : error."
[759]	1417	[ -f ${2} ] && ls -l ${2}
	1418	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[623]	1419	cat out_rsync
	1420	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
	1421	else
[759]	1422
	1423	if [ X${JobType} = XRUN ] ; then
	1424	if [ X${3} = X ] ; then
	1425	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	1426	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	1427	fi
	1428	fi
	1429
[623]	1430	\rm out_rsync
	1431	fi
	1432	fi
	1433	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1434	return 0
	1435	}
	1436
	1437	#D-#==================================================
[622]	1438	#D-function IGCM_sys_Get
	1439	#D-* Purpose: Get a file from ${ARCHIVE}
	1440	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	1441	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	1442	function IGCM_sys_Get {
[623]	1443	IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]	1444
[773]	1445	typeset DEST dm_liste target file_work
[667]	1446	typeset NB_ESSAI DELAI status i
[622]	1447
[623]	1448	if ( $DEBUG_sys ) ; then
	1449	echo "IGCM_sys_Get :" $@
	1450	fi
[667]	1451
	1452	# number of tentative
[668]	1453	NB_ESSAI=3
[667]	1454	# time delay between tentative
[668]	1455	DELAI=2
[667]	1456
[623]	1457	if [ $DRYRUN -le 2 ]; then
	1458	if [ X${1} = X'/l' ] ; then
	1459	eval set +A dm_liste \${${2}}
	1460	else
	1461	eval set +A dm_liste ${1}
[622]	1462	fi
[623]	1463	eval DEST=\${${#}}
[622]	1464
[623]	1465	#=====================================================
	1466	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
	1467	#=====================================================
[622]	1468
[623]	1469	# Is it an R_OUT file (not R_IN) ?
	1470	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
	1471	#if [ $? -eq 0 ] ; then
[785]	1472	# # Yes ? then we try to get it in SCRATCHDIR
	1473	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
	1474	# if [ -f ${file_work[0]} ] ; then
	1475	# IGCM_sys_Cp ${file_work[*]} ${DEST}
	1476	# IGCM_debug_PopStack "IGCM_sys_Get"
	1477	# return
	1478	# fi
[623]	1479	#fi
[622]	1480
[623]	1481	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
[715]	1482	status=$?
	1483	if [ ${status} -gt 0 ] ; then
	1484	echo "WARNING IGCM_sys_Get : error code ${status}"
[623]	1485	cat out_rsync
	1486	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	1487	fi
[622]	1488
[715]	1489	#if [ ${status} -gt 0 ] ; then
[785]	1490	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
	1491	# cat out_rsync
	1492	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	1493	# sleep 30
	1494	# echo "We try another time"
	1495	## dmget ${dm_liste[*]} > out_rsync 2>&1
	1496	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
	1497	# status=$?
	1498	# if [ ${status} -gt 0 ] ; then
	1499	# echo "ERROR IGCM_sys_Get : again demigration error :"
	1500	# cat out_rsync
	1501	# IGCM_debug_Exit "IGCM_sys_Get"
[623]	1502	# fi
[785]	1503	# else
	1504	# echo "ERROR IGCM_sys_Get : demigration error :"
	1505	# cat out_rsync
	1506	# IGCM_debug_Exit "IGCM_sys_Get"
	1507	# fi
[623]	1508	#fi
[622]	1509
[719]	1510	# #RSYNC WITH NETWORK SSH CALL
	1511	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
	1512	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
	1513
	1514	# #RSYNC WITH NFS USE
	1515	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
	1516	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
	1517
	1518	# status=$?
	1519	# IGCM_sys_Rsync_out $status
	1520
	1521	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1522	# (( status=status+$? ))
	1523
[623]	1524	#USUAL WAY
	1525	if [ X${1} = X'/l' ] ; then
	1526	for target in ${dm_liste[*]} ; do
	1527	local_file=$( basename ${target} )
[668]	1528	# test if the target file is present before the loop
	1529	IGCM_sys_TestFileArchive ${target}
[715]	1530	status=$?
	1531	if [ ${status} -gt 0 ] ; then
[701]	1532	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]	1533	IGCM_debug_Exit "IGCM_sys_Get"
	1534	else
[719]	1535	i=0
[668]	1536	while [ $i -lt $NB_ESSAI ] ; do
	1537	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1538	status=$?
	1539	if [ ${status} -gt 0 ]; then
[713]	1540	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]	1541	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	1542	sleep $DELAI
[668]	1543	else
[676]	1544	break
[668]	1545	fi
	1546	(( i = i + 1 ))
	1547	done
[719]	1548	if [ ${status} -gt 0 ] ; then
	1549	echo "IGCM_sys_Get : error"
	1550	cat out_rsync
	1551	\rm out_rsync
	1552	IGCM_debug_Exit "IGCM_sys_Get"
	1553	else
	1554	\rm out_rsync
	1555	fi
[668]	1556	fi
[623]	1557	done
	1558	else
[719]	1559	i=0
	1560	while [ $i -lt $NB_ESSAI ] ; do
	1561	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
	1562	status=$?
	1563	if [ ${status} -gt 0 ]; then
	1564	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1565	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	1566	sleep $DELAI
	1567	else
	1568	break
	1569	fi
	1570	(( i = i + 1 ))
	1571	done
	1572	if [ ${status} -gt 0 ] ; then
	1573	echo "IGCM_sys_Get : error"
	1574	cat out_rsync
	1575	\rm out_rsync
	1576	IGCM_debug_Exit "IGCM_sys_Get"
	1577	else
	1578	\rm out_rsync
	1579	fi
[623]	1580	fi
	1581	fi
	1582	IGCM_debug_PopStack "IGCM_sys_Get"
	1583	}
[622]	1584
[623]	1585	#D-#==================================================
	1586	#D-function IGCM_sys_GetBuffer
	1587	#D-* Purpose: Get a file from ${SCRATCHDIR}
	1588	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
	1589	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
	1590	function IGCM_sys_GetBuffer {
	1591	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
	1592
[773]	1593	typeset DEST buf_liste target file_work
[719]	1594	typeset NB_ESSAI DELAI status i
[623]	1595
	1596	if ( $DEBUG_sys ) ; then
	1597	echo "IGCM_sys_GetBuffer :" $@
	1598	fi
[719]	1599
	1600	# number of tentative
	1601	NB_ESSAI=3
	1602	# time delay between tentative
	1603	DELAI=2
	1604
[623]	1605	if [ $DRYRUN -le 2 ]; then
	1606	if [ X${1} = X'/l' ] ; then
	1607	eval set +A buf_liste \${${2}}
[622]	1608	else
[623]	1609	eval set +A buf_liste ${1}
[622]	1610	fi
[623]	1611	eval DEST=\${${#}}
	1612
	1613	#USUAL WAY
	1614	if [ X${1} = X'/l' ] ; then
	1615	for target in ${buf_liste[*]} ; do
	1616	local_file=$( basename ${target} )
[719]	1617	i=0
	1618	while [ $i -lt $NB_ESSAI ] ; do
	1619	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1620	status=$?
	1621	if [ ${status} -gt 0 ]; then
	1622	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1623	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
	1624	sleep $DELAI
	1625	else
	1626	break
	1627	fi
	1628	(( i = i + 1 ))
	1629	done
	1630	if [ ${status} -gt 0 ] ; then
	1631	echo "IGCM_sys_Get : error"
	1632	cat out_rsync
	1633	\rm out_rsync
	1634	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1635	else
	1636	\rm out_rsync
	1637	fi
[623]	1638	done
	1639	else
[719]	1640	i=0
	1641	while [ $i -lt $NB_ESSAI ] ; do
	1642	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
	1643	status=$?
	1644	if [ ${status} -gt 0 ]; then
	1645	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1646	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
	1647	sleep $DELAI
	1648	else
	1649	break
	1650	fi
	1651	(( i = i + 1 ))
	1652	done
	1653	if [ ${status} -gt 0 ] ; then
	1654	echo "IGCM_sys_Get : error"
	1655	cat out_rsync
	1656	\rm out_rsync
	1657	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1658	else
	1659	\rm out_rsync
	1660	fi
[623]	1661	fi
	1662	fi
	1663	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
[622]	1664	}
	1665
	1666	#D-#==================================================
	1667	#D-function IGCM_sys_GetDate_FichWork
	1668	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
	1669	#D-* Examples:
	1670	#D-
	1671	function IGCM_sys_GetDate_FichWork {
[623]	1672	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
	1673	if ( $DEBUG_sys ) ; then
	1674	echo "IGCM_sys_GetDate_FichWork :" $@
	1675	fi
	1676	typeset dateF
	1677	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
	1678	eval ${2}=${dateF[5]}
	1679
[773]	1680	# donne la date filesys d'un fichier sur la machine work
[623]	1681	IGCM_debug_PopStack "IGCM_sys_FichWork"
[622]	1682	}
	1683
	1684	#D-#==================================================
	1685	#D-function IGCM_sys_GetDate_FichArchive
[623]	1686	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
[622]	1687	#D-* Examples:
	1688	#D-
	1689	function IGCM_sys_GetDate_FichArchive {
[623]	1690	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
	1691	if ( $DEBUG_sys ) ; then
	1692	echo "IGCM_sys_GetDate_FichArchive :" $@
	1693	fi
	1694	typeset dateF
	1695	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
	1696	eval ${2}=${dateF[5]}
	1697
	1698	IGCM_debug_PopStack "IGCM_sys_FichArchive"
	1699	}
	1700
	1701	#D-#==================================================
[835]	1702	#D-function IGCM_sys_GetDate_Monitoring
	1703	#D-* Purpose: get the last year for which the monitoring has been computed
	1704	#D-* Examples:
	1705	#D-
	1706	function IGCM_sys_GetDate_Monitoring {
	1707	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
	1708	if ( $DEBUG_sys ) ; then
	1709	echo "IGCM_sys_GetDate_Monitoring :" $@
	1710	fi
	1711
	1712	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
	1713
	1714	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
	1715	}
	1716
	1717	#D-#==================================================
[623]	1718	#D-function IGCM_sys_Dods_Rm
	1719	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	1720	#D-* Examples:
	1721	#D-
	1722	function IGCM_sys_Dods_Rm {
	1723	if ( $DEBUG_sys ) ; then
	1724	echo "IGCM_sys_Dods_Rm :" $@
	1725	fi
[715]	1726	typeset status
[623]	1727	if [ $DRYRUN = 0 ]; then
	1728
	1729	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	1730	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	1731	# echo "Nothing has been done."
	1732	# return
	1733	# fi
	1734
[649]	1735	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
[715]	1736	status=$?
[623]	1737
[785]	1738	# if [ ${status} -gt 0 ] ; then
	1739	# echo "IGCM_sys_Dods_Rm : error."
	1740	# cat out_dods_rm
	1741	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	1742	# else
	1743	# rm out_dods_rm
	1744	# fi
[623]	1745
	1746	fi
[715]	1747	return $status
[623]	1748	}
	1749
	1750	#D-#==================================================
	1751	#D-function IGCM_sys_Dods_Cp
	1752	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	1753	#D-* Examples:
	1754	#D-
	1755	function IGCM_sys_Dods_Cp {
	1756	if ( $DEBUG_sys ) ; then
	1757	echo "IGCM_sys_Dods_Cp :" $@
	1758	fi
[715]	1759	typeset status
[623]	1760	if [ $DRYRUN = 0 ]; then
	1761
	1762	# if [ ! -d ${R_SAVE}/${1} ] ; then
	1763	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	1764	# echo "Nothing has been done."
	1765	# return
	1766	# fi
	1767
[649]	1768	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]	1769	status=$?
[623]	1770
[715]	1771	# if [ ${status} -gt 0 ] ; then
[623]	1772	# echo "IGCM_sys_Dods_Cp : error."
	1773	# cat out_dods_cp
	1774	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	1775	# else
	1776	# rm out_dods_cp
	1777	# fi
	1778
	1779	fi
[715]	1780	return $status
[623]	1781	}
	1782
	1783	#D-#==================================================
	1784	#D-function IGCM_sys_Put_Dods
	1785	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
	1786	#D-* Examples:
	1787	#D-
	1788	function IGCM_sys_Put_Dods {
	1789	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	1790	if ( $DEBUG_sys ) ; then
	1791	echo "IGCM_sys_Put_Dods :" $@
	1792	fi
[711]	1793	#set -vx
[715]	1794	typeset status
[623]	1795	if [ $DRYRUN = 0 ]; then
	1796	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
	1797	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
	1798	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
	1799	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
	1800	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1801	return
[622]	1802	fi
[623]	1803
	1804	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
	1805	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
	1806	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
	1807	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
	1808	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1809	return
	1810	fi
	1811	#
	1812	if [ -d ${R_SAVE}/${1} ] ; then
	1813	cd ${R_SAVE}
	1814	elif [ -d ${R_FIGR}/${1} ] ; then
	1815	cd ${R_FIGR}
	1816	fi
	1817
	1818	IGCM_sys_Dods_Rm ${1}
	1819	IGCM_sys_Dods_Cp ${1}
[715]	1820	status=0
[623]	1821
[715]	1822	if [ ${status} -gt 0 ] ; then
[623]	1823	echo "IGCM_sys_Put_Dods : error."
	1824	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1825	fi
	1826	fi
	1827	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]	1828	}
	1829
	1830	##############################################################
	1831	# REBUILD OPERATOR
	1832
[811]	1833	#D-#==================================================
	1834	#D-function IGCM_sys_rebuild
	1835	#D-* Purpose: rebuild parallel files
	1836	#D-* Examples:
	1837	#D-
[622]	1838	function IGCM_sys_rebuild {
[623]	1839	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
	1840	if ( $DEBUG_sys ) ; then
	1841	echo "IGCM_sys_rebuild :" $@
	1842	fi
[716]	1843
[734]	1844	typeset NB_ESSAI DELAI status i firstArg
[716]	1845	# number of tentative
	1846	NB_ESSAI=3
	1847	# time delay between tentative
	1848	DELAI=2
	1849
	1850	i=0
	1851	while [ $i -lt $NB_ESSAI ] ; do
[863]	1852	~p86ipsl/X64_CURIE/bin/rebuild -f -o $@ > out_rsync 2>&1
[716]	1853	status=$?
	1854	if [ ${status} -gt 0 ] ; then
	1855	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
	1856	cat out_rsync
	1857	\rm out_rsync
	1858	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[734]	1859	firstArg=${1}
	1860	\rm ${firstArg}
[733]	1861	sleep $DELAI
[716]	1862	else
	1863	\rm out_rsync
	1864	break
	1865	fi
	1866	(( i = i + 1 ))
	1867	done
	1868
[725]	1869	if [ ${status} -gt 0 ] ; then
[773]	1870	echo "IGCM_sys_rebuild : rebuild error code is ${status}"
[623]	1871	IGCM_debug_Exit "rebuild"
	1872	fi
[622]	1873
[623]	1874	IGCM_debug_PopStack "IGCM_sys_rebuild"
[622]	1875	}
	1876
[811]	1877	#D-#==================================================
	1878	#D-function IGCM_sys_rebuild_station
[912]	1879	#D-* Purpose: rebuild parallel files describing station
[811]	1880	#D-* Examples:
	1881	#D-
[697]	1882	function IGCM_sys_rebuild_station {
	1883	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
	1884	typeset i list_opt file_in file_out prefix_invert list_invert
	1885	if ( $DEBUG_sys ) ; then
	1886	echo "IGCM_sys_rebuild_station :" $@
	1887	fi
	1888	list_opt=$@
	1889
[773]	1890	# Invert Axis : t,x -> x,t
[697]	1891	# t,pres,x -> x,t,pres
	1892	# So that we can concatenate along x
	1893	i=0
	1894	for file_in in ${list_opt} ; do
	1895	(( i = i + 1))
	1896	[ ${i} = 1 ] && file_out=${file_in} && continue
	1897	prefix_invert=$( basename ${file_in} .nc )
	1898	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
	1899	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
	1900	done
	1901
	1902	# Concatenate
	1903	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
	1904
	1905	# Re-ivert file
	1906	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
	1907
[922]	1908	# Station re-ordering is too expansive to be run within libIGCM
[697]	1909	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
	1910	# This re-ordering must be done "in memory" by the cmorization process
	1911	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
	1912	# BEGIN reordering
	1913
	1914	# Only LMDZ text output contains the exact ordering of the station.
	1915	# We isolate this in the code below:
	1916	# 0 38 -157.5000000000000 70.98591549295774
	1917	# 0 54 27.49999999999999 67.18309859154928
	1918	# 0 56 -62.50000000000001 82.39436619718309
	1919	# 0 79 12.49999999999999 78.59154929577466
	1920	# 0 116 -165.0000000000000 76.05633802816901
	1921	# 0 117 130.0000000000000 70.98591549295774
	1922	# 0 118 110.0000000000000 87.46478873239437
	1923	# 1 40 4.999999999999995 51.97183098591550
	1924	# typeset iStation iProc list_opt file_in file_out prefix_invert
	1925	# typeset -Z4 j4
	1926	# typeset -Z3 j3
	1927
	1928	# unset list_opt
	1929	# set +A list_opt $@
	1930
	1931	# Filename after rebuild
	1932	# file_out=${list_opt[0]}
	1933	# Prefix of output files
	1934	# prefix_invert=$( basename ${file_out} .nc )
	1935	# Number of procs
	1936	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
	1937
	1938	# iProc=0
	1939	# while [ ${iProc} -lt ${num_proc} ] ; do
	1940	# Array containing Station as a number
	1941	# unset proc_stn
[912]	1942	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| gawk ' {print $2}' )
[697]	1943	# Number of stations produced by processor proc
	1944	# stationLast=${#proc_stn[*]}
	1945	# Proc number on 4 digits
	1946	# j4=${iProc}
	1947	# Init
	1948	# iStation=0
	1949	# while [ ${iStation} -lt ${stationLast} ] ; do
	1950	# Station number on 3 digits
	1951	# j3=${proc_stn[${iStation}]}
	1952	# Extract station
[773]	1953	# Invert Axis : t,x -> x,t
[697]	1954	# t,pres,x -> x,t,pres
	1955	# So that we can concatenate along x
	1956	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
	1957	# (( iStation = iStation + 1 ))
	1958	# done
	1959	# (( iProc = iProc + 1 ))
	1960	# done
	1961
	1962	# Concatenate all station along x
	1963	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
	1964
	1965	# Re-invert file
	1966	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
	1967
	1968	# END reordering
	1969
	1970	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
	1971	}
	1972
[622]	1973	############################################################
	1974	# Activate Running Environnment Variables
	1975
[811]	1976	#D-#==================================================
	1977	#D-function IGCM_sys_desactiv_variables
	1978	#D-* Purpose: set environement variables prior to execution
	1979	#D-* Examples:
	1980	#D-
[622]	1981	function IGCM_sys_activ_variables {
[623]	1982	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1983	if ( $DEBUG_sys ) ; then
	1984	echo "IGCM_sys_activ_variables"
	1985	fi
[622]	1986
[811]	1987	# --------------------------------------------------------------------
	1988	#D- MPI specifications
	1989	# --------------------------------------------------------------------
	1990
	1991	# --------------------------------------------------------------------
	1992	#D- Other specifications
	1993	# --------------------------------------------------------------------
	1994
[623]	1995	ulimit -s unlimited
[622]	1996
[623]	1997	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]	1998	}
	1999
	2000	############################################################
	2001	# Desactivate Running Environnment Variables
	2002
[811]	2003	#D-#==================================================
	2004	#D-function IGCM_sys_desactiv_variables
	2005	#D-* Purpose: unset environement variables after execution
	2006	#D-* Examples:
	2007	#D-
[622]	2008	function IGCM_sys_desactiv_variables {
[623]	2009	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	2010	if ( $DEBUG_sys ) ; then
	2011	echo "IGCM_sys_desactiv_variables"
	2012	fi
[811]	2013	# --------------------------------------------------------------------
	2014	#D- MPI specifications
	2015	# --------------------------------------------------------------------
	2016
	2017	# --------------------------------------------------------------------
	2018	#D- Other specifications
	2019	# --------------------------------------------------------------------
	2020
[623]	2021	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]	2022	}
	2023
[623]	2024	############################################################
	2025	# Build MPI/OMP scripts run file (dummy function)
[622]	2026
[811]	2027	#D-#==================================================
	2028	#D-function IGCM_sys_build_run_file
	2029	#D-* Purpose: build run file (deprecated)
	2030	#D-* Examples:
	2031	#D-
[622]	2032	function IGCM_sys_build_run_file {
	2033
[811]	2034	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]	2035
	2036	}
	2037
	2038	############################################################
	2039	# Build MPI/OMP scripts
[811]	2040
	2041	#D-#==================================================
	2042	#D-function IGCM_sys_build_execution_scripts
	2043	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	2044	#D-* Examples:
	2045	#D-
[623]	2046	function IGCM_sys_build_execution_scripts
[622]	2047	{
[623]	2048	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	2049	if ( $DEBUG_sys ) ; then
	2050	echo "IGCM_sys_build_execution_scripts " $@
	2051	fi
[622]	2052
[623]	2053	EXECUTION=${HOST_MPIRUN_COMMAND}
[622]	2054
[623]	2055	if ( ${OK_PARA_MPMD} ) ; then
[622]	2056
[623]	2057	if [ -f run_file ] ; then
	2058	IGCM_sys_Rm -f run_file
	2059	fi
	2060	touch run_file
[622]	2061
	2062	# run_file construction
	2063
	2064	# Then first loop on the components for the coupler ie oasis
	2065
[811]	2066	## the coupler ie oasis must be the first one
[622]	2067	for comp in ${config_ListOfComponents[*]} ; do
	2068
[623]	2069	eval ExeNameIn=\${config_Executable_${comp}[0]}
	2070	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	2071
[773]	2072	# for CPL component only
[623]	2073	if [ "X${comp}" = "XCPL" ] ; then
[622]	2074
[623]	2075	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	2076	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	2077	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	2078	fi
[622]	2079	done
	2080
	2081	# Then second loop on the components
	2082
	2083	for comp in ${config_ListOfComponents[*]} ; do
	2084
[623]	2085	eval ExeNameIn=\${config_Executable_${comp}[0]}
	2086	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	2087
[773]	2088	# Only if we really have an executable for the component and not the coupler ie oasis:
[623]	2089	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]	2090
[623]	2091	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	2092	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	2093	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	2094	fi
[622]	2095	done
	2096
	2097	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
	2098
	2099	IGCM_sys_Chmod u+x run_file
	2100	if ( $DEBUG_sys ) ; then
	2101	echo "run_file contains : "
	2102	cat run_file
	2103	fi
	2104
[623]	2105	else # Only one executable. launch it.
[622]	2106
	2107	for comp in ${config_ListOfComponents[*]} ; do
	2108
[685]	2109	# Only if we really have an executable for the component :
[623]	2110	eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]	2111	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]	2112
[685]	2113	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	2114	echo "" >> script_${ExeNameOut}.ksh
	2115	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	2116
	2117	if ( ${OK_PARA_OMP} ) ; then
	2118	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	2119	echo "" >> script_${ExeNameOut}.ksh
	2120	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	2121	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	2122	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	2123	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	2124	fi
	2125
	2126	if ( ${OK_PARA_MPI} ) ; then
[623]	2127	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]	2128	# Default : ccc_mprun used if nb_proc gt 1
	2129	# to have out/err per process on different files
	2130	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
	2131	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	2132	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]	2133	else
[685]	2134	# Default : ccc_mprun is NOT used if nb_proc eq 1
	2135	# to have out/err per process on different files
	2136	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	2137	echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
	2138	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]	2139	fi
[685]	2140
	2141	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
	2142	cat script_${ExeNameOut}.ksh
	2143
[623]	2144	fi
[622]	2145
	2146	done
	2147
[623]	2148	fi
[622]	2149
[623]	2150	IGCM_debug_Print 1 "sys Curie : execution command is "
	2151	IGCM_debug_Print 1 "$EXECUTION"
[622]	2152
[623]	2153	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]	2154	}
	2155
[714]	2156	############################################################
[811]	2157	# Check of space available on temporary filesytems.
	2158
	2159	#D-#==================================================
	2160	#D-function IGCM_sys_check_quota
	2161	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	2162	#D-* Examples:
	2163	#D-
[714]	2164	function IGCM_sys_check_quota {
[776]	2165	IGCM_debug_PushStack "IGCM_sys_check_quota"
	2166	if ( $DEBUG_sys ) ; then
	2167	echo "IGCM_sys_check_quota"
	2168	fi
	2169	# Limit of quota (in %)
	2170	limit_quota=90
	2171
	2172	# Check of the volume
[912]	2173	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
	2174	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
[776]	2175
	2176	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	2177
	2178	unit_avail=${volume_avail: -1}
	2179	unit_quota=${volume_quota: -1}
	2180
	2181	if [ "${unit_quota}" = "*" ] ; then
[811]	2182	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	2183	IGCM_debug_Print 1 "More than 100% of your quota is used"
	2184	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	2185	IGCM_debug_Print 1 "You must have more than 10% available to run"
	2186	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	2187	IGCM_debug_Verif_Exit
[714]	2188	fi
[776]	2189
	2190	temp_avail=${volume_avail%%${volume_avail: -1}*}
	2191	temp_quota=${volume_quota%%${volume_quota: -1}*}
	2192
	2193	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	2194
	2195	# Convertion
	2196	if [ ${volume_avail: -1} = "T" ] ; then
	2197	(( temp_avail = temp_avail * 1000000000000 ))
	2198	elif [ ${volume_avail: -1} = "G" ] ; then
	2199	(( temp_avail = temp_avail * 1000000000 ))
	2200	elif [ ${volume_avail: -1} = "M" ] ; then
	2201	(( temp_avail = temp_avail * 1000000 ))
	2202	elif [ ${volume_avail: -1} = "k" ] ; then
	2203	(( temp_avail = temp_avail * 1000 ))
	2204	else
	2205	(( temp_avail = volume_avail ))
	2206	fi
	2207	if [ ${volume_quota: -1} = "T" ] ; then
	2208	(( temp_quota = temp_quota * 1000000000000 ))
	2209	elif [ ${volume_quota: -1} = "G" ] ; then
	2210	(( temp_quota = temp_quota * 1000000000 ))
	2211	elif [ ${volume_quota: -1} = "M" ] ; then
	2212	(( temp_quota = temp_quota * 1000000 ))
	2213	elif [ ${volume_quota: -1} = "k" ] ; then
	2214	(( temp_quota = temp_quota * 1000 ))
	2215	else
	2216	(( temp_quota = volume_quota ))
	2217	fi
	2218	fi
	2219
	2220	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	2221	# echo "volume ratio is " $quota_volume
	2222
	2223	if [ ${quota_volume} -ge ${limit_quota} ] ; then
	2224	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	2225	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	2226	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	2227	IGCM_debug_Print 1 "You must have more than 10% available to run"
	2228	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	2229	IGCM_debug_Verif_Exit
	2230	fi
	2231
	2232	fi
	2233
	2234	# Check of the number of inodes
	2235
[912]	2236	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
	2237	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
[776]	2238
	2239	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
	2240
	2241	unit_avail=${inode_avail: -1}
	2242	unit_quota=${inode_quota: -1}
	2243
	2244	if [ "${unit_quota}" = "*" ] ; then
[811]	2245	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	2246	IGCM_debug_Print 1 "More than 100% of your quota is used"
	2247	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	2248	IGCM_debug_Print 1 "You must have more than 10% available to run"
	2249	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	2250	IGCM_debug_Verif_Exit
[776]	2251	fi
	2252
	2253	temp_avail=${inode_avail%%${inode_avail: -1}*}
	2254	temp_quota=${inode_quota%%${inode_quota: -1}*}
	2255
	2256	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	2257
	2258	# Convertion
	2259	if [ ${inode_avail: -1} = "T" ] ; then
	2260	(( temp_avail = temp_avail * 1000000000000 ))
	2261	elif [ ${inode_avail: -1} = "G" ] ; then
	2262	(( temp_avail = temp_avail * 1000000000 ))
	2263	elif [ ${inode_avail: -1} = "M" ] ; then
	2264	(( temp_avail = temp_avail * 1000000 ))
	2265	elif [ ${inode_avail: -1} = "k" ] ; then
	2266	(( temp_avail = temp_avail * 1000 ))
	2267	else
	2268	(( temp_avail = inode_avail ))
	2269	fi
	2270
	2271	if [ ${inode_quota: -1} = "T" ] ; then
	2272	(( temp_quota = temp_quota * 1000000000000 ))
	2273	elif [ ${inode_quota: -1} = "G" ] ; then
	2274	(( temp_quota = temp_quota * 1000000000 ))
	2275	elif [ ${inode_quota: -1} = "M" ] ; then
	2276	(( temp_quota = temp_quota * 1000000 ))
	2277	elif [ ${inode_quota: -1} = "k" ] ; then
	2278	(( temp_quota = temp_quota * 1000 ))
	2279	else
	2280	(( temp_quota = inode_quota ))
	2281	fi
	2282	fi
	2283	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	2284	# echo "inode ratio is " $quota_inode
	2285
	2286	if [ ${quota_inode} -ge ${limit_quota} ] ; then
	2287	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	2288	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
	2289	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	2290	IGCM_debug_Print 1 "You must have more than 10% available to run"
	2291	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	2292	IGCM_debug_Verif_Exit
	2293	fi
	2294	fi
	2295	IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]	2296	}
	2297
[773]	2298	#D-#==================================================
	2299	#D-function IGCM_sys_CountJobInQueue
	2300	#D-* Purpose: Check if job_name is currently
	2301	#D- running or in queue
	2302	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	2303	#D-
	2304	function IGCM_sys_CountJobInQueue {
	2305	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	2306	if ( $DEBUG_sys ) ; then
	2307	echo "IGCM_sys_CountJobInQueue"
	2308	fi
	2309
	2310	#NbRun=$( ccc_mstat -f \| grep -c ${JobName} )
	2311
	2312	# With -f option, the full job name is given in the last column
[834]	2313	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]	2314
	2315	eval ${2}=${NbRun}
	2316
	2317	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	2318	}
	2319
[622]	2320	##############################################################
	2321	# NCO OPERATOR
	2322
[811]	2323	#D-#==================================================
	2324	#D-function IGCM_sys_ncap2
	2325	#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
	2326	#D-* Examples:
	2327	#D-
[623]	2328	function IGCM_sys_ncap2 {
	2329	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
	2330	if ( $DEBUG_sys ) ; then
	2331	echo "IGCM_sys_ncap2 :" $@
	2332	fi
[713]	2333
	2334	typeset NB_ESSAI DELAI status i
	2335	# number of tentative
	2336	NB_ESSAI=3
	2337	# time delay between tentative
	2338	DELAI=2
	2339
	2340	i=0
	2341	while [ $i -lt $NB_ESSAI ] ; do
	2342	ncap2 "$@" > out_rsync 2>&1
	2343	status=$?
	2344	if [ ${status} -gt 0 ] ; then
	2345	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
	2346	cat out_rsync
	2347	\rm out_rsync
	2348	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2349	sleep $DELAI
[713]	2350	else
	2351	\rm out_rsync
	2352	break
	2353	fi
	2354	(( i = i + 1 ))
	2355	done
	2356
	2357	if [ ${status} -gt 0 ] ; then
[811]	2358	echo "IGCM_sys_ncap2 : ncap2 error"
	2359	IGCM_debug_Exit "ncap2"
[623]	2360	fi
	2361
	2362	IGCM_debug_PopStack "IGCM_sys_ncap2"
	2363	}
	2364
[811]	2365	#D-#==================================================
	2366	#D-function IGCM_sys_ncatted
	2367	#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
	2368	#D-* Examples:
	2369	#D-
[622]	2370	function IGCM_sys_ncatted {
[623]	2371	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
	2372	if ( $DEBUG_sys ) ; then
	2373	echo "IGCM_sys_ncatted :" $@
	2374	fi
[713]	2375
	2376	typeset NB_ESSAI DELAI status i
	2377	# number of tentative
	2378	NB_ESSAI=3
	2379	# time delay between tentative
	2380	DELAI=2
	2381
	2382	i=0
	2383	while [ $i -lt $NB_ESSAI ] ; do
	2384	ncatted "$@" > out_rsync 2>&1
	2385	status=$?
	2386	if [ ${status} -gt 0 ] ; then
	2387	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
	2388	cat out_rsync
	2389	\rm out_rsync
	2390	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2391	sleep $DELAI
[713]	2392	else
	2393	\rm out_rsync
	2394	break
	2395	fi
	2396	(( i = i + 1 ))
	2397	done
	2398
	2399	if [ ${status} -gt 0 ] ; then
[811]	2400	echo "IGCM_sys_ncatted : ncatted error"
	2401	IGCM_debug_Exit "ncatted"
[623]	2402	fi
[622]	2403
[623]	2404	IGCM_debug_PopStack "IGCM_sys_ncatted"
[622]	2405	}
	2406
[811]	2407	#D-#==================================================
	2408	#D-function IGCM_sys_ncbo
	2409	#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
	2410	#D-* Examples:
	2411	#D-
[622]	2412	function IGCM_sys_ncbo {
[623]	2413	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
	2414	if ( $DEBUG_sys ) ; then
	2415	echo "IGCM_sys_ncbo :" $@
	2416	fi
[713]	2417
	2418	typeset NB_ESSAI DELAI status i
	2419	# number of tentative
	2420	NB_ESSAI=3
	2421	# time delay between tentative
	2422	DELAI=2
	2423
	2424	i=0
	2425	while [ $i -lt $NB_ESSAI ] ; do
	2426	ncbo $@ > out_rsync 2>&1
	2427	status=$?
	2428	if [ ${status} -gt 0 ] ; then
	2429	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
	2430	cat out_rsync
	2431	\rm out_rsync
	2432	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2433	sleep $DELAI
[713]	2434	else
	2435	\rm out_rsync
	2436	break
	2437	fi
	2438	(( i = i + 1 ))
	2439	done
	2440
	2441	if [ ${status} -gt 0 ] ; then
[811]	2442	echo "IGCM_sys_ncbo : ncbo error"
	2443	IGCM_debug_Exit "ncbo"
[623]	2444	fi
[622]	2445
[623]	2446	IGCM_debug_PopStack "IGCM_sys_ncbo"
[622]	2447	}
	2448
[811]	2449	#D-#==================================================
	2450	#D-function IGCM_sys_ncdif
	2451	#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
	2452	#D-* Examples:
	2453	#D-
[622]	2454	function IGCM_sys_ncdiff {
[623]	2455	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
	2456	if ( $DEBUG_sys ) ; then
	2457	echo "IGCM_sys_ncdiff :" $@
	2458	fi
[713]	2459
	2460	typeset NB_ESSAI DELAI status i
	2461	# number of tentative
	2462	NB_ESSAI=3
	2463	# time delay between tentative
	2464	DELAI=2
	2465
	2466	i=0
	2467	while [ $i -lt $NB_ESSAI ] ; do
	2468	ncdiff $@ > out_rsync 2>&1
	2469	status=$?
	2470	if [ ${status} -gt 0 ] ; then
	2471	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
	2472	cat out_rsync
	2473	\rm out_rsync
	2474	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2475	sleep $DELAI
[713]	2476	else
	2477	\rm out_rsync
	2478	break
	2479	fi
	2480	(( i = i + 1 ))
	2481	done
	2482
	2483	if [ ${status} -gt 0 ] ; then
[811]	2484	echo "IGCM_sys_ncdiff : ncdiff error"
	2485	IGCM_debug_Exit "ncdiff"
[623]	2486	fi
[622]	2487
[623]	2488	IGCM_debug_PopStack "IGCM_sys_ncdiff"
[622]	2489	}
	2490
[811]	2491	#D-#==================================================
	2492	#D-function IGCM_sys_ncea
	2493	#D-* Purpose: encapsulate ncea call so as to manage error code and retry
	2494	#D-* Examples:
	2495	#D-
[622]	2496	function IGCM_sys_ncea {
[623]	2497	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
	2498	if ( $DEBUG_sys ) ; then
	2499	echo "IGCM_sys_ncea :" $@
	2500	fi
[713]	2501
	2502	typeset NB_ESSAI DELAI status i
	2503	# number of tentative
	2504	NB_ESSAI=3
	2505	# time delay between tentative
	2506	DELAI=2
	2507
	2508	i=0
	2509	while [ $i -lt $NB_ESSAI ] ; do
	2510	ncea $@ > out_rsync 2>&1
	2511	status=$?
	2512	if [ ${status} -gt 0 ] ; then
	2513	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
	2514	cat out_rsync
	2515	\rm out_rsync
	2516	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2517	sleep $DELAI
[713]	2518	else
	2519	\rm out_rsync
	2520	break
	2521	fi
	2522	(( i = i + 1 ))
	2523	done
	2524
	2525	if [ ${status} -gt 0 ] ; then
[811]	2526	echo "IGCM_sys_ncea : ncea error"
	2527	IGCM_debug_Exit "ncea"
[623]	2528	fi
[622]	2529
[623]	2530	IGCM_debug_PopStack "IGCM_sys_ncea"
[622]	2531	}
	2532
[811]	2533	#D-#==================================================
	2534	#D-function IGCM_sys_ncecat
	2535	#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
	2536	#D-* Examples:
	2537	#D-
[622]	2538	function IGCM_sys_ncecat {
[623]	2539	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
	2540	if ( $DEBUG_sys ) ; then
	2541	echo "IGCM_sys_ncecat :" $@
	2542	fi
[713]	2543
	2544	typeset NB_ESSAI DELAI status i
	2545	# number of tentative
	2546	NB_ESSAI=3
	2547	# time delay between tentative
	2548	DELAI=2
	2549
	2550	i=0
	2551	while [ $i -lt $NB_ESSAI ] ; do
	2552	ncecat $@ > out_rsync 2>&1
	2553	status=$?
	2554	if [ ${status} -gt 0 ] ; then
	2555	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
	2556	cat out_rsync
	2557	\rm out_rsync
	2558	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2559	sleep $DELAI
[713]	2560	else
	2561	\rm out_rsync
	2562	break
	2563	fi
	2564	(( i = i + 1 ))
	2565	done
	2566
	2567	if [ ${status} -gt 0 ] ; then
[811]	2568	echo "IGCM_sys_ncecat : ncecat error"
	2569	IGCM_debug_Exit "ncecat"
[623]	2570	fi
[622]	2571
[623]	2572	IGCM_debug_PopStack "IGCM_sys_ncecat"
[622]	2573	}
	2574
[811]	2575	#D-#==================================================
	2576	#D-function IGCM_sys_ncflint
	2577	#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
	2578	#D-* Examples:
	2579	#D-
[622]	2580	function IGCM_sys_ncflint {
[623]	2581	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
	2582	if ( $DEBUG_sys ) ; then
	2583	echo "IGCM_sys_ncflint :" $@
	2584	fi
[713]	2585
	2586	typeset NB_ESSAI DELAI status i
	2587	# number of tentative
	2588	NB_ESSAI=3
	2589	# time delay between tentative
	2590	DELAI=2
	2591
	2592	i=0
	2593	while [ $i -lt $NB_ESSAI ] ; do
	2594	ncflint $@ > out_rsync 2>&1
	2595	status=$?
	2596	if [ ${status} -gt 0 ] ; then
	2597	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
	2598	cat out_rsync
	2599	\rm out_rsync
	2600	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2601	sleep $DELAI
[713]	2602	else
	2603	\rm out_rsync
	2604	break
	2605	fi
	2606	(( i = i + 1 ))
	2607	done
	2608
	2609	if [ ${status} -gt 0 ] ; then
[811]	2610	echo "IGCM_sys_ncflint : ncflint error"
	2611	IGCM_debug_Exit "ncflint"
[623]	2612	fi
[622]	2613
[623]	2614	IGCM_debug_PopStack "IGCM_sys_ncflint"
[622]	2615	}
	2616
[811]	2617	#D-#==================================================
	2618	#D-function IGCM_sys_ncks
	2619	#D-* Purpose: encapsulate ncks call so as to manage error code and retry
	2620	#D-* Examples:
	2621	#D-
[622]	2622	function IGCM_sys_ncks {
[623]	2623	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
	2624	if ( $DEBUG_sys ) ; then
	2625	echo "IGCM_sys_ncks :" $@
	2626	fi
[713]	2627
	2628	typeset NB_ESSAI DELAI status i
	2629	# number of tentative
	2630	NB_ESSAI=3
	2631	# time delay between tentative
	2632	DELAI=2
	2633
	2634	i=0
	2635	while [ $i -lt $NB_ESSAI ] ; do
	2636	ncks $@ > out_rsync 2>&1
	2637	status=$?
	2638	if [ ${status} -gt 0 ] ; then
	2639	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
	2640	cat out_rsync
	2641	\rm out_rsync
	2642	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2643	sleep $DELAI
[713]	2644	else
	2645	\rm out_rsync
	2646	break
	2647	fi
	2648	(( i = i + 1 ))
	2649	done
	2650
	2651	if [ ${status} -gt 0 ] ; then
[811]	2652	echo "IGCM_sys_ncks : ncks error"
	2653	IGCM_debug_Exit "ncks"
[623]	2654	fi
[622]	2655
[623]	2656	IGCM_debug_PopStack "IGCM_sys_ncks"
[622]	2657	}
	2658
[811]	2659	#D-#==================================================
	2660	#D-function IGCM_sys_ncpdq
	2661	#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
	2662	#D-* Examples:
	2663	#D-
[622]	2664	function IGCM_sys_ncpdq {
[623]	2665	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
	2666	if ( $DEBUG_sys ) ; then
	2667	echo "IGCM_sys_ncpdq :" $@
	2668	fi
[713]	2669
	2670	typeset NB_ESSAI DELAI status i
	2671	# number of tentative
	2672	NB_ESSAI=3
	2673	# time delay between tentative
	2674	DELAI=2
	2675
	2676	i=0
	2677	while [ $i -lt $NB_ESSAI ] ; do
	2678	ncpdq $@ > out_rsync 2>&1
	2679	status=$?
	2680	if [ ${status} -gt 0 ] ; then
	2681	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
	2682	cat out_rsync
	2683	\rm out_rsync
	2684	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2685	sleep $DELAI
[713]	2686	else
	2687	\rm out_rsync
	2688	break
	2689	fi
	2690	(( i = i + 1 ))
	2691	done
	2692
	2693	if [ ${status} -gt 0 ] ; then
[811]	2694	echo "IGCM_sys_ncpdq : ncpdq error"
	2695	IGCM_debug_Exit "ncpdq"
[623]	2696	fi
[622]	2697
[623]	2698	IGCM_debug_PopStack "IGCM_sys_ncpdq"
[622]	2699	}
	2700
[811]	2701	#D-#==================================================
	2702	#D-function IGCM_sys_ncra
	2703	#D-* Purpose: encapsulate ncra call so as to manage error code and retry
	2704	#D-* Examples:
	2705	#D-
[622]	2706	function IGCM_sys_ncra {
[623]	2707	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
	2708	if ( $DEBUG_sys ) ; then
	2709	echo "IGCM_sys_ncra :" $@
	2710	fi
[713]	2711
	2712	typeset NB_ESSAI DELAI status i
	2713	# number of tentative
	2714	NB_ESSAI=3
	2715	# time delay between tentative
	2716	DELAI=2
	2717
	2718	i=0
	2719	while [ $i -lt $NB_ESSAI ] ; do
	2720	ncra $@ > out_rsync 2>&1
	2721	status=$?
	2722	if [ ${status} -gt 0 ] ; then
	2723	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
	2724	cat out_rsync
	2725	\rm out_rsync
	2726	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2727	sleep $DELAI
[713]	2728	else
	2729	\rm out_rsync
	2730	break
	2731	fi
	2732	(( i = i + 1 ))
	2733	done
	2734
	2735	if [ ${status} -gt 0 ] ; then
[811]	2736	echo "IGCM_sys_ncra : ncra error"
	2737	IGCM_debug_Exit "ncra"
[623]	2738	fi
[622]	2739
[623]	2740	IGCM_debug_PopStack "IGCM_sys_ncra"
[622]	2741	}
	2742
[811]	2743	#D-#==================================================
	2744	#D-function IGCM_sys_ncrcat
	2745	#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
	2746	#D-* Examples:
	2747	#D-
[622]	2748	function IGCM_sys_ncrcat {
[623]	2749	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
	2750	if ( $DEBUG_sys ) ; then
	2751	echo "IGCM_sys_ncrcat :" $@
	2752	fi
[713]	2753
[725]	2754	typeset NB_ESSAI DELAI status i lastArg
[713]	2755	# number of tentative
	2756	NB_ESSAI=3
	2757	# time delay between tentative
	2758	DELAI=2
	2759
	2760	i=0
	2761	while [ $i -lt $NB_ESSAI ] ; do
	2762	ncrcat $@ > out_rsync 2>&1
	2763	status=$?
	2764	if [ ${status} -gt 0 ] ; then
	2765	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
	2766	cat out_rsync
	2767	\rm out_rsync
	2768	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2769	sleep $DELAI
[721]	2770	elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" out_rsync )" = "X" ] ; then
	2771	IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
	2772	cat out_rsync
[725]	2773	# remove files having corrupted time axis
[734]	2774	eval lastArg=\${$#}
[725]	2775	IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
	2776	\rm ${lastArg}
[721]	2777	\rm out_rsync
	2778	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2779	sleep $DELAI
[713]	2780	else
	2781	\rm out_rsync
	2782	break
	2783	fi
	2784	(( i = i + 1 ))
	2785	done
	2786
	2787	if [ ${status} -gt 0 ] ; then
[811]	2788	echo "IGCM_sys_ncrcat : ncrcat error"
	2789	#IGCM_debug_Exit "ncrcat"
[623]	2790	fi
[622]	2791
[623]	2792	IGCM_debug_PopStack "IGCM_sys_ncrcat"
[622]	2793	}
	2794
[811]	2795	#D-#==================================================
	2796	#D-function IGCM_sys_ncrename
	2797	#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
	2798	#D-* Examples:
	2799	#D-
[622]	2800	function IGCM_sys_ncrename {
[623]	2801	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
	2802	if ( $DEBUG_sys ) ; then
	2803	echo "IGCM_sys_ncrename :" $@
	2804	fi
[713]	2805
	2806	typeset NB_ESSAI DELAI status i
	2807	# number of tentative
	2808	NB_ESSAI=3
	2809	# time delay between tentative
	2810	DELAI=2
	2811
	2812	i=0
	2813	while [ $i -lt $NB_ESSAI ] ; do
	2814	ncrename $@ > out_rsync 2>&1
	2815	status=$?
	2816	if [ ${status} -gt 0 ] ; then
	2817	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
	2818	cat out_rsync
	2819	\rm out_rsync
	2820	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2821	sleep $DELAI
[713]	2822	else
	2823	\rm out_rsync
	2824	break
	2825	fi
	2826	(( i = i + 1 ))
	2827	done
	2828
	2829	if [ ${status} -gt 0 ] ; then
[811]	2830	echo "IGCM_sys_ncrename : ncrename error"
	2831	IGCM_debug_Exit "ncrename"
[623]	2832	fi
[622]	2833
[623]	2834	IGCM_debug_PopStack "IGCM_sys_ncrename"
[622]	2835	}
	2836
[811]	2837	#D-#==================================================
	2838	#D-function IGCM_sys_ncwa
	2839	#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
	2840	#D-* Examples:
	2841	#D-
[622]	2842	function IGCM_sys_ncwa {
[623]	2843	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
	2844	if ( $DEBUG_sys ) ; then
	2845	echo "IGCM_sys_ncwa :" $@
	2846	fi
[713]	2847
	2848	typeset NB_ESSAI DELAI status i
	2849	# number of tentative
	2850	NB_ESSAI=3
	2851	# time delay between tentative
	2852	DELAI=2
	2853
	2854	i=0
	2855	while [ $i -lt $NB_ESSAI ] ; do
	2856	ncwa $@ > out_rsync 2>&1
	2857	status=$?
	2858	if [ ${status} -gt 0 ] ; then
	2859	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
	2860	cat out_rsync
	2861	\rm out_rsync
	2862	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]	2863	sleep $DELAI
[713]	2864	else
	2865	\rm out_rsync
	2866	break
	2867	fi
	2868	(( i = i + 1 ))
	2869	done
	2870
	2871	if [ ${status} -gt 0 ] ; then
[811]	2872	echo "IGCM_sys_ncwa : ncwa error"
	2873	IGCM_debug_Exit "ncwa"
[623]	2874	fi
[622]	2875
[623]	2876	IGCM_debug_PopStack "IGCM_sys_ncwa"
[622]	2877	}
[623]	2878
	2879	##############################################################
	2880	# CDO OPERATOR
	2881
[811]	2882	#D-#==================================================
	2883	#D-function IGCM_sys_cdo
	2884	#D-* Purpose: encapsulate cdo call so as to manage error code and retry
	2885	#D-* Examples:
	2886	#D-
[623]	2887	function IGCM_sys_cdo {
	2888	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
	2889	if ( $DEBUG_sys ) ; then
	2890	echo "IGCM_sys_cdo :" $@
	2891	fi
[713]	2892
	2893	typeset status
	2894
	2895	\cdo $@ > out_rsync 2>&1
	2896	status=$?
	2897	if [ ${status} -gt 0 ] ; then
	2898	echo "IGCM_sys_cdo : error code ${status}"
	2899	cat out_rsync
	2900	\rm out_rsync
[623]	2901	IGCM_debug_PopStack "IGCM_sys_cdo"
	2902	return 1
	2903	else
	2904	IGCM_debug_PopStack "IGCM_sys_cdo"
	2905	return 0
	2906	fi
	2907
	2908	IGCM_debug_PopStack "IGCM_sys_cdo"
	2909	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: