Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_mercurex9.ksh @ 765

Last change on this file since 765 was 743, checked in by sdipsl, 12 years ago
Minimal modification when directing output to the scratch during TEST simulation
Property svn:keywords set to `Revision Author Date`
File size: 58.0 KB

Rev	Line
[192]	1	#!/bin/ksh
[99]	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip
[373]	5	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
[99]	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[719]	16	# if you use the prefix #D- for comments to be extracted.
[99]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[192]	21	#D-LibIGCM_sys for Mercure SX9
[99]	22	#D-#==================================================
	23	#D-
[719]	24	#D- This ksh library if a layer under some usefull
[99]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
[111]	34	# gawk specific location on SX9
	35	export PATH=/applications/gawk-3.0.4/bin:${PATH}
	36
[99]	37	#====================================================
	38	# set DEBUG_sys to true to output calls of function
	39	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	40
	41	#====================================================
	42	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	43	typeset -r DRYRUN=${DRYRUN:=0}
	44
	45	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	46	# -------------------------------------------------------------------------------------
	47	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
	48	# \| \| Cp/Exe param files \| \| Chmod \| \|
	49	# \| \| Qsub \| \| \| \|
	50	# -------------------------------------------------------------------------------------
	51	# \| 0 \| yes \| yes \| yes \| yes \|
	52	# -------------------------------------------------------------------------------------
	53	# \| 1 \| yes \| yes \| yes \| no \|
	54	# -------------------------------------------------------------------------------------
	55	# \| 2 \| yes \| yes \| no \| no \|
	56	# -------------------------------------------------------------------------------------
	57	# \| 3 \| yes \| no \| no \| no \|
	58	# -------------------------------------------------------------------------------------
	59
	60	#=====================================================
	61	# Global Variables :
	62	#=====================================================
	63	# Language : "fr" or "en"
	64	typeset -r MYLANG="fr"
	65
	66	#=====================================================
[192]	67	# Host and user names
	68	# $hostname ou hostname
[228]	69	typeset HOST=${HOST:=$( hostname )}
[99]	70	# $username ou whoami
[228]	71	typeset LOGIN=${LOGIN:=$( whoami )}
[99]	72	# $hostname of the MASTER job
	73	typeset -r MASTER=mercure
	74
	75	#D-
	76	#D-#==================================================
	77	#D-Program used in libIGCM
	78	#D-#==================================================
	79
[742]	80	# Submit command
	81	typeset SUBMIT=${SUBMIT:=qsub}
[99]	82	# rsync with path
	83	typeset -r RSYNC=/home/cont003/p86denv/SX_RSYNC/bin/rsync
	84	# RSYNC_opt args to rsync
[192]	85	typeset -r RSYNC_opt="-va"
[99]	86	# ie storage filesystem
[540]	87	typeset -r STOREHOST=${MASTER}
[99]	88
	89	#====================================================
	90	# Host specific DIRECTORIES
	91	#====================================================
	92
[257]	93	# ============ CESIUM START ============ #
	94
[99]	95	#====================================================
[192]	96	#- Mirror libIGCM from mercure to cesium if needed
[587]	97	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	98	#if [ ! ${ROOTSYS} = "home" ] ; then
	99	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	100	#else
	101	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	102	#fi
[192]	103
	104	#====================================================
[99]	105	#- libIGCM_POST
[587]	106	#if ( ${MirrorlibIGCM} ) ; then
	107	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	108	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	109	#else
	110	# typeset -r libIGCM_POST=${libIGCM}
	111	#fi
[99]	112
[257]	113	# ============ CESIUM END ============ #
	114
[99]	115	#====================================================
[717]	116	#- MirrorlibIGCM for frontend
[587]	117	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
[257]	118
	119	#====================================================
[717]	120	#- libIGCM_POST for frontend
[587]	121	typeset -r libIGCM_POST=${libIGCM}
[257]	122
	123	#====================================================
[587]	124	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	125	typeset -r R_EXE="${MODIPSL}/bin"
	126
	127	#====================================================
	128	#- SUBMIT_DIR : submission dir
	129	typeset SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
	130
	131	#====================================================
[99]	132	#- IN
[545]	133	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
[717]	134	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
[99]	135
	136	#====================================================
[725]	137	#- ARCHIVE (dedicated to large files)
	138	typeset ARCHIVE=${CCCSTOREDIR}
	139
	140	#====================================================
	141	#- STORAGE (dedicated to small/medium files)
	142	typeset STORAGE=${CCCWORKDIR}
	143
	144	#====================================================
[544]	145	#- R_OUT
[725]	146	typeset R_OUT=${ARCHIVE}/IGCM_OUT
[99]	147
	148	#====================================================
[587]	149	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[725]	150	typeset R_FIG=${STORAGE}/IGCM_OUT
[587]	151
	152	#====================================================
[544]	153	#- R_BUF (ONLY FOR double copy an scratch)
	154	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
	155
	156	#====================================================
	157	#- BIG_DIR : BIG_DIR to store files waiting for rebuild
	158	typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
	159
	160	#====================================================
[99]	161	#- OUT_POST
	162	typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
	163
	164	#====================================================
	165	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
	166	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${LOCALTMPDIR}}
	167
	168	#====================================================
	169	#- HOST_MPIRUN_COMMAND
	170	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="mpirun"}
	171
	172	#====================================================
	173	#- Max number of arguments passed to nco operator or demigration command
	174	UNIX_MAX_LIMIT=360
	175
[616]	176	#====================================================
	177	#- set PackDefault true on NEC SX9
	178	PackDefault=true
	179
	180	#====================================================
	181	#- Default number of MPI task for IPSL coupled model
	182	#- required for backward compatibility
	183	#-
	184	DEFAULT_NUM_PROC_OCE=1
	185	DEFAULT_NUM_PROC_CPL=1
	186	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - 1 ))
	187	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
	188
[99]	189	#D-#==================================================
[725]	190	#D-function IGCM_sys_ChangeArchive
	191	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
	192	#D-* Examples:
	193	#D-
	194	function IGCM_sys_ChangeArchive {
	195	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
	196
[743]	197	R_OUT=${SCRATCHDIR}/IGCM_OUT
	198	R_FIG=${SCRATCHDIR}/IGCM_OUT
[725]	199
	200	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
	201	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
	202
	203	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
	204	}
	205
	206	#D-#==================================================
[531]	207	#D-function IGCM_sys_RshMaster
	208	#D-* Purpose: Connection to frontend machine.
	209	#D-* Examples:
	210	#D-
	211	function IGCM_sys_RshMaster {
[544]	212	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
	213	ssh -t ${MASTER} /bin/ksh <<-EOF
	214	export libIGCM=${libIGCM}
	215	export DEBUG_debug=${DEBUG_debug}
	216	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
	217	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
	218	${@}
[531]	219	EOF
[544]	220	if [ $? -gt 0 ] ; then
	221	echo "IGCM_sys_RshMaster : erreur."
	222	IGCM_debug_Exit "IGCM_sys_RshMaster"
	223	fi
	224	IGCM_debug_PopStack "IGCM_sys_RshMaster"
[531]	225	}
	226
	227	#D-#==================================================
[99]	228	#D-function IGCM_sys_RshArchive
	229	#D-* Purpose: Archive rsh command
	230	#D-* Examples:
	231	#D-
	232	function IGCM_sys_RshArchive {
[544]	233	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	234	/bin/ksh <<-EOF
	235	${@}
[99]	236	EOF
[544]	237	if [ $? -gt 0 ] ; then
	238	echo "IGCM_sys_RshArchive : erreur."
	239	IGCM_debug_Exit "IGCM_sys_RshArchive"
	240	fi
	241	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[99]	242	}
	243
	244	#D-#==================================================
	245	#D-function IGCM_sys_RshPost
	246	#D-* Purpose: Post-process rsh command
	247	#D-* Examples:
	248	#D-
	249	function IGCM_sys_RshPost {
[544]	250	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
	251	if ( $DEBUG_sys ) ; then
	252	echo "IGCM_sys_RshPost :" $@
	253	fi
[167]	254
[544]	255	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
	256	#cat tmp_IGCM_sys_RshPost_$$
	257	# keep standard input (stdin) for the loop onto temporary file
	258	cat >tmp_IGCM_sys_RshPost_$$
	259
	260	# ============ FRONTEND START ============ #
	261
[582]	262	/bin/ksh <tmp_IGCM_sys_RshPost_$$
	263	if [ $? -gt 0 ] ; then
	264	echo "IGCM_sys_RshPost : erreur."
	265	IGCM_debug_Exit "IGCM_sys_RshPost"
	266	fi
[717]	267	# delete temporary file
[582]	268	\rm tmp_IGCM_sys_RshPost_$$
[544]	269
	270	# ============ FRONTEND END ============ #
	271
[257]	272	# ============ CESIUM START ============ #
[582]	273	# typeset NB_ESSAI DELAI status i
	274	# # number of tentative
	275	# NB_ESSAI=10
	276	# # time delay between tentative
	277	# DELAI=10
	278	# (( i = 0 ))
	279	# while [ $i -lt $NB_ESSAI ] ; do
	280	# ssh -t mercure01 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
	281	# status=$?
	282	# if [ ${status} -gt 0 ]; then
	283	# IGCM_debug_Print 2 "IGCM_sys_RshPost : ssh failed ${i}/${NB_ESSAI}"
	284	# IGCM_debug_Print 2 "IGCM_sys_RshPost : sleep ${DELAI} seconds and try again."
	285	# sleep $DELAI
	286	# else
	287	# break
	288	# fi
	289	# (( i = i + 1 ))
	290	# done
	291	# # delete temporary file
	292	# /bin/rm tmp_IGCM_sys_RshPost_$$
[257]	293	# ============ CESIUM END ============ #
[170]	294
[544]	295	IGCM_debug_PopStack "IGCM_sys_RshPost"
[99]	296	}
	297
	298	#D-#==================================================
[153]	299	#D-function IGCM_sys_SendMail
	300	#D-* Purpose: Send mail when simulation is over
	301	#D-* Examples:
	302	#D-
	303	function IGCM_sys_SendMail {
[544]	304	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
	305	if ( $DEBUG_sys ) ; then
	306	echo "IGCM_sys_SendMail :" $@
	307	fi
[153]	308
[544]	309	if ( ${ExitFlag} ) ; then
	310	status=failed
	311	else
	312	status=completed
	313	fi
[717]	314
[544]	315	cat << END_MAIL > job_end.mail
[717]	316	Dear ${LOGIN},
[437]	317
[452]	318	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
[153]	319	Job started : ${DateBegin}
	320	Job ended : ${DateEnd}
[434]	321	Output files are available in ${R_SAVE}
[616]	322	Files to be rebuild are temporarily available in ${REBUILD_DIR}
	323	Pre-packed files are temporarily available in ${R_BUFR}
[452]	324	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
[153]	325	END_MAIL
	326
[544]	327	if [ X"${config_UserChoices_MailName}" != X ] ; then
	328	mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
	329	elif [ -f ~/.forward ] ; then
	330	mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
	331	else
	332	mailx -s "${config_UserChoices_JobName} ${status}" ${LOGIN} < job_end.mail
	333	fi
[153]	334
[544]	335	if [ $? -gt 0 ] ; then
	336	echo "IGCM_sys_SendMail : erreur."
	337	IGCM_debug_Exit "IGCM_sys_SendMail"
	338	fi
	339	IGCM_debug_PopStack "IGCM_sys_SendMail"
[153]	340	}
	341
	342	#D-#==================================================
[99]	343	#D-function IGCM_sys_Mkdir
	344	#D-* Purpose: Master locale mkdir command
	345	#D-* Examples:
	346	#D-
	347	function IGCM_sys_Mkdir {
[544]	348	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
	349	if ( $DEBUG_sys ) ; then
	350	echo "IGCM_sys_Mkdir :" $@
	351	fi
[742]	352
	353	typeset NB_ESSAI DELAI status i
	354	# number of tentative
	355	NB_ESSAI=3
	356	# time delay between tentative
	357	DELAI=2
	358
[544]	359	if [ ! -d ${1} ]; then
[742]	360	i=0
	361	while [ $i -lt $NB_ESSAI ] ; do
	362	\mkdir -p $1
	363	status=$?
	364	if [ ${status} -gt 0 ] ; then
	365	IGCM_debug_Print 2 "IGCM_sys_Mkdir : mkdir failed error code ${status} ${i}/${NB_ESSAI}"
	366	IGCM_debug_Print 2 "IGCM_sys_Mkdir : sleep ${DELAI} seconds and try again."
	367	sleep $DELAI
	368	else
	369	break
	370	fi
	371	(( i = i + 1 ))
	372	done
[544]	373	fi
	374	# vérification :
	375	if [ ! -d ${1} ] ; then
	376	echo "IGCM_sys_Mkdir : erreur."
	377	IGCM_debug_Exit "IGCM_sys_Mkdir"
	378	fi
	379	IGCM_debug_PopStack "IGCM_sys_Mkdir"
[99]	380	}
	381
	382	#D-#==================================================
	383	#D-function IGCM_sys_MkdirArchive
	384	#D-* Purpose: Mkdir on Archive
	385	#D-* Examples:
	386	#D-
	387	function IGCM_sys_MkdirArchive {
[544]	388	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	389	if ( $DEBUG_sys ) ; then
	390	echo "IGCM_sys_MkdirArchive :" $@
	391	fi
[742]	392
	393	typeset NB_ESSAI DELAI status i
	394	# number of tentative
	395	NB_ESSAI=3
	396	# time delay between tentative
	397	DELAI=2
	398
[717]	399	if [ ! -d ${1} ]; then
[742]	400	i=0
	401	while [ $i -lt $NB_ESSAI ] ; do
	402	\mkdir -p $1
	403	status=$?
	404	if [ ${status} -gt 0 ] ; then
	405	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status} ${i}/${NB_ESSAI}"
	406	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : sleep ${DELAI} seconds and try again."
	407	sleep $DELAI
	408	else
	409	break
	410	fi
	411	(( i = i + 1 ))
	412	done
[544]	413	fi
[742]	414
	415	# vérification :
	416	if [ ! -d ${1} ] ; then
	417	echo "IGCM_sys_MkdirArchive : erreur."
	418	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
	419	fi
[544]	420	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[99]	421	}
	422
	423	#D-#==================================================
	424	#D-function IGCM_sys_MkdirWork
	425	#D-* Purpose: Mkdir on Work
	426	#D-* Examples:
	427	#D-
	428	function IGCM_sys_MkdirWork {
[544]	429	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
	430	if ( $DEBUG_sys ) ; then
	431	echo "IGCM_sys_MkdirWork :" $@
	432	fi
	433	#- creation de repertoire sur le serveur fichier
[717]	434	if [ ! -d ${1} ]; then
[544]	435	\mkdir -p $1
	436	if [ $? -gt 0 ] ; then
	437	echo "IGCM_sys_MkdirWork : erreur."
	438	IGCM_debug_Exit "IGCM_sys_MkdirWork"
[99]	439	fi
[544]	440	fi
	441	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
[99]	442	}
	443
	444	#D-#==================================================
	445	#D-function IGCM_sys_Cd
	446	#D-* Purpose: master cd command
	447	#D-* Examples:
	448	#D-
	449	function IGCM_sys_Cd {
[544]	450	IGCM_debug_PushStack "IGCM_sys_Cd" $@
	451	if ( $DEBUG_sys ) ; then
	452	echo "IGCM_sys_Cd :" $@
	453	fi
	454	\cd $1
	455	if [ $? -gt 0 ] ; then
	456	echo "IGCM_sys_Cd : erreur."
	457	IGCM_debug_Exit "IGCM_sys_Cd"
	458	fi
	459	IGCM_debug_PopStack "IGCM_sys_Cd"
[99]	460	}
	461
	462	#D-#==================================================
	463	#D-function IGCM_sys_Chmod
	464	#D-* Purpose: Chmod
	465	#D-* Examples:
	466	#D-
	467	function IGCM_sys_Chmod {
[544]	468	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
	469	if ( $DEBUG_sys ) ; then
	470	echo "IGCM_sys_Chmod :" $@
	471	fi
	472	if [ $DRYRUN -le 1 ]; then
	473	\chmod $@
	474	if [ $? -gt 0 ] ; then
	475	echo "IGCM_sys_Chmod : erreur."
	476	IGCM_debug_Exit "IGCM_sys_Chmod"
[99]	477	fi
[544]	478	else
	479	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	480	fi
	481	IGCM_debug_PopStack "IGCM_sys_Chmod"
[99]	482	}
	483
	484	#D-#==================================================
	485	#D-function IGCM_sys_FileSize
[544]	486	#D-* Purpose: Filesize
[99]	487	#D-* Examples:
	488	#D-
	489	function IGCM_sys_FileSize {
[544]	490	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
[99]	491
[544]	492	typeset sizeF
	493	set +A sizeF -- $( ls -la ${1} )
	494	if [ $? -gt 0 ] ; then
	495	IGCM_debug_Exit "IGCM_sys_FileSize"
	496	fi
	497	eval ${2}=${sizeF[4]}
[99]	498
[544]	499	IGCM_debug_PopStack "IGCM_sys_FileSize"
[99]	500	}
	501
	502	#D-#==================================================
	503	#D-function IGCM_sys_TestDir
	504	#D-* Purpose: Test Directory that must exists
	505	#D-* Examples:
	506	#D-
	507	function IGCM_sys_TestDir {
[544]	508	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
	509	if ( $DEBUG_sys ) ; then
	510	echo "IGCM_sys_TestDir :" $@
	511	fi
	512	typeset ExistFlag
	513	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	514	IGCM_debug_PopStack "IGCM_sys_TestDir"
[99]	515
[544]	516	return ${ExistFlag}
[99]	517	}
	518
	519	#D-#==================================================
	520	#D-function IGCM_sys_TestDirArchive
	521	#D-* Purpose: Test Directory that must exists on Archive
	522	#D-* Examples:
	523	#D-
	524	function IGCM_sys_TestDirArchive {
[544]	525	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	526	if ( $DEBUG_sys ) ; then
	527	echo "IGCM_sys_TestDirArchive :" $@
	528	fi
	529	typeset ExistFlag
	530	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	531	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
[99]	532
[544]	533	return ${ExistFlag}
[99]	534	}
	535
	536	#D-#==================================================
[548]	537	#D-function IGCM_sys_IsFileArchived
	538	#D-* Purpose: Test file that must NOT EXISTS on Archive
	539	#D-* Examples:
	540	#D-
	541	function IGCM_sys_IsFileArchived {
	542	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	543	if ( $DEBUG_sys ) ; then
	544	echo "IGCM_sys_IsFileArchived :" $@
	545	fi
	546	typeset IsArchivedFlag
[618]	547	IsArchivedFlag=$( [ X$( echo $1 \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[548]	548	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	549
	550	return ${IsArchivedFlag}
	551	}
	552
	553	#D-#==================================================
[99]	554	#D-function IGCM_sys_TestFileArchive
	555	#D-* Purpose: Test file that must NOT EXISTS on Archive
	556	#D-* Examples:
	557	#D-
	558	function IGCM_sys_TestFileArchive {
[544]	559	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	560	typeset ExistFlag
	561	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	562	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[99]	563
[544]	564	return ${ExistFlag}
[99]	565	}
	566
	567	#D-#==================================================
[544]	568	#D-function IGCM_sys_TestFileBuffer
	569	#D-* Purpose: Test file that must NOT EXISTS on Buffer
	570	#D-* Examples:
	571	#D-
	572	function IGCM_sys_TestFileBuffer {
	573	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
	574	typeset ExistFlag
	575	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	576	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
	577
	578	return ${ExistFlag}
	579	}
	580
	581	#D-#==================================================
[99]	582	#D-function IGCM_sys_CountFileArchive
	583	#D-* Purpose: Count files on Archive filesystem
	584	#D-* Examples:
	585	#D-
	586	function IGCM_sys_CountFileArchive {
[544]	587	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	588	ls ${@} 2>/dev/null \| wc -l
	589	if [ $? -gt 0 ] ; then
	590	echo "IGCM_sys_CountFileArchive : erreur."
	591	fi
	592	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[99]	593	}
	594
	595	#D-#==================================================
[544]	596	#D-function IGCM_sys_CountFileBuffer
	597	#D-* Purpose: Count files on Scratch filesystem
	598	#D-* Examples:
	599	#D-
	600	function IGCM_sys_CountFileBuffer {
	601	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
	602	ls ${@} 2>/dev/null \| wc -l
	603	if [ $? -gt 0 ] ; then
	604	echo "IGCM_sys_CountFileBuffer : erreur."
	605	fi
	606	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
	607	}
	608
	609	#D-#==================================================
[99]	610	#D-function IGCM_sys_Tree
	611	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	612	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	613	#D-
	614	function IGCM_sys_Tree {
[544]	615	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	616	if ( $DEBUG_sys ) ; then
	617	echo "IGCM_sys_Tree :" $@
	618	fi
[99]	619
[544]	620	\ls -lR ${@}
[99]	621
[544]	622	IGCM_debug_PopStack "IGCM_sys_Tree"
[99]	623	}
	624
	625	#D-#==================================================
	626	#D-function IGCM_sys_Tar
[544]	627	#D-* Purpose: master tar command
[99]	628	#D-* Examples:
	629	#D-
	630	function IGCM_sys_Tar {
[544]	631	IGCM_debug_PushStack "IGCM_sys_Tar" $@
	632	if ( $DEBUG_sys ) ; then
	633	echo "IGCM_sys_Tar :" $@
	634	fi
[742]	635	IGCM_sys_RshMaster "\tar $@"
[544]	636	if [ $? -gt 0 ] ; then
	637	echo "IGCM_sys_Tar : erreur."
	638	IGCM_debug_Exit "IGCM_sys_Tar"
	639	fi
	640	IGCM_debug_PopStack "IGCM_sys_Tar"
[99]	641	}
	642
	643	#D-#==================================================
	644	#D-function IGCM_sys_UnTar
	645	#D-* Purpose: master un-tar command
	646	#D-* Examples:
	647	#D-
	648	function IGCM_sys_UnTar {
[544]	649	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
	650	if ( $DEBUG_sys ) ; then
	651	echo "IGCM_sys_UnTar :" $@
	652	fi
	653	\tar xvf $1
	654	if [ $? -gt 0 ] ; then
	655	echo "IGCM_sys_UnTar : erreur."
	656	IGCM_debug_Exit "IGCM_sys_UnTar"
	657	fi
	658	IGCM_debug_PopStack "IGCM_sys_UnTar"
[99]	659	}
	660
	661	#D-#==================================================
	662	#D-function IGCM_sys_Qsub
	663	#D-* Purpose: Qsub new job
	664	#D-* Examples:
	665	#D-
	666	function IGCM_sys_Qsub {
[544]	667	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	668	if ( $DEBUG_sys ) ; then
	669	echo "IGCM_sys_Qsub :" $@
	670	fi
	671	/usr/bin/nqsII/qsub -o ${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} < $1
	672	if [ $? -gt 0 ] ; then
	673	echo "IGCM_sys_Qsub : erreur -o ${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} $@."
	674	IGCM_debug_Exit "IGCM_sys_Qsub"
	675	fi
	676	IGCM_debug_PopStack "IGCM_sys_Qsub"
[99]	677	}
	678
	679	#D-#==================================================
	680	#D-function IGCM_sys_QsubPost
	681	#D-* Purpose: Qsub new job on scalaire
	682	#D-* Examples:
	683	#D-
	684	function IGCM_sys_QsubPost {
[544]	685	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	686	if ( $DEBUG_sys ) ; then
	687	echo "IGCM_sys_QsubPost :" $@
	688	fi
	689
	690	# ============ FRONTEND START ============ #
	691
[582]	692	/usr/bin/nqsII/qsub -q scalaire -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM}/$1.job -v ${listVarEnv}
[544]	693
	694	# ============ FRONTEND END ============ #
	695
	696	# ============ CESIUM START ============ #
[582]	697	# typeset NB_ESSAI DELAI status i
	698	# # number of tentative
	699	# NB_ESSAI=10
	700	# # time delay between tentative
	701	# DELAI=10
	702	# (( i = 0 ))
	703	# while [ $i -lt $NB_ESSAI ] ; do
	704	# /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
	705	# status=$?
	706	# if [ ${status} -gt 0 ]; then
	707	# sleep $DELAI
	708	# else
	709	# break
	710	# fi
	711	# (( i = i + 1 ))
	712	# done
[544]	713	# ============ CESIUM END ============ #
	714
	715	if [ $? -gt 0 ] ; then
	716	echo "IGCM_sys_QsubPost : erreur " $@
	717	IGCM_debug_Exit "IGCM_sys_QsubPost"
	718	fi
	719	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[99]	720	}
	721
	722	#D-*************************
[717]	723	#D- File transfer functions
[99]	724	#D-*************************
	725	#D-
	726
	727	#D-#==================================================
	728	#D-function IGCM_sys_Rsync_out
	729	#D-* Purpose: treat return val of rsync
	730	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
	731	#D- Error values and explanations can depend on your system version.
	732	function IGCM_sys_Rsync_out {
[719]	733	status=$1
	734	if [ ! $status ] ; then
[544]	735	echo "rsync error !"
	736	fi
[99]	737
[544]	738	if [ $MYLANG = "fr" ]; then
[719]	739	case $status in
[544]	740	0) return ;;
	741	1) echo "Erreur de rsync ; RERR_SYNTAX : "
	742	echo "Erreur de syntaxe ou d'utilisation."
	743	return;;
	744	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
	745	echo "Incompatibilité de protocole."
	746	return;;
	747	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
	748	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
	749	echo "répertoires"
	750	return;;
	751	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
	752	echo "Action demandée non supportée : une tentative de manipulation de"
	753	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
	754	echo "été faite ; ou une option qui est supportée par le client mais"
	755	echo "pas par le serveur a été spécifiée."
	756	return;;
	757	10) echo "Erreur de rsync ; RERR_SOCKETIO"
	758	echo "Erreur dans le socket d'entrée sortie"
	759	return;;
	760	11) echo "Erreur de rsync ; RERR_FILEIO"
	761	echo "Erreur d'entrée sortie fichier"
	762	return;;
	763	12) echo "Erreur de rsync ; RERR_STREAMIO"
	764	echo "Erreur dans flux de donnée du protocole rsync"
	765	return;;
	766	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
	767	echo "Erreur avec les diagnostics du programme"
	768	return;;
	769	14) echo "Erreur de rsync ; RERR_IPC"
	770	echo "Erreur dans le code IPC"
	771	return;;
	772	20) echo "Erreur de rsync ; RERR_SIGNAL"
	773	echo "SIGUSR1 ou SIGINT reçu"
	774	return;;
	775	21) echo "Erreur de rsync ; RERR_WAITCHILD"
	776	echo "Une erreur retournée par waitpid()"
	777	return;;
	778	22) echo "Erreur de rsync ; RERR_MALLOC"
	779	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
	780	return;;
	781	23) echo ""
	782	echo "Erreur fichier inexistant"
	783	return;;
	784	30) echo "Erreur de rsync ; RERR_TIMEOUT"
	785	echo "Temps d'attente écoulé dans l'envoi/réception de données"
	786	return;;
[719]	787	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
[544]	788	return;;
	789	esac
	790	elif [ $MYLANG = "en" ] ; then
[719]	791	case $status in
[717]	792	0) return;;
[544]	793	1) echo "rsync error : Syntax or usage error "
	794	return;;
	795	2) echo "rsync error : Protocol incompatibility "
	796	return;;
	797	3) echo "rsync error : Errors selecting input/output files, dirs"
	798	return;;
	799	4) echo "rsync error : Requested action not supported: an attempt"
	800	echo "was made to manipulate 64-bit files on a platform that cannot support"
	801	echo "them; or an option was specified that is supported by the client and"
	802	echo "not by the server."
	803	return;;
	804	5) echo "rsync error : Error starting client-server protocol"
	805	return;;
	806	10) echo "rsync error : Error in socket I/O "
	807	return;;
	808	11) echo "rsync error : Error in file I/O "
	809	return;;
	810	12) echo "rsync error : Error in rsync protocol data stream "
	811	return;;
	812	13) echo "rsync error : Errors with program diagnostics "
	813	return;;
	814	14) echo "rsync error : Error in IPC code "
	815	return;;
	816	20) echo "rsync error : Received SIGUSR1 or SIGINT "
	817	return;;
	818	21) echo "rsync error : Some error returned by waitpid() "
	819	return;;
	820	22) echo "rsync error : Error allocating core memory buffers "
	821	return;;
	822	23) echo "rsync error : Partial transfer due to error"
	823	return;;
	824	24) echo "rsync error : Partial transfer due to vanished source files"
	825	return;;
	826	30) echo "rsync error : Timeout in data send/receive "
	827	return;;
[719]	828	*) echo "rsync error : return code of rsync unknown :" $status
[544]	829	return;;
	830	esac
	831	else
	832	echo "unknown language $MYLANG."
	833	return
	834	fi
[99]	835	}
[192]	836
[99]	837	#D-#==================================================
[192]	838	#D-function IGCM_sys_Miror_libIGCM
	839	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
	840	#D-* Examples:
	841	#D-
	842	function IGCM_sys_Mirror_libIGCM {
[544]	843	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
	844	if ( $DEBUG_sys ) ; then
	845	echo "IGCM_sys_Mirror_libIGCM"
	846	fi
[192]	847
[717]	848	typeset status DEST
[192]	849
[544]	850	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
[192]	851
[544]	852	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
	853	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
[717]	854	status=$?
[192]	855
[717]	856	if [ ${status} -gt 0 ] ; then
[544]	857	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
	858	cat out_rsync
	859	fi
	860	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
[192]	861	}
	862
	863	#D-#==================================================
[99]	864	#D-function IGCM_sys_Cp
	865	#D-* Purpose: generic cp
	866	#D-* Examples:
	867	#D-
	868	function IGCM_sys_Cp {
[544]	869	IGCM_debug_PushStack "IGCM_sys_Cp" $@
	870	if ( $DEBUG_sys ) ; then
	871	echo "IGCM_sys_Cp :" $@
	872	fi
[99]	873
[717]	874	typeset status
[99]	875
[544]	876	echo cp $@ > out_rsync 2>&1
	877	\cp $@ >> out_rsync 2>&1
[717]	878	status=$?
[99]	879
[717]	880	if [ ${status} -gt 0 ] ; then
	881	echo "IGCM_sys_Cp : error code ${status}"
[544]	882	cat out_rsync
	883	IGCM_debug_Exit "IGCM_sys_Cp"
	884	else
	885	\rm out_rsync
	886	fi
	887	IGCM_debug_PopStack "IGCM_sys_Cp"
[99]	888	}
	889
	890	#D-#==================================================
	891	#D-function IGCM_sys_Rm
	892	#D-* Purpose: generic rm
	893	#D-* Examples:
	894	#D-
	895	function IGCM_sys_Rm {
[544]	896	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
	897	if ( $DEBUG_sys ) ; then
	898	echo "IGCM_sys_Rm :" $@
	899	fi
[99]	900
[717]	901	typeset status
[99]	902
[544]	903	echo rm $@ > out_rsync 2>&1
	904	\rm $@ >> out_rsync 2>&1
[717]	905	status=$?
[99]	906
[717]	907	if [ ${status} -gt 0 ] ; then
	908	echo "IGCM_sys_Rm : error code ${status}"
[544]	909	cat out_rsync
	910	IGCM_debug_Exit "IGCM_sys_Rm"
	911	else
	912	\rm out_rsync
	913	fi
	914	IGCM_debug_PopStack "IGCM_sys_Rm"
[99]	915	}
	916
	917	#D-#==================================================
	918	#D-function IGCM_sys_Mv
	919	#D-* Purpose: generic move
	920	#D-* Examples:
	921	#D-
	922	function IGCM_sys_Mv {
[544]	923	IGCM_debug_PushStack "IGCM_sys_Mv" $@
	924	if ( $DEBUG_sys ) ; then
	925	echo "IGCM_sys_Mv :" $@
	926	fi
[99]	927
[544]	928	if [ $DRYRUN = 0 ]; then
[99]	929
[717]	930	typeset status
	931
[544]	932	echo mv $@ > out_rsync 2>&1
	933	\mv $@ >> out_rsync 2>&1
[717]	934	status=$?
	935
	936	if [ ${status} -gt 0 ] ; then
	937	echo "IGCM_sys_Mv : error code ${status}"
[544]	938	cat out_rsync
	939	IGCM_debug_Exit "IGCM_sys_Mv"
[173]	940	else
[544]	941	\rm out_rsync
[99]	942	fi
[544]	943	else
	944	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	945	fi
[99]	946
[544]	947	IGCM_debug_PopStack "IGCM_sys_Mv"
[99]	948	}
	949
	950	#D-#==================================================
	951	#D-function IGCM_sys_Put_Dir
	952	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	953	#D-* Examples:
	954	#D-
	955	function IGCM_sys_Put_Dir {
[544]	956	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	957	if ( $DEBUG_sys ) ; then
	958	echo "IGCM_sys_Put_Dir :" $@
	959	fi
	960	if [ $DRYRUN = 0 ]; then
	961	if [ ! -d ${1} ] ; then
	962	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	963	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	964	return
[99]	965	fi
	966
[717]	967	typeset status
[99]	968
[544]	969	# Only if we use rsync
	970	#IGCM_sys_TestDirArchive $( dirname $2 )
	971	#
	972	#USUAL WAY
	973	\cp -r $1 $2 > out_rsync 2>&1
[717]	974	status=$?
[99]	975
[717]	976	if [ ${status} -gt 0 ] ; then
	977	echo "IGCM_sys_Put_Dir : error code ${status}"
[544]	978	cat out_rsync
	979	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[99]	980	else
[544]	981	\rm out_rsync
[99]	982	fi
[544]	983	else
	984	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	985	fi
	986	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[99]	987	}
	988
	989	#D-#==================================================
	990	#D-function IGCM_sys_Get_Dir
[544]	991	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[99]	992	#D-* Examples:
	993	#D-
	994	function IGCM_sys_Get_Dir {
[544]	995	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	996	if ( $DEBUG_sys ) ; then
	997	echo "IGCM_sys_Get_Dir :" $@
	998	fi
	999	if [ $DRYRUN = 0 ]; then
	1000	# if [ ! -d ${1} ] ; then
	1001	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
	1002	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
	1003	# return
	1004	# fi
[99]	1005
[717]	1006	typeset NB_ESSAI DELAI status i
	1007	# number of tentative
	1008	NB_ESSAI=3
	1009	# time delay between tentative
	1010	DELAI=2
[99]	1011
[544]	1012	# Only if we use rsync
	1013	#IGCM_sys_TestDirArchive $( dirname $2 )
	1014	#
	1015	# USUAL WAY
[717]	1016	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
	1017	ccc_hsm get -r $1
[99]	1018
[717]	1019	i=0
	1020	while [ $i -lt $NB_ESSAI ] ; do
	1021	\cp -ur $1 $2 >> out_rsync 2>&1
	1022	status=$?
	1023	if [ ${status} -gt 0 ]; then
	1024	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1025	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	1026	sleep $DELAI
	1027	else
	1028	break
	1029	fi
	1030	(( i = i + 1 ))
	1031	done
	1032
	1033	if [ ${status} -gt 0 ] ; then
[544]	1034	echo "IGCM_sys_Get_Dir : error."
	1035	cat out_rsync
	1036	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[99]	1037	else
[544]	1038	\rm out_rsync
[99]	1039	fi
[544]	1040	else
	1041	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1042	fi
	1043	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[99]	1044	}
	1045
[689]	1046	#====================================================
	1047	#- Call IGCM_sys_Mirror_libIGCM now !
	1048	if ( $MirrorlibIGCM ) ; then
	1049	IGCM_sys_Mirror_libIGCM
	1050	fi
	1051
[99]	1052	#D-#==================================================
	1053	#D-function IGCM_sys_Put_Rest
[544]	1054	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[99]	1055	#D- File and target directory must exist.
	1056	#D-* Examples:
	1057	#D-
	1058	function IGCM_sys_Put_Rest {
[544]	1059	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	1060	if ( $DEBUG_sys ) ; then
	1061	echo "IGCM_sys_Put_Rest :" $@
	1062	fi
	1063	if [ $DRYRUN = 0 ]; then
	1064	if [ ! -f ${1} ] ; then
	1065	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	1066	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[99]	1067	fi
	1068
[717]	1069	typeset status
[544]	1070	#
	1071	if [ X${JobType} = XRUN ] ; then
	1072	IGCM_sys_Chmod 444 ${1}
	1073	fi
[99]	1074
[544]	1075	#
	1076	# USUAL WAY
	1077	\cp $1 $2 > out_rsync 2>&1
[717]	1078	status=$?
[544]	1079
[717]	1080	# #RSYNC WITH NETWORK SSH CALL
	1081	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1082	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[99]	1083
[717]	1084	# #RSYNC WITH NFS USE
	1085	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1086	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
[99]	1087
[717]	1088	# status=$?
	1089	# IGCM_sys_Rsync_out $status
[99]	1090
[717]	1091	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1092	# (( status=status+$? ))
[99]	1093
[717]	1094	if [ ${status} -gt 0 ] ; then
	1095	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
[544]	1096	cat out_rsync
	1097	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[99]	1098	else
[544]	1099	\rm out_rsync
[99]	1100	fi
[544]	1101	else
	1102	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1103	fi
	1104	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[99]	1105	}
	1106
	1107	#D-#==================================================
[544]	1108	#D-function IGCM_sys_PutBuffer_Rest
	1109	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
	1110	#D- File and target directory must exist.
	1111	#D-* Examples:
	1112	#D-
	1113	function IGCM_sys_PutBuffer_Rest {
	1114	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
	1115	if ( $DEBUG_sys ) ; then
	1116	echo "IGCM_sys_PutBuffer_Rest :" $@
	1117	fi
	1118	if [ $DRYRUN = 0 ]; then
	1119	if [ ! -f ${1} ] ; then
	1120	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
	1121	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1122	fi
	1123
[717]	1124	typeset status
[544]	1125	#
	1126	if [ X${JobType} = XRUN ] ; then
	1127	IGCM_sys_Chmod 444 ${1}
	1128	fi
	1129
	1130	#
	1131	# USUAL WAY
	1132	\cp $1 $2 > out_rsync 2>&1
[717]	1133	status=$?
[544]	1134
[717]	1135	if [ ${status} -gt 0 ] ; then
	1136	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
[544]	1137	cat out_rsync
	1138	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1139	else
	1140	\rm out_rsync
	1141	fi
	1142	else
	1143	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1144	fi
	1145	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
	1146	}
	1147
	1148	#D-#==================================================
[99]	1149	#D-function IGCM_sys_Put_Out
[544]	1150	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[99]	1151	#D-* Examples:
	1152	#D-
	1153	function IGCM_sys_Put_Out {
[544]	1154	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	1155	if ( $DEBUG_sys ) ; then
	1156	echo "IGCM_sys_Put_Out :" $@
	1157	fi
[717]	1158
	1159	typeset NB_ESSAI DELAI status i exist skip
	1160
	1161	# number of tentative
	1162	NB_ESSAI=3
	1163	# time delay between tentative
	1164	DELAI=2
	1165
[544]	1166	if [ $DRYRUN = 0 ]; then
	1167	if [ ! -f ${1} ] ; then
	1168	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	1169	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1170	return 1
[99]	1171	fi
[544]	1172	#
	1173	IGCM_sys_MkdirArchive $( dirname $2 )
	1174	#
[99]	1175
[544]	1176	#=====================================================
	1177	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
	1178	#=====================================================
[99]	1179
[544]	1180	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
	1181	#if [ $? -eq 0 ] ; then
	1182	# typeset WORKPATH FILEPATH
	1183	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
[717]	1184	# IGCM_sys_MkdirWork ${WORKPATH}
[544]	1185	# FILEPATH=${WORKPATH}/$( basename $2 )
	1186	# #
	1187	# IGCM_sys_Cp ${1} ${FILEPATH}
	1188	#fi
[99]	1189
[544]	1190	if [ X${JobType} = XRUN ] ; then
	1191	if [ X${3} = X ] ; then
[717]	1192	IGCM_sys_Chmod 444 ${1}
[544]	1193	fi
	1194	fi
	1195	#
[99]	1196
[717]	1197	exist=false
	1198	skip=false
	1199	if [ -f $2 ] ; then
	1200	IGCM_debug_Print 1 "$2 already exist"
	1201	#dmget $2
	1202	ccc_hsm get $2
	1203	exist=true
	1204	if [ "X$( diff $1 $2 )" = X ] ; then
	1205	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	1206	skip=true
	1207	else
	1208	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	1209	skip=false
	1210	fi
	1211	fi
	1212	#
	1213	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	1214	IGCM_sys_Chmod u+w $2
	1215	fi
[99]	1216
[717]	1217	if [ X${skip} = Xfalse ] ; then
[719]	1218	i=0
[717]	1219	while [ $i -lt $NB_ESSAI ] ; do
	1220	# USUAL WAY
	1221	\cp $1 $2 > out_rsync 2>&1
	1222	status=$?
	1223	if [ ${status} -gt 0 ]; then
	1224	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1225	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
	1226	sleep $DELAI
	1227	else
	1228	break
	1229	fi
	1230	(( i = i + 1 ))
	1231	done
	1232	fi
[99]	1233
[717]	1234	# #RSYNC WITH NETWORK SSH CALL
	1235	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1236	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[99]	1237
[717]	1238	# #RSYNC WITH NFS USE
	1239	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1240	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
[99]	1241
[717]	1242	# status=$?
	1243	# IGCM_sys_Rsync_out $status
	1244
	1245	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1246	# (( status=status+$? ))
	1247
	1248	if [ ${status} -gt 0 ] ; then
[544]	1249	echo "IGCM_sys_Put_Out : error."
	1250	cat out_rsync
	1251	IGCM_debug_Exit "IGCM_sys_Put_Out"
[99]	1252	else
[544]	1253	\rm out_rsync
[99]	1254	fi
[544]	1255	else
	1256	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1257	fi
	1258	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1259	return 0
[99]	1260	}
	1261
	1262	#D-#==================================================
[544]	1263	#D-function IGCM_sys_PutBuffer_Out
	1264	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
	1265	#D-* Examples:
	1266	#D-
	1267	function IGCM_sys_PutBuffer_Out {
	1268	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
	1269	if ( $DEBUG_sys ) ; then
	1270	echo "IGCM_sys_PutBuffer_Out :" $@
	1271	fi
[717]	1272
	1273	typeset NB_ESSAI DELAI status i exist skip
	1274
	1275	# number of tentative
	1276	NB_ESSAI=3
	1277	# time delay between tentative
	1278	DELAI=2
	1279
[544]	1280	if [ $DRYRUN = 0 ]; then
	1281	if [ ! -f ${1} ] ; then
	1282	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
	1283	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1284	return 1
	1285	fi
	1286	#
	1287	IGCM_sys_Mkdir $( dirname $2 )
	1288	#
	1289
	1290	if [ X${JobType} = XRUN ] ; then
	1291	if [ X${3} = X ] ; then
[717]	1292	IGCM_sys_Chmod 444 ${1}
[544]	1293	fi
	1294	fi
[717]	1295
	1296	exist=false
	1297	skip=false
	1298	if [ -f $2 ] ; then
	1299	IGCM_debug_Print 1 "$2 already exist"
	1300	exist=true
	1301	if [ "X$( diff $1 $2 )" = X ] ; then
	1302	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	1303	skip=true
	1304	else
	1305	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	1306	skip=false
	1307	fi
	1308	fi
[544]	1309	#
[717]	1310	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	1311	IGCM_sys_Chmod u+w $2
	1312	fi
[544]	1313
[717]	1314	if [ X${skip} = Xfalse ] ; then
	1315	i=0
	1316	while [ $i -lt $NB_ESSAI ] ; do
	1317	# USUAL WAY
	1318	\cp $1 $2 > out_rsync 2>&1
	1319	status=$?
	1320	if [ ${status} -gt 0 ]; then
	1321	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1322	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
	1323	sleep $DELAI
	1324	else
	1325	break
	1326	fi
	1327	(( i = i + 1 ))
	1328	done
	1329	fi
	1330
	1331	if [ ${status} -gt 0 ] ; then
[544]	1332	echo "IGCM_sys_PutBuffer_Out : error."
	1333	cat out_rsync
	1334	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
	1335	else
	1336	\rm out_rsync
	1337	fi
	1338	else
	1339	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1340	fi
	1341	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1342	return 0
	1343	}
	1344
	1345	#D-#==================================================
[99]	1346	#D-function IGCM_sys_Get
	1347	#D-* Purpose: Get a file from ${ARCHIVE}
	1348	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	1349	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	1350	function IGCM_sys_Get {
[544]	1351	IGCM_debug_PushStack "IGCM_sys_Get" $@
[99]	1352
[719]	1353	typeset DEST dm_liste ifile target file_work
[717]	1354	typeset NB_ESSAI DELAI status i
[99]	1355
[544]	1356	if ( $DEBUG_sys ) ; then
	1357	echo "IGCM_sys_Get :" $@
	1358	fi
[717]	1359
	1360	# number of tentative
	1361	NB_ESSAI=3
	1362	# time delay between tentative
	1363	DELAI=2
	1364
[544]	1365	if [ $DRYRUN -le 2 ]; then
	1366	if [ X${1} = X'/l' ] ; then
	1367	# test if the first file is present in the old computation :
	1368	eval set +A dm_liste \${${2}}
	1369	else
	1370	eval set +A dm_liste ${1}
[99]	1371	fi
[544]	1372	eval DEST=\${${#}}
[99]	1373
[544]	1374	#=====================================================
	1375	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
	1376	#=====================================================
[99]	1377
[544]	1378	# Is it an R_OUT file (not R_IN) ?
	1379	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
	1380	#if [ $? -eq 0 ] ; then
	1381	# # Yes ? then we try to get it in SCRATCHDIR
	1382	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
	1383	# if [ -f ${file_work[0]} ] ; then
	1384	# IGCM_sys_Cp ${file_work[*]} ${DEST}
	1385	# IGCM_debug_PopStack "IGCM_sys_Get"
	1386	# return
	1387	# fi
	1388	#fi
[99]	1389
[587]	1390	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
[717]	1391	status=$?
	1392	if [ ${status} -gt 0 ] ; then
	1393	echo "WARNING IGCM_sys_Get : error code ${status}"
[544]	1394	cat out_rsync
	1395	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	1396	fi
[99]	1397
[717]	1398	#if [ ${status} -gt 0 ] ; then
[544]	1399	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
	1400	# cat out_rsync
	1401	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	1402	# sleep 30
	1403	# echo "We try another time"
[587]	1404	## dmget ${dm_liste[*]} > out_rsync 2>&1
	1405	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
[717]	1406	# status=$?
	1407	# if [ ${status} -gt 0 ] ; then
	1408	# echo "ERROR IGCM_sys_Get : again demigration error :"
	1409	# cat out_rsync
	1410	# IGCM_debug_Exit "IGCM_sys_Get"
[544]	1411	# fi
	1412	# else
	1413	# echo "ERROR IGCM_sys_Get : demigration error :"
	1414	# cat out_rsync
	1415	# IGCM_debug_Exit "IGCM_sys_Get"
	1416	# fi
	1417	#fi
[99]	1418
[719]	1419	# #RSYNC WITH NETWORK SSH CALL
	1420	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
	1421	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
	1422
	1423	# #RSYNC WITH NFS USE
	1424	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
	1425	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
	1426
	1427	# status=$?
	1428	# IGCM_sys_Rsync_out $status
	1429
	1430	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1431	# (( status=status+$? ))
	1432
[544]	1433	#USUAL WAY
	1434	if [ X${1} = X'/l' ] ; then
	1435	for target in ${dm_liste[*]} ; do
[717]	1436	local_file=$( basename ${target} )
	1437	# test if the target file is present before the loop
	1438	IGCM_sys_TestFileArchive ${target}
	1439	status=$?
	1440	if [ ${status} -gt 0 ] ; then
	1441	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
	1442	IGCM_debug_Exit "IGCM_sys_Get"
	1443	else
[719]	1444	i=0
[717]	1445	while [ $i -lt $NB_ESSAI ] ; do
	1446	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1447	status=$?
	1448	if [ ${status} -gt 0 ]; then
	1449	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1450	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	1451	sleep $DELAI
	1452	else
	1453	break
	1454	fi
	1455	(( i = i + 1 ))
	1456	done
[719]	1457	if [ ${status} -gt 0 ] ; then
	1458	echo "IGCM_sys_Get : error"
	1459	cat out_rsync
	1460	\rm out_rsync
	1461	IGCM_debug_Exit "IGCM_sys_Get"
	1462	else
	1463	\rm out_rsync
	1464	fi
[717]	1465	fi
[544]	1466	done
	1467	else
[719]	1468	i=0
	1469	while [ $i -lt $NB_ESSAI ] ; do
	1470	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
	1471	status=$?
	1472	if [ ${status} -gt 0 ]; then
	1473	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1474	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	1475	sleep $DELAI
	1476	else
	1477	break
	1478	fi
	1479	(( i = i + 1 ))
	1480	done
	1481	if [ ${status} -gt 0 ] ; then
	1482	echo "IGCM_sys_Get : error"
	1483	cat out_rsync
	1484	\rm out_rsync
	1485	IGCM_debug_Exit "IGCM_sys_Get"
	1486	else
	1487	\rm out_rsync
	1488	fi
[544]	1489	fi
	1490	else
	1491	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1492	fi
	1493	IGCM_debug_PopStack "IGCM_sys_Get"
[99]	1494	}
	1495
	1496	#D-#==================================================
[544]	1497	#D-function IGCM_sys_GetBuffer
	1498	#D-* Purpose: Get a file from ${SCRATCHDIR}
	1499	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
	1500	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
	1501	function IGCM_sys_GetBuffer {
	1502	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
	1503
[719]	1504	typeset DEST buf_liste ifile target file_work
	1505	typeset NB_ESSAI DELAI status i
[544]	1506
	1507	if ( $DEBUG_sys ) ; then
	1508	echo "IGCM_sys_GetBuffer :" $@
	1509	fi
[719]	1510
	1511	# number of tentative
	1512	NB_ESSAI=3
	1513	# time delay between tentative
	1514	DELAI=2
	1515
[544]	1516	if [ $DRYRUN -le 2 ]; then
	1517	if [ X${1} = X'/l' ] ; then
	1518	# test if the first file is present in the old computation :
	1519	eval set +A buf_liste \${${2}}
	1520	else
	1521	eval set +A buf_liste ${1}
	1522	fi
	1523	eval DEST=\${${#}}
	1524
	1525	#USUAL WAY
	1526	if [ X${1} = X'/l' ] ; then
	1527	for target in ${buf_liste[*]} ; do
[717]	1528	local_file=$( basename ${target} )
[719]	1529	i=0
	1530	while [ $i -lt $NB_ESSAI ] ; do
	1531	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1532	status=$?
	1533	if [ ${status} -gt 0 ]; then
	1534	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1535	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
	1536	sleep $DELAI
	1537	else
	1538	break
	1539	fi
	1540	(( i = i + 1 ))
	1541	done
	1542	if [ ${status} -gt 0 ] ; then
	1543	echo "IGCM_sys_Get : error"
	1544	cat out_rsync
	1545	\rm out_rsync
	1546	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1547	else
	1548	\rm out_rsync
	1549	fi
[544]	1550	done
	1551	else
[719]	1552	i=0
	1553	while [ $i -lt $NB_ESSAI ] ; do
	1554	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
	1555	status=$?
	1556	if [ ${status} -gt 0 ]; then
	1557	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
	1558	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
	1559	sleep $DELAI
	1560	else
	1561	break
	1562	fi
	1563	(( i = i + 1 ))
	1564	done
	1565	if [ ${status} -gt 0 ] ; then
	1566	echo "IGCM_sys_Get : error"
	1567	cat out_rsync
	1568	\rm out_rsync
	1569	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1570	else
	1571	\rm out_rsync
	1572	fi
[544]	1573	fi
	1574	else
	1575	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1576	fi
	1577	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
	1578	}
	1579
	1580	#D-#==================================================
[99]	1581	#D-function IGCM_sys_GetDate_FichWork
	1582	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
	1583	#D-* Examples:
	1584	#D-
	1585	function IGCM_sys_GetDate_FichWork {
[544]	1586	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
	1587	if ( $DEBUG_sys ) ; then
	1588	echo "IGCM_sys_GetDate_FichWork :" $@
	1589	fi
	1590	typeset dateF
[689]	1591	set +A dateF -- $( IGCM_sys_RshMaster "ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1}" )
[544]	1592	eval ${2}=${dateF[5]}
[540]	1593
[99]	1594	# donne la date filesys d'un fichier sur la machine work
[544]	1595	IGCM_debug_PopStack "IGCM_sys_FichWork"
[99]	1596	}
	1597
	1598	#D-#==================================================
	1599	#D-function IGCM_sys_GetDate_FichArchive
[717]	1600	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
[99]	1601	#D-* Examples:
	1602	#D-
	1603	function IGCM_sys_GetDate_FichArchive {
[544]	1604	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
	1605	if ( $DEBUG_sys ) ; then
	1606	echo "IGCM_sys_GetDate_FichArchive :" $@
	1607	fi
	1608	typeset dateF
[689]	1609	set +A dateF -- $( IGCM_sys_RshMaster "ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1}" )
[544]	1610	eval ${2}=${dateF[5]}
[540]	1611
[544]	1612	IGCM_debug_PopStack "IGCM_sys_FichArchive"
[99]	1613	}
	1614
	1615	##############################################################
	1616	# REBUILD OPERATOR
	1617
	1618	function IGCM_sys_rebuild {
[544]	1619	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
	1620	if ( $DEBUG_sys ) ; then
	1621	echo "IGCM_sys_rebuild :" $@
	1622	fi
[717]	1623
[734]	1624	typeset NB_ESSAI DELAI status i firstArg
[717]	1625	# number of tentative
	1626	NB_ESSAI=3
	1627	# time delay between tentative
	1628	DELAI=2
	1629
	1630	i=0
	1631	while [ $i -lt $NB_ESSAI ] ; do
	1632	/home/cont003/p86ipsl/SX8/bin/rebuild -f -o $@ > out_rsync 2>&1
	1633	status=$?
	1634	if [ ${status} -gt 0 ] ; then
	1635	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
	1636	cat out_rsync
	1637	\rm out_rsync
	1638	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[734]	1639	firstArg=${1}
	1640	\rm ${firstArg}
[733]	1641	sleep $DELAI
[717]	1642	else
	1643	\rm out_rsync
	1644	break
	1645	fi
	1646	(( i = i + 1 ))
	1647	done
	1648
[725]	1649	if [ ${status} -gt 0 ] ; then
[717]	1650	echo "IGCM_sys_rebuild : rebuild error"
[544]	1651	IGCM_debug_Exit "rebuild"
	1652	fi
[99]	1653
[544]	1654	IGCM_debug_PopStack "IGCM_sys_rebuild"
[99]	1655	}
	1656
[697]	1657	function IGCM_sys_rebuild_station {
	1658	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
	1659	typeset i list_opt file_in file_out prefix_invert list_invert
	1660	if ( $DEBUG_sys ) ; then
	1661	echo "IGCM_sys_rebuild_station :" $@
	1662	fi
	1663	list_opt=$@
	1664
	1665	# Invert Axis : t,x -> x,t
	1666	# t,pres,x -> x,t,pres
	1667	# So that we can concatenate along x
	1668	i=0
	1669	for file_in in ${list_opt} ; do
	1670	(( i = i + 1))
	1671	[ ${i} = 1 ] && file_out=${file_in} && continue
	1672	prefix_invert=$( basename ${file_in} .nc )
	1673	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
	1674	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
	1675	done
	1676
	1677	# Concatenate
	1678	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
	1679
	1680	# Re-ivert file
	1681	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
	1682
	1683	# Station re-ordering is too expansive to be run within libICGM
	1684	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
	1685	# This re-ordering must be done "in memory" by the cmorization process
	1686	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
	1687	# BEGIN reordering
	1688
	1689	# Only LMDZ text output contains the exact ordering of the station.
	1690	# We isolate this in the code below:
	1691	# 0 38 -157.5000000000000 70.98591549295774
	1692	# 0 54 27.49999999999999 67.18309859154928
	1693	# 0 56 -62.50000000000001 82.39436619718309
	1694	# 0 79 12.49999999999999 78.59154929577466
	1695	# 0 116 -165.0000000000000 76.05633802816901
	1696	# 0 117 130.0000000000000 70.98591549295774
	1697	# 0 118 110.0000000000000 87.46478873239437
	1698	# 1 40 4.999999999999995 51.97183098591550
	1699	# typeset iStation iProc list_opt file_in file_out prefix_invert
	1700	# typeset -Z4 j4
	1701	# typeset -Z3 j3
	1702
	1703	# unset list_opt
	1704	# set +A list_opt $@
	1705
	1706	# Filename after rebuild
	1707	# file_out=${list_opt[0]}
	1708	# Prefix of output files
	1709	# prefix_invert=$( basename ${file_out} .nc )
	1710	# Number of procs
	1711	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
	1712
	1713	# iProc=0
	1714	# while [ ${iProc} -lt ${num_proc} ] ; do
	1715	# Array containing Station as a number
	1716	# unset proc_stn
	1717	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| awk ' {print $2}' )
	1718	# Number of stations produced by processor proc
	1719	# stationLast=${#proc_stn[*]}
	1720	# Proc number on 4 digits
	1721	# j4=${iProc}
	1722	# Init
	1723	# iStation=0
	1724	# while [ ${iStation} -lt ${stationLast} ] ; do
	1725	# Station number on 3 digits
	1726	# j3=${proc_stn[${iStation}]}
	1727	# Extract station
	1728	# Invert Axis : t,x -> x,t
	1729	# t,pres,x -> x,t,pres
	1730	# So that we can concatenate along x
	1731	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
	1732	# (( iStation = iStation + 1 ))
	1733	# done
	1734	# (( iProc = iProc + 1 ))
	1735	# done
	1736
	1737	# Concatenate all station along x
	1738	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
	1739
	1740	# Re-invert file
	1741	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
	1742
	1743	# END reordering
	1744
	1745	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
	1746	}
	1747
[99]	1748	############################################################
	1749	# Activate Running Environnment Variables
	1750
	1751	function IGCM_sys_activ_variables {
[544]	1752	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1753	if ( $DEBUG_sys ) ; then
	1754	echo "IGCM_sys_activ_variables"
	1755	fi
[99]	1756
	1757	# --------------------------------------------------------------------
	1758	#D- MPI specifications
	1759	# --------------------------------------------------------------------
	1760
	1761	#D-- MPISUSPEND
[544]	1762	export MPISUSPEND=${MPISUSPEND:=OFF}
[136]	1763
	1764	#D-- MPIPROGINF #other choices : ALL_DETAIL2
[544]	1765	export MPIPROGINF=ALL
[136]	1766	#D- activate ftrace (with -ftrace)
[544]	1767	export F_FTRACE=YES
[136]	1768	#D- communication information (with -ftrace)
[544]	1769	export MPICOMMINF=DETAIL
[99]	1770
	1771	# --------------------------------------------------------------------
	1772	#D- Other specifications
	1773	# --------------------------------------------------------------------
	1774
	1775	#D- max number of character/line in output job
[544]	1776	export F_SYSLEN=5000
[99]	1777	#D- number of error that can be admitted on the NEC
[544]	1778	export F_ERRCNT=0
[99]	1779	#D- global performance
[544]	1780	export F_PROGINF=DETAIL
[136]	1781
[99]	1782	#D- I/O performance (FORTRAN I/O only not netCDF)
[544]	1783	export F_FILEINF=${F_FILEINF:=NO}
[136]	1784	#D- netCDF I/O performance
[544]	1785	export NC_FILEINF=${NC_FILEINF:=NO}
[99]	1786
[544]	1787	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[99]	1788	}
	1789
	1790	############################################################
	1791	# Desactivate Running Environnment Variables
	1792
	1793	function IGCM_sys_desactiv_variables {
[544]	1794	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1795	if ( $DEBUG_sys ) ; then
	1796	echo "IGCM_sys_desactiv_variables"
	1797	fi
[99]	1798	# --------------------------------------------------------------------
	1799	#D- MPI specifications
	1800	# --------------------------------------------------------------------
	1801
	1802	#D-- MPIPROGINF
[544]	1803	export MPIPROGINF=NO
[99]	1804
	1805	# --------------------------------------------------------------------
	1806	#D- Other specifications
	1807	# --------------------------------------------------------------------
	1808
	1809	#D- global performance
[544]	1810	export F_PROGINF=NO
[99]	1811
[544]	1812	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[99]	1813	}
	1814
	1815	############################################################
[621]	1816	# Build MPI/OMP scripts
[717]	1817	function IGCM_sys_build_execution_scripts
[621]	1818	{
[664]	1819	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1820	if ( $DEBUG_sys ) ; then
	1821	echo "IGCM_sys_build_execution_scripts " $@
	1822	fi
[99]	1823
[664]	1824	typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
	1825
	1826	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
	1827	IGCM_debug_Exit "IGCM_sys_mercurex9 build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
	1828	fi
[621]	1829
[664]	1830	if ( ${OK_PARA_MPMD} ) ; then
[621]	1831
[664]	1832	if [ -f run_file ] ; then
	1833	IGCM_sys_Rm -f run_file
	1834	fi
	1835	touch run_file
[621]	1836
[664]	1837	if ( ${OK_PARA_OMP} ) ; then
	1838	IGCM_debug_Print 2 "Error in config.card on SX9 : OpenMP not available"
	1839	exit 1
	1840	else
[621]	1841	# OLD :
	1842	#PBS -v BATCH_NUM_PROC_TOT=4
	1843	#PBS -l cpunum_job=${BATCH_NUM_PROC_TOT}
[664]	1844	echo "Job_${config_UserChoices_JobName} includes BATCH_NUM_PROC_TOT = \c"
	1845	cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} \| grep -- "-v BATCH_NUM_PROC_TOT" \| sed -e "s/.BATCH_NUM_PROC_TOT = //"
	1846	NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} \| grep -- "-v BATCH_NUM_PROC_TOT" \| sed -e "s/.BATCH_NUM_PROC_TOT = //" )
	1847	NbProc_Job=${NbProc_Job:=0}
	1848	if [ ${NbProc_Job} -eq 0 ] ; then
	1849	IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no BATCH_NUM_PROC_TOT defined with MPI only run."
	1850	exit 1
	1851	fi
	1852	fi
[621]	1853
	1854	# run_file construction
	1855
	1856	# Then first loop on the components for the coupler ie oasis
	1857
	1858	### the coupler ie oasis must be the first one
	1859	for comp in ${config_ListOfComponents[*]} ; do
[664]	1860
	1861	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1862	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1863
	1864	# for CPL component only
	1865	if [ "X${comp}" = "XCPL" ] ; then
	1866
	1867	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1868	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[621]	1869
[664]	1870	if ( ${OK_PARA_MPI} ) ; then
	1871	echo "-p ${comp_proc_mpi_loc} -e ./${ExeNameOut}" >> run_file
[621]	1872	fi
[664]	1873	fi
[621]	1874	done
	1875
	1876	# Then second loop on the components
	1877
	1878	for comp in ${config_ListOfComponents[*]} ; do
[664]	1879
	1880	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1881	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1882
	1883	# Only if we really have an executable for the component and not the coupler ie oasis:
	1884	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
	1885
	1886	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1887	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[621]	1888
[664]	1889	if ( ${OK_PARA_MPI} ) ; then
	1890	echo "-p ${comp_proc_mpi_loc} -e ./${ExeNameOut}" >> run_file
[621]	1891	fi
[664]	1892	fi
[621]	1893	done
	1894
	1895	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
	1896
	1897	IGCM_sys_Chmod u+x run_file
	1898	if ( $DEBUG_sys ) ; then
	1899	echo "run_file contains : "
	1900	cat run_file
	1901	fi
	1902
[664]	1903	else # Only one executable. launch it.
[621]	1904
	1905	for comp in ${config_ListOfComponents[*]} ; do
	1906
[664]	1907	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1908	eval ExeNameOut=\${config_Executable_${comp}[1]}
[621]	1909
[664]	1910	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
	1911	EXECUTION="time mpirun -np ${BATCH_NUM_PROC_TOT} ./${ExeNameOut}"
	1912	fi
[621]	1913	done
	1914
[664]	1915	fi
[621]	1916
[664]	1917	IGCM_debug_Print 1 "sys mercurex9 : execution command is "
	1918	IGCM_debug_Print 1 "$EXECUTION"
[621]	1919
[664]	1920	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[621]	1921
	1922	}
	1923
[99]	1924	function IGCM_sys_build_run_file {
	1925
[621]	1926	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
	1927
[99]	1928	}
[651]	1929
	1930	############################################################
	1931	# Check of space available on temporary filesytems
	1932	function IGCM_sys_check_quota {
	1933	IGCM_debug_PushStack "IGCM_sys_check_quota"
[664]	1934
[651]	1935	if ( $DEBUG_sys ) ; then
	1936	echo "IGCM_sys_check_quota "
	1937	fi
	1938	# Limit of quota (in %)
[705]	1939	limit_quota=95
[651]	1940
[652]	1941	# Check of the volume (use of RshMaster waiting for ccc_quota command available on SX9 nodes)
[674]	1942	# volume_quota=$( IGCM_sys_RshMaster "/applications/ccc-tools/ccc_quota \| grep ' scratch' \| awk '{print \$2}'" )
	1943	# volume_avail=$( IGCM_sys_RshMaster "/applications/ccc-tools/ccc_quota \| grep ' scratch' \| awk '{print \$3}'" )
[651]	1944
[674]	1945	volume_quota=$(quota -h \| grep 'scratch' \| awk '{print $2}')
	1946	volume_avail=$(quota -h \| grep 'scratch' \| awk '{print $3}')
[652]	1947
[651]	1948	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	1949
[652]	1950	unit_quota=$( echo $volume_quota \| awk '{print(substr($0, length($0), length($0)))}' )
	1951	unit_avail=$( echo $volume_avail \| awk '{print(substr($0, length($0), length($0)))}' )
[651]	1952
[696]	1953	if [ "${unit_quota}" = "*" ] ; then
	1954	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1955	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1956	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1957	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1958	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1959	IGCM_debug_Verif_Exit
	1960	fi
	1961
[652]	1962	temp_avail=${volume_avail%%${unit_avail}*}
	1963	temp_quota=${volume_quota%%${unit_quota}*}
[651]	1964
	1965	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1966
	1967	# Convertion
[652]	1968	if [ ${unit_avail} = "T" ] ; then
[651]	1969	(( temp_avail = temp_avail * 1000000000000 ))
[652]	1970	elif [ ${unit_avail} = "G" ] ; then
[651]	1971	(( temp_avail = temp_avail * 1000000000 ))
[652]	1972	elif [ ${unit_avail} = "M" ] ; then
[651]	1973	(( temp_avail = temp_avail * 1000000 ))
[652]	1974	elif [ ${unit_avail} = "k" ] ; then
[651]	1975	(( temp_avail = temp_avail * 1000 ))
	1976	else
	1977	(( temp_avail = volume_avail ))
	1978	fi
[652]	1979	if [ ${unit_quota} = "T" ] ; then
[651]	1980	(( temp_quota = temp_quota * 1000000000000 ))
[652]	1981	elif [ ${unit_quota} = "G" ] ; then
[651]	1982	(( temp_quota = temp_quota * 1000000000 ))
[652]	1983	elif [ ${unit_quota} = "M" ] ; then
[651]	1984	(( temp_quota = temp_quota * 1000000 ))
[652]	1985	elif [ ${unit_quota} = "k" ] ; then
[651]	1986	(( temp_quota = temp_quota * 1000 ))
	1987	else
	1988	(( temp_quota = volume_quota ))
	1989	fi
	1990	fi
	1991
	1992	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1993	# echo "volume ratio is " $quota_volume
	1994
	1995	if [ ${quota_volume} -ge ${limit_quota} ] ; then
[652]	1996	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
[654]	1997	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	1998	IGCM_debug_Print 1 "Use the ccc_quota command to check"
[651]	1999	IGCM_debug_Print 1 "You must have more than 10% available to run"
	2000	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	2001	IGCM_debug_Verif_Exit
	2002	fi
	2003
	2004	fi
	2005	IGCM_debug_PopStack "IGCM_sys_check_quota"
	2006	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: