Context Navigation

source: branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh @ 571

Last change on this file since 571 was 571, checked in by mafoipsl, 12 years ago
First try to merge libIGCM_MPI_OpenMP branch with libIGCM trunk revision 569. Tested on vargas with PackFrequency?=NONE included into config.card.
Property svn:keywords set to `Revision Author Date`
File size: 59.3 KB

Rev	Line
[222]	1	#!/bin/ksh
	2
	3	#**************************************************************
[373]	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
[222]	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
	16	# if you use the prefix #D- for comments to be extracted.
	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
	21	#D-LibIGCM_sys for Titane
	22	#D-#==================================================
	23	#D-
	24	#D- This ksh library if a layer under some usefull
	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
	45	# \| \| Cp/Exe param files \| \| Chmod \| \|
	46	# \| \| Qsub \| \| \| \|
	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[245]	64	# Host and user names
	65	# $hostname ou hostname
	66	typeset HOST=${HOST:=$( hostname )}
[222]	67	# $username ou whoami
[245]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[222]	69	# $hostname of the MASTER job
	70	typeset MASTER=titane
	71
	72	#D-
	73	#D-#==================================================
	74	#D-Program used in libIGCM
	75	#D-#==================================================
	76
	77	# rsync with path
	78	typeset -r RSYNC=/usr/bin/rsync
	79	# RSYNC_opt args to rsync
	80	typeset -r RSYNC_opt="-va"
	81	# ie storage filesystem
[571]	82	typeset -r STOREHOST=${MASTER}
[222]	83
	84	#====================================================
[451]	85	# Source default environment
	86	#====================================================
	87	. /etc/profile
	88
	89	#====================================================
[305]	90	# Set environment tools (ferret, nco, cdo)
	91	#====================================================
[451]	92	. /home/cont003/p86ipsl/.atlas_env_titane_ksh
[222]	93
[571]	94	module switch nco/4.0.8 nco/3.9.4_netcdf4
	95
[222]	96	#====================================================
	97	# Specific for ocean additionnal diagnostic
[451]	98	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	99	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
[222]	100
	101	#====================================================
	102	# Host specific DIRECTORIES
	103	#====================================================
	104
	105	#====================================================
	106	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	107	typeset -r R_EXE="${MODIPSL}/bin"
	108
	109	#====================================================
	110	#- SUBMIT_DIR : submission dir
[534]	111	typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
[222]	112
	113	#====================================================
	114	#- ARCHIVE
	115	typeset -r ARCHIVE=${DMFDIR}
	116
[571]	117	# ============ CESIUM START ============ #
	118
[222]	119	#====================================================
	120	#- Mirror libIGCM from titane to cesium if needed
[571]	121	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	122	#if [ ! ${ROOTSYS} = "home" ] ; then
	123	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	124	#else
	125	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	126	#fi
[222]	127
	128	#====================================================
	129	#- libIGCM_POST
[571]	130	#if ( ${MirrorlibIGCM} ) ; then
	131	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	132	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	133	#else
	134	# typeset -r libIGCM_POST=${libIGCM}
	135	#fi
[222]	136
[571]	137	# ============ CESIUM END ============ #
	138
[222]	139	#====================================================
[571]	140	#- MirrorlibIGCM for frontend
	141	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	142
	143	#====================================================
	144	#- libIGCM_POST for frontend
	145	typeset -r libIGCM_POST=${libIGCM}
	146
	147	#====================================================
[222]	148	#- IN
[571]	149	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
[438]	150	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
[222]	151
	152	#====================================================
[571]	153	#- R_OUT
[222]	154	typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
	155
	156	#====================================================
[571]	157	#- R_BUF (ONLY FOR double copy an scratch)
	158	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
[222]	159
	160	#====================================================
[571]	161	#- BIG_DIR : BIG_DIR to store files waiting for rebuild
	162	typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
	163
	164	#====================================================
[222]	165	#- OUT_POST
	166	typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
	167
	168	#====================================================
	169	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[459]	170	if [ ! X${LSB_QUEUE} = Xmono ] ; then
[571]	171	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
[459]	172	else
[571]	173	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
[459]	174	fi
[222]	175
	176	#====================================================
	177	#- HOST_MPIRUN_COMMAND
	178	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
	179
	180	#====================================================
	181	#- Max number of arguments passed to nco operator or demigration command
	182	UNIX_MAX_LIMIT=360
	183
[490]	184	NUM_COREPERNODE=8
	185
[570]	186	#====================================================
	187	#- Default number of MPI task for IPSL coupled model
	188	#- required for backward compatibility
	189	#-
	190	DEFAULT_NUM_PROC_OCE=5
	191	DEFAULT_NUM_PROC_CPL=1
	192	DEFAULT_NUM_PROC_ATM=26
	193	DEFAULT_NUM_PROC_TOTAL=32
[490]	194
[222]	195	#D-#==================================================
	196	#D-function IGCM_sys_RshMaster
[571]	197	#D-* Purpose: Connection to frontend machine.
[222]	198	#D-* Examples:
	199	#D-
	200	function IGCM_sys_RshMaster {
[571]	201	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
	202	/bin/ksh <<-EOF
[222]	203	export libIGCM=${libIGCM}
	204	export DEBUG_debug=${DEBUG_debug}
	205	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
	206	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
	207	${@}
	208	EOF
[571]	209	if [ $? -gt 0 ] ; then
	210	echo "IGCM_sys_RshMaster : erreur."
	211	IGCM_debug_Exit "IGCM_sys_RshMaster"
	212	fi
	213	IGCM_debug_PopStack "IGCM_sys_RshMaster"
[222]	214	}
	215
	216	#D-#==================================================
	217	#D-function IGCM_sys_RshArchive
	218	#D-* Purpose: Archive rsh command
	219	#D-* Examples:
	220	#D-
	221	function IGCM_sys_RshArchive {
[571]	222	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	223	/bin/ksh <<-EOF
[222]	224	${@}
	225	EOF
[571]	226	if [ $? -gt 0 ] ; then
	227	echo "IGCM_sys_RshArchive : erreur."
	228	IGCM_debug_Exit "IGCM_sys_RshArchive"
	229	fi
	230	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[222]	231	}
	232
	233	#D-#==================================================
	234	#D-function IGCM_sys_RshPost
	235	#D-* Purpose: Post-process rsh command
	236	#D-* Examples:
	237	#D-
	238	function IGCM_sys_RshPost {
[571]	239	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
	240	if ( $DEBUG_sys ) ; then
	241	echo "IGCM_sys_RshPost :" $@
	242	fi
[324]	243
[571]	244	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
	245	#cat tmp_IGCM_sys_RshPost_$$
	246	# keep standard input (stdin) for the loop onto temporary file
	247	cat >tmp_IGCM_sys_RshPost_$$
[222]	248
[571]	249	# ============ FRONTEND START ============ #
[451]	250
[571]	251	/bin/ksh <tmp_IGCM_sys_RshPost_$$
	252	if [ $? -gt 0 ] ; then
	253	echo "IGCM_sys_RshPost : erreur."
	254	IGCM_debug_Exit "IGCM_sys_RshPost"
	255	fi
	256	# delete temporary file
	257	\rm tmp_IGCM_sys_RshPost_$$
[451]	258
[571]	259	# ============ FRONTEND END ============ #
	260
	261	# ============ CESIUM START ============ #
	262	# typeset NB_ESSAI DELAI status i
	263	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
	264	# #little hack so that rebuild submission is done on titane not an cesium
	265	#
	266	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
	267	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
	268	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
	269	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
	270	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
	271	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
	272	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
	273	#
	274	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
	275	# cat tmp_IGCM_sys_RshPost_$$
	276	#
	277	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
	278	# if [ $? -gt 0 ] ; then
	279	# echo "IGCM_sys_RshPost : erreur."
	280	# IGCM_debug_Exit "IGCM_sys_RshPost"
	281	# fi
	282	# # delete temporary file
	283	# \rm tmp_IGCM_sys_RshPost_$$
	284	#
	285	# else
	286	# # number of tentative
	287	# NB_ESSAI=10
	288	# # time delay between tentative
	289	# DELAI=10
	290	# i=0
	291	# while [ $i -ne $NB_ESSAI ] ; do
	292	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
	293	# status=$?
	294	# if [ ${status} -ne 0 ];
	295	# then
	296	# sleep $DELAI
	297	# else
	298	# break
	299	# fi
	300	# let i=$i+1
	301	# done
	302	# # delete temporary file
	303	# \rm tmp_IGCM_sys_RshPost_$$
	304	#
	305	# if [ ${status} -gt 0 ] ; then
	306	# echo "IGCM_sys_RshPost : erreur."
	307	# IGCM_debug_Exit "IGCM_sys_RshPost"
	308	# fi
	309	# fi
	310
	311	# ============ CESIUM END ============ #
	312
	313	IGCM_debug_PopStack "IGCM_sys_RshPost"
[222]	314	}
	315
	316	#D-#==================================================
	317	#D-function IGCM_sys_SendMail
	318	#D-* Purpose: Send mail when simulation is over
	319	#D-* Examples:
	320	#D-
	321	function IGCM_sys_SendMail {
[571]	322	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
	323	if ( $DEBUG_sys ) ; then
	324	echo "IGCM_sys_SendMail :" $@
	325	fi
[222]	326
[571]	327	if ( ${ExitFlag} ) ; then
	328	status=failed
	329	else
	330	status=completed
	331	fi
[452]	332
[222]	333	cat << END_MAIL > job_end.mail
	334	Dear ${LOGIN},
	335
[452]	336	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
[222]	337	Job started : ${DateBegin}
	338	Job ended : ${DateEnd}
[434]	339	Output files are available in ${R_SAVE}
[452]	340	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
[222]	341	END_MAIL
	342
	343	if [ ! -z ${config_UserChoices_MailName} ] ; then
[571]	344	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
[222]	345	elif [ -f ~/.forward ] ; then
[571]	346	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
[222]	347	fi
	348	sleep 10
	349	rm -f job_end.mail
	350
[571]	351	if [ $? -gt 0 ] ; then
	352	echo "IGCM_sys_SendMail : erreur."
	353	IGCM_debug_Exit "IGCM_sys_SendMail"
	354	fi
	355	IGCM_debug_PopStack "IGCM_sys_SendMail"
[222]	356	}
	357
	358	#D-#==================================================
	359	#D-function IGCM_sys_Mkdir
	360	#D-* Purpose: Master locale mkdir command
	361	#D-* Examples:
	362	#D-
	363	function IGCM_sys_Mkdir {
[571]	364	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
	365	if ( $DEBUG_sys ) ; then
	366	echo "IGCM_sys_Mkdir :" $@
	367	fi
	368	if [ ! -d ${1} ]; then
	369	\mkdir -p $1
	370	if [ $? -gt 0 ] ; then
	371	echo "IGCM_sys_Mkdir : erreur."
	372	IGCM_debug_Exit "IGCM_sys_Mkdir"
[222]	373	fi
[571]	374	fi
	375	# vérification :
	376	if [ ! -d ${1} ] ; then
	377	echo "IGCM_sys_Mkdir : erreur."
	378	IGCM_debug_Exit "IGCM_sys_Mkdir"
	379	fi
	380	IGCM_debug_PopStack "IGCM_sys_Mkdir"
[222]	381	}
	382
	383	#D-#==================================================
	384	#D-function IGCM_sys_MkdirArchive
	385	#D-* Purpose: Mkdir on Archive
	386	#D-* Examples:
	387	#D-
	388	function IGCM_sys_MkdirArchive {
[571]	389	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	390	if ( $DEBUG_sys ) ; then
	391	echo "IGCM_sys_MkdirArchive :" $@
	392	fi
	393	#- creation de repertoire sur le serveur fichier
	394	if [ ! -d ${1} ]; then
	395	\mkdir -p $1
	396	if [ $? -gt 0 ] ; then
	397	echo "IGCM_sys_MkdirArchive : erreur."
	398	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[222]	399	fi
[571]	400	fi
	401	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[222]	402	}
	403
	404	#D-#==================================================
	405	#D-function IGCM_sys_MkdirWork
	406	#D-* Purpose: Mkdir on Work
	407	#D-* Examples:
	408	#D-
	409	function IGCM_sys_MkdirWork {
[571]	410	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
	411	if ( $DEBUG_sys ) ; then
	412	echo "IGCM_sys_MkdirWork :" $@
	413	fi
	414	#- creation de repertoire sur le serveur fichier
	415	if [ ! -d ${1} ]; then
	416	\mkdir -p $1
	417	if [ $? -gt 0 ] ; then
	418	echo "IGCM_sys_MkdirWork : erreur."
	419	IGCM_debug_Exit "IGCM_sys_MkdirWork"
[222]	420	fi
[571]	421	fi
	422	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
[222]	423	}
	424
	425	#D-#==================================================
	426	#D-function IGCM_sys_Cd
	427	#D-* Purpose: master cd command
	428	#D-* Examples:
	429	#D-
	430	function IGCM_sys_Cd {
[571]	431	IGCM_debug_PushStack "IGCM_sys_Cd" $@
	432	if ( $DEBUG_sys ) ; then
	433	echo "IGCM_sys_Cd :" $@
	434	fi
	435	\cd $1
	436	if [ $? -gt 0 ] ; then
	437	echo "IGCM_sys_Cd : erreur."
	438	IGCM_debug_Exit "IGCM_sys_Cd"
	439	fi
	440	IGCM_debug_PopStack "IGCM_sys_Cd"
[222]	441	}
	442
	443	#D-#==================================================
	444	#D-function IGCM_sys_Chmod
	445	#D-* Purpose: Chmod
	446	#D-* Examples:
	447	#D-
	448	function IGCM_sys_Chmod {
[571]	449	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
	450	if ( $DEBUG_sys ) ; then
	451	echo "IGCM_sys_Chmod :" $@
	452	fi
	453	if [ $DRYRUN -le 1 ]; then
	454	\chmod $@
	455	if [ $? -gt 0 ] ; then
	456	echo "IGCM_sys_Chmod : erreur."
	457	IGCM_debug_Exit "IGCM_sys_Chmod"
[222]	458	fi
[571]	459	else
	460	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	461	fi
	462	IGCM_debug_PopStack "IGCM_sys_Chmod"
[222]	463	}
	464
	465	#D-#==================================================
	466	#D-function IGCM_sys_FileSize
	467	#D-* Purpose: Filesize
	468	#D-* Examples:
	469	#D-
	470	function IGCM_sys_FileSize {
[571]	471	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
[222]	472
[571]	473	typeset sizeF
	474	set +A sizeF -- $( ls -la ${1} )
	475	if [ $? -gt 0 ] ; then
	476	IGCM_debug_Exit "IGCM_sys_FileSize"
	477	fi
	478	eval ${2}=${sizeF[4]}
[222]	479
[571]	480	IGCM_debug_PopStack "IGCM_sys_FileSize"
[222]	481	}
	482
	483	#D-#==================================================
	484	#D-function IGCM_sys_TestDir
	485	#D-* Purpose: Test Directory that must exists
	486	#D-* Examples:
	487	#D-
	488	function IGCM_sys_TestDir {
[571]	489	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
	490	if ( $DEBUG_sys ) ; then
	491	echo "IGCM_sys_TestDir :" $@
	492	fi
	493	typeset ExistFlag
	494	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	495	IGCM_debug_PopStack "IGCM_sys_TestDir"
[222]	496
[571]	497	return ${ExistFlag}
[222]	498	}
	499
	500	#D-#==================================================
	501	#D-function IGCM_sys_TestDirArchive
	502	#D-* Purpose: Test Directory that must exists on Archive
	503	#D-* Examples:
	504	#D-
	505	function IGCM_sys_TestDirArchive {
[571]	506	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	507	if ( $DEBUG_sys ) ; then
	508	echo "IGCM_sys_TestDirArchive :" $@
	509	fi
	510	typeset ExistFlag
	511	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	512	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
[222]	513
[571]	514	return ${ExistFlag}
[222]	515	}
	516
	517	#D-#==================================================
[571]	518	#D-function IGCM_sys_IsFileArchived
	519	#D-* Purpose: Test file that must NOT EXISTS on Archive
	520	#D-* Examples:
	521	#D-
	522	function IGCM_sys_IsFileArchived {
	523	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	524	if ( $DEBUG_sys ) ; then
	525	echo "IGCM_sys_IsFileArchived :" $@
	526	fi
	527	typeset IsArchivedFlag
	528	IsArchivedFlag=$( [ X$( echo $1 \| grep ^\/dmnfs ) != X ] && echo 0 \|\| echo 1 )
	529	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	530
	531	return ${IsArchivedFlag}
	532	}
	533
	534	#D-#==================================================
[222]	535	#D-function IGCM_sys_TestFileArchive
	536	#D-* Purpose: Test file that must NOT EXISTS on Archive
	537	#D-* Examples:
	538	#D-
	539	function IGCM_sys_TestFileArchive {
[571]	540	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	541	if ( $DEBUG_sys ) ; then
	542	echo "IGCM_sys_TestFileArchive :" $@
	543	fi
	544	typeset ExistFlag
	545	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	546	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[222]	547
[571]	548	return ${ExistFlag}
[222]	549	}
	550
	551	#D-#==================================================
[571]	552	#D-function IGCM_sys_TestFileBuffer
	553	#D-* Purpose: Test file that must NOT EXISTS on Buffer
	554	#D-* Examples:
	555	#D-
	556	function IGCM_sys_TestFileBuffer {
	557	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
	558	typeset ExistFlag
	559	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	560	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
	561
	562	return ${ExistFlag}
	563	}
	564
	565	#D-#==================================================
[222]	566	#D-function IGCM_sys_CountFileArchive
	567	#D-* Purpose: Count files on Archive filesystem
	568	#D-* Examples:
	569	#D-
	570	function IGCM_sys_CountFileArchive {
[571]	571	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	572	ls ${@} 2>/dev/null \| wc -l
	573	if [ $? -gt 0 ] ; then
	574	echo "IGCM_sys_CountFileArchive : erreur."
	575	fi
	576	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[222]	577	}
	578
	579	#D-#==================================================
[571]	580	#D-function IGCM_sys_CountFileBuffer
	581	#D-* Purpose: Count files on Scratch filesystem
	582	#D-* Examples:
	583	#D-
	584	function IGCM_sys_CountFileBuffer {
	585	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
	586	ls ${@} 2>/dev/null \| wc -l
	587	if [ $? -gt 0 ] ; then
	588	echo "IGCM_sys_CountFileBuffer : erreur."
	589	fi
	590	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
	591	}
	592
	593	#D-#==================================================
[222]	594	#D-function IGCM_sys_Tree
	595	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	596	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	597	#D-
	598	function IGCM_sys_Tree {
[571]	599	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	600	if ( $DEBUG_sys ) ; then
	601	echo "IGCM_sys_Tree :" $@
	602	fi
[222]	603
[571]	604	\ls -lR ${@}
[222]	605
[571]	606	IGCM_debug_PopStack "IGCM_sys_Tree"
[222]	607	}
	608
	609	#D-#==================================================
	610	#D-function IGCM_sys_Tar
[571]	611	#D-* Purpose: master tar command
[222]	612	#D-* Examples:
	613	#D-
	614	function IGCM_sys_Tar {
[571]	615	IGCM_debug_PushStack "IGCM_sys_Tar" $@
	616	if ( $DEBUG_sys ) ; then
	617	echo "IGCM_sys_Tar :" $@
	618	fi
	619	\tar cf $@
	620	if [ $? -gt 0 ] ; then
	621	echo "IGCM_sys_Tar : erreur."
	622	IGCM_debug_Exit "IGCM_sys_Tar"
	623	fi
	624	IGCM_debug_PopStack "IGCM_sys_Tar"
[222]	625	}
	626
	627	#D-#==================================================
	628	#D-function IGCM_sys_UnTar
	629	#D-* Purpose: master un-tar command
	630	#D-* Examples:
	631	#D-
	632	function IGCM_sys_UnTar {
[571]	633	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
	634	if ( $DEBUG_sys ) ; then
	635	echo "IGCM_sys_UnTar :" $@
	636	fi
	637	\tar xvf $1
	638	if [ $? -gt 0 ] ; then
	639	echo "IGCM_sys_UnTar : erreur."
	640	IGCM_debug_Exit "IGCM_sys_UnTar"
	641	fi
	642	IGCM_debug_PopStack "IGCM_sys_UnTar"
[222]	643	}
	644
	645	#D-#==================================================
	646	#D-function IGCM_sys_Qsub
	647	#D-* Purpose: Qsub new job
	648	#D-* Examples:
	649	#D-
	650	function IGCM_sys_Qsub {
[571]	651	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	652	if ( $DEBUG_sys ) ; then
	653	echo "IGCM_sys_Qsub :" $@
	654	fi
	655	typeset options
	656	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
	657	options=" -o ${Script_Output} -e ${Script_Output}"
	658	else
	659	options=" -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
	660	fi
[467]	661
[571]	662	/usr/local/bin/ccc_msub ${options} < $1
	663	if [ $? -gt 0 ] ; then
	664	echo "IGCM_sys_Qsub : erreur ${options} $1"
	665	IGCM_debug_Exit "IGCM_sys_Qsub"
	666	fi
	667	IGCM_debug_PopStack "IGCM_sys_Qsub"
[222]	668	}
	669
	670	#D-#==================================================
	671	#D-function IGCM_sys_QsubPost
	672	#D-* Purpose: Qsub new job on scalaire
	673	#D-* Examples:
	674	#D-
	675	function IGCM_sys_QsubPost {
[571]	676	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	677	if ( $DEBUG_sys ) ; then
	678	echo "IGCM_sys_QsubPost :" $@
	679	fi
	680
	681	# Need to export listVarEnv for cesium
	682	# NO Need to export listVarEnv for titane
	683	# ============ FRONTEND START ============ #
	684
	685	/usr/local/bin/ccc_msub -p ${BRIDGE_MSUB_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
	686
	687	# ============ FRONTEND END ============ #
	688
	689	# ============ CESIUM START ============ #
	690	# typeset NB_ESSAI DELAI status i
	691	# # number of tentative
	692	# NB_ESSAI=10
	693	# # time delay between tentative
	694	# DELAI=10
	695	# (( i = 0 ))
	696	# while [ $i -lt $NB_ESSAI ] ; do
	697	# /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
	698	# status=$?
	699	# if [ ${status} -gt 0 ]; then
	700	# sleep $DELAI
	701	# else
	702	# break
	703	# fi
	704	# (( i = i + 1 ))
	705	# done
	706	# ============ CESIUM END ============ #
	707
	708	if [ $? -gt 0 ] ; then
	709	echo "IGCM_sys_QsubPost : erreur " $@
	710	IGCM_debug_Exit "IGCM_sys_QsubPost"
	711	fi
	712	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[222]	713	}
	714
	715	#D-*************************
	716	#D- File transfer functions
	717	#D-*************************
	718	#D-
	719
	720	#D-#==================================================
	721	#D-function IGCM_sys_Rsync_out
	722	#D-* Purpose: treat return val of rsync
	723	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
	724	#D- Error values and explanations can depend on your system version.
	725	function IGCM_sys_Rsync_out {
[571]	726	RET=$1
	727	if [ ! $RET ] ; then
	728	echo "rsync error !"
	729	fi
[222]	730
[571]	731	if [ $MYLANG = "fr" ]; then
	732	case $RET in
	733	0) return ;;
	734	1) echo "Erreur de rsync ; RERR_SYNTAX : "
	735	echo "Erreur de syntaxe ou d'utilisation."
	736	return;;
	737	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
	738	echo "Incompatibilité de protocole."
	739	return;;
	740	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
	741	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
	742	echo "répertoires"
	743	return;;
	744	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
	745	echo "Action demandée non supportée : une tentative de manipulation de"
	746	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
	747	echo "été faite ; ou une option qui est supportée par le client mais"
	748	echo "pas par le serveur a été spécifiée."
	749	return;;
	750	10) echo "Erreur de rsync ; RERR_SOCKETIO"
	751	echo "Erreur dans le socket d'entrée sortie"
	752	return;;
	753	11) echo "Erreur de rsync ; RERR_FILEIO"
	754	echo "Erreur d'entrée sortie fichier"
	755	return;;
	756	12) echo "Erreur de rsync ; RERR_STREAMIO"
	757	echo "Erreur dans flux de donnée du protocole rsync"
	758	return;;
	759	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
	760	echo "Erreur avec les diagnostics du programme"
	761	return;;
	762	14) echo "Erreur de rsync ; RERR_IPC"
	763	echo "Erreur dans le code IPC"
	764	return;;
	765	20) echo "Erreur de rsync ; RERR_SIGNAL"
	766	echo "SIGUSR1 ou SIGINT reçu"
	767	return;;
	768	21) echo "Erreur de rsync ; RERR_WAITCHILD"
	769	echo "Une erreur retournée par waitpid()"
	770	return;;
	771	22) echo "Erreur de rsync ; RERR_MALLOC"
	772	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
	773	return;;
	774	23) echo ""
	775	echo "Erreur fichier inexistant"
	776	return;;
	777	30) echo "Erreur de rsync ; RERR_TIMEOUT"
	778	echo "Temps d'attente écoulé dans l'envoi/réception de données"
	779	return;;
	780	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
	781	return;;
	782	esac
	783	elif [ $MYLANG = "en" ] ; then
	784	case $RET in
	785	0) return;;
	786	1) echo "rsync error : Syntax or usage error "
	787	return;;
	788	2) echo "rsync error : Protocol incompatibility "
	789	return;;
	790	3) echo "rsync error : Errors selecting input/output files, dirs"
	791	return;;
	792	4) echo "rsync error : Requested action not supported: an attempt"
	793	echo "was made to manipulate 64-bit files on a platform that cannot support"
	794	echo "them; or an option was specified that is supported by the client and"
	795	echo "not by the server."
	796	return;;
	797	5) echo "rsync error : Error starting client-server protocol"
	798	return;;
	799	10) echo "rsync error : Error in socket I/O "
	800	return;;
	801	11) echo "rsync error : Error in file I/O "
	802	return;;
	803	12) echo "rsync error : Error in rsync protocol data stream "
	804	return;;
	805	13) echo "rsync error : Errors with program diagnostics "
	806	return;;
	807	14) echo "rsync error : Error in IPC code "
	808	return;;
	809	20) echo "rsync error : Received SIGUSR1 or SIGINT "
	810	return;;
	811	21) echo "rsync error : Some error returned by waitpid() "
	812	return;;
	813	22) echo "rsync error : Error allocating core memory buffers "
	814	return;;
	815	23) echo "rsync error : Partial transfer due to error"
	816	return;;
	817	24) echo "rsync error : Partial transfer due to vanished source files"
	818	return;;
	819	30) echo "rsync error : Timeout in data send/receive "
	820	return;;
	821	*) echo "rsync error : return code of rsync unknown :" $RET
	822	return;;
	823	esac
	824	else
	825	echo "unknown language $MYLANG."
	826	return
	827	fi
[222]	828	}
	829
	830	#D-#==================================================
	831	#D-function IGCM_sys_Miror_libIGCM
	832	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
	833	#D-* Examples:
	834	#D-
	835	function IGCM_sys_Mirror_libIGCM {
[571]	836	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
	837	if ( $DEBUG_sys ) ; then
	838	echo "IGCM_sys_Mirror_libIGCM"
	839	fi
[222]	840
[571]	841	typeset RET DEST
[222]	842
[571]	843	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
[222]	844
[571]	845	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
	846	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
	847	RET=$?
[222]	848
[571]	849	if [ ${RET} -gt 0 ] ; then
	850	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
	851	cat out_rsync
	852	fi
	853	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
[222]	854	}
	855
	856	#====================================================
	857	#- Call IGCM_sys_Mirror_libIGCM now !
	858	if ( $MirrorlibIGCM ) ; then
[571]	859	IGCM_sys_Mirror_libIGCM
[222]	860	fi
	861
	862	#D-#==================================================
	863	#D-function IGCM_sys_Cp
	864	#D-* Purpose: generic cp
	865	#D-* Examples:
	866	#D-
	867	function IGCM_sys_Cp {
[571]	868	IGCM_debug_PushStack "IGCM_sys_Cp" $@
	869	if ( $DEBUG_sys ) ; then
	870	echo "IGCM_sys_Cp :" $@
	871	fi
[222]	872
[571]	873	typeset RET
[222]	874
[571]	875	echo cp $@ > out_rsync 2>&1
	876	\cp $@ >> out_rsync 2>&1
	877	RET=$?
[222]	878
[571]	879	if [ ${RET} -gt 0 ] ; then
	880	echo "IGCM_sys_Cp : error."
	881	cat out_rsync
	882	IGCM_debug_Exit "IGCM_sys_Cp"
	883	else
	884	\rm out_rsync
	885	fi
	886	IGCM_debug_PopStack "IGCM_sys_Cp"
[222]	887	}
	888
	889	#D-#==================================================
	890	#D-function IGCM_sys_Rm
	891	#D-* Purpose: generic rm
	892	#D-* Examples:
	893	#D-
	894	function IGCM_sys_Rm {
[571]	895	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
	896	if ( $DEBUG_sys ) ; then
	897	echo "IGCM_sys_Rm :" $@
	898	fi
[222]	899
[571]	900	typeset RET
[222]	901
[571]	902	echo rm $@ > out_rsync 2>&1
	903	\rm $@ >> out_rsync 2>&1
	904	RET=$?
[222]	905
[571]	906	if [ ${RET} -gt 0 ] ; then
	907	echo "IGCM_sys_Rm : error."
	908	cat out_rsync
	909	IGCM_debug_Exit "IGCM_sys_Rm"
	910	else
	911	\rm out_rsync
	912	fi
	913	IGCM_debug_PopStack "IGCM_sys_Rm"
[222]	914	}
	915
	916	#D-#==================================================
[467]	917	#D-function IGCM_sys_RmRunDir
	918	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	919	#D- scheduler will do the job)
	920	#D-* Examples:
	921	#D-
	922	function IGCM_sys_RmRunDir {
[571]	923	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
	924	if ( $DEBUG_sys ) ; then
	925	echo "IGCM_sys_RmRunDir :" $@
	926	fi
[467]	927
[571]	928	typeset RET
[467]	929
[571]	930	echo rm $@ > out_rsync 2>&1
	931	\rm $@ >> out_rsync 2>&1
	932	RET=$?
[467]	933
[571]	934	if [ ${RET} -gt 0 ] ; then
	935	echo "IGCM_sys_RmRunDir : error."
	936	cat out_rsync
	937	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	938	else
	939	\rm out_rsync
	940	fi
	941	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[467]	942	}
	943
	944	#D-#==================================================
[222]	945	#D-function IGCM_sys_Mv
	946	#D-* Purpose: generic move
	947	#D-* Examples:
	948	#D-
	949	function IGCM_sys_Mv {
[571]	950	IGCM_debug_PushStack "IGCM_sys_Mv" $@
	951	if ( $DEBUG_sys ) ; then
	952	echo "IGCM_sys_Mv :" $@
	953	fi
[222]	954
[571]	955	if [ $DRYRUN = 0 ]; then
[222]	956
[571]	957	typeset RET
[222]	958
[571]	959	echo mv $@ > out_rsync 2>&1
	960	\mv $@ >> out_rsync 2>&1
	961	RET=$?
	962
	963	if [ ${RET} -gt 0 ] ; then
	964	echo "IGCM_sys_Mv : error in mv."
	965	cat out_rsync
	966	IGCM_debug_Exit "IGCM_sys_Mv"
[222]	967	else
[571]	968	\rm out_rsync
[222]	969	fi
[571]	970	else
	971	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	972	fi
[222]	973
[571]	974	IGCM_debug_PopStack "IGCM_sys_Mv"
[222]	975	}
	976
	977	#D-#==================================================
	978	#D-function IGCM_sys_Put_Dir
	979	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	980	#D-* Examples:
	981	#D-
	982	function IGCM_sys_Put_Dir {
[571]	983	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	984	if ( $DEBUG_sys ) ; then
	985	echo "IGCM_sys_Put_Dir :" $@
	986	fi
	987	if [ $DRYRUN = 0 ]; then
	988	if [ ! -d ${1} ] ; then
	989	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	990	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	991	return
[222]	992	fi
	993
[571]	994	typeset RET
[222]	995
[571]	996	# Only if we use rsync
	997	#IGCM_sys_TestDirArchive $( dirname $2 )
	998	#
	999	#USUAL WAY
	1000	\cp -r $1 $2 > out_rsync 2>&1
	1001	RET=$?
[222]	1002
[571]	1003	if [ ${RET} -gt 0 ] ; then
	1004	echo "IGCM_sys_Put_Dir : error."
	1005	cat out_rsync
	1006	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[222]	1007	else
[571]	1008	\rm out_rsync
[222]	1009	fi
[571]	1010	else
	1011	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1012	fi
	1013	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[222]	1014	}
	1015
	1016	#D-#==================================================
	1017	#D-function IGCM_sys_Get_Dir
[571]	1018	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[222]	1019	#D-* Examples:
	1020	#D-
	1021	function IGCM_sys_Get_Dir {
[571]	1022	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	1023	if ( $DEBUG_sys ) ; then
	1024	echo "IGCM_sys_Get_Dir :" $@
	1025	fi
	1026	if [ $DRYRUN = 0 ]; then
	1027	# if [ ! -d ${1} ] ; then
	1028	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
	1029	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
	1030	# return
	1031	# fi
[222]	1032
[571]	1033	typeset RET
[222]	1034
[571]	1035	# Only if we use rsync
	1036	#IGCM_sys_TestDirArchive $( dirname $2 )
	1037	#
	1038	# USUAL WAY
	1039	# add dmfind/dmget (to demigrate all offline files) :
	1040	dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget
	1041	\cp -r $1 $2 > out_rsync 2>&1
	1042	RET=$?
[222]	1043
[571]	1044	if [ ${RET} -gt 0 ] ; then
	1045	echo "IGCM_sys_Get_Dir : error."
	1046	cat out_rsync
	1047	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[222]	1048	else
[571]	1049	\rm out_rsync
[222]	1050	fi
[571]	1051	else
	1052	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1053	fi
	1054	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[222]	1055	}
	1056
	1057	#D-#==================================================
	1058	#D-function IGCM_sys_Get_Master
	1059	#D-* Purpose: Copy a complete directory from MASTER filesystem
	1060	#D-* Examples:
	1061	#D-
	1062	function IGCM_sys_Get_Master {
[571]	1063	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
	1064	if ( $DEBUG_sys ) ; then
	1065	echo "IGCM_sys_Get_Master :" $@
	1066	fi
	1067	if [ $DRYRUN = 0 ]; then
	1068	if [ ! -d ${1} ] ; then
	1069	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
	1070	IGCM_debug_PopStack "IGCM_sys_Get_Master"
	1071	return
[222]	1072	fi
	1073
[571]	1074	typeset RET
[222]	1075
[571]	1076	#USUAL WAY
	1077	\cp -r $1 $2 > out_rsync 2>&1
	1078	RET=$?
[222]	1079
[571]	1080	if [ ${RET} -gt 0 ] ; then
	1081	echo "IGCM_sys_Get_Master : error."
	1082	cat out_rsync
	1083	IGCM_debug_Exit "IGCM_sys_Get_Master"
[222]	1084	else
[571]	1085	\rm out_rsync
[222]	1086	fi
[571]	1087	else
	1088	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1089	fi
	1090	IGCM_debug_PopStack "IGCM_sys_Get_Master"
[222]	1091	}
	1092
	1093	#D-#==================================================
	1094	#D-function IGCM_sys_Put_Rest
[571]	1095	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[222]	1096	#D- File and target directory must exist.
	1097	#D-* Examples:
	1098	#D-
	1099	function IGCM_sys_Put_Rest {
[571]	1100	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	1101	if ( $DEBUG_sys ) ; then
	1102	echo "IGCM_sys_Put_Rest :" $@
	1103	fi
	1104	if [ $DRYRUN = 0 ]; then
	1105	if [ ! -f ${1} ] ; then
	1106	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	1107	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[222]	1108	fi
	1109
[571]	1110	typeset RET
	1111	#
	1112	if [ X${JobType} = XRUN ] ; then
	1113	IGCM_sys_Chmod 444 ${1}
	1114	fi
[222]	1115
[571]	1116	#
	1117	# USUAL WAY
	1118	\cp $1 $2 > out_rsync 2>&1
	1119	RET=$?
	1120
[245]	1121	# #RSYNC WITH NETWORK SSH CALL
[571]	1122	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1123	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[222]	1124
	1125	# #RSYNC WITH NFS USE
	1126	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1127	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
	1128
	1129	# RET=$?
	1130	# IGCM_sys_Rsync_out $RET
	1131
	1132	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1133	# (( RET=RET+$? ))
	1134
[571]	1135	if [ ${RET} -gt 0 ] ; then
	1136	echo "IGCM_sys_Put_Rest : error."
	1137	cat out_rsync
	1138	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[222]	1139	else
[571]	1140	\rm out_rsync
[222]	1141	fi
[571]	1142	else
	1143	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1144	fi
	1145	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[222]	1146	}
	1147
	1148	#D-#==================================================
[571]	1149	#D-function IGCM_sys_PutBuffer_Rest
	1150	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
	1151	#D- File and target directory must exist.
	1152	#D-* Examples:
	1153	#D-
	1154	function IGCM_sys_PutBuffer_Rest {
	1155	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
	1156	if ( $DEBUG_sys ) ; then
	1157	echo "IGCM_sys_PutBuffer_Rest :" $@
	1158	fi
	1159	if [ $DRYRUN = 0 ]; then
	1160	if [ ! -f ${1} ] ; then
	1161	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
	1162	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1163	fi
	1164
	1165	typeset RET
	1166	#
	1167	if [ X${JobType} = XRUN ] ; then
	1168	IGCM_sys_Chmod 444 ${1}
	1169	fi
	1170
	1171	#
	1172	# USUAL WAY
	1173	\cp $1 $2 > out_rsync 2>&1
	1174	RET=$?
	1175
	1176	if [ ${RET} -gt 0 ] ; then
	1177	echo "IGCM_sys_PutBuffer_Rest : error."
	1178	cat out_rsync
	1179	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
	1180	else
	1181	\rm out_rsync
	1182	fi
	1183	else
	1184	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1185	fi
	1186	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
	1187	}
	1188
	1189	#D-#==================================================
[222]	1190	#D-function IGCM_sys_Put_Out
[571]	1191	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[222]	1192	#D-* Examples:
	1193	#D-
	1194	function IGCM_sys_Put_Out {
[571]	1195	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	1196	if ( $DEBUG_sys ) ; then
	1197	echo "IGCM_sys_Put_Out :" $@
	1198	fi
	1199	if [ $DRYRUN = 0 ]; then
	1200	if [ ! -f ${1} ] ; then
	1201	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	1202	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1203	return 1
[222]	1204	fi
[571]	1205	#
	1206	IGCM_sys_MkdirArchive $( dirname $2 )
	1207	#
	1208	typeset RET
[222]	1209
[571]	1210	#=====================================================
	1211	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
	1212	#=====================================================
[222]	1213
[571]	1214	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
	1215	#if [ $? -eq 0 ] ; then
	1216	# typeset WORKPATH FILEPATH
	1217	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
	1218	# IGCM_sys_MkdirWork ${WORKPATH}
	1219	# FILEPATH=${WORKPATH}/$( basename $2 )
	1220	# #
	1221	# IGCM_sys_Cp ${1} ${FILEPATH}
	1222	#fi
[222]	1223
[571]	1224	if [ X${JobType} = XRUN ] ; then
	1225	if [ X${3} = X ] ; then
	1226	IGCM_sys_Chmod 444 ${1}
	1227	fi
	1228	fi
	1229	#
	1230	# USUAL WAY
	1231	\cp $1 $2 > out_rsync 2>&1
	1232	RET=$?
[222]	1233
[245]	1234	# #RSYNC WITH NETWORK SSH CALL
[571]	1235	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
	1236	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
[222]	1237
	1238	# #RSYNC WITH NFS USE
	1239	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
	1240	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
	1241
	1242	# RET=$?
	1243	# IGCM_sys_Rsync_out $RET
	1244
	1245	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1246	# (( RET=RET+$? ))
	1247
[571]	1248	if [ ${RET} -gt 0 ] ; then
	1249	echo "IGCM_sys_Put_Out : error."
	1250	cat out_rsync
	1251	IGCM_debug_Exit "IGCM_sys_Put_Out"
[222]	1252	else
[571]	1253	\rm out_rsync
[222]	1254	fi
[571]	1255	else
	1256	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1257	fi
	1258	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	1259	return 0
[222]	1260	}
	1261
	1262	#D-#==================================================
[571]	1263	#D-function IGCM_sys_PutBuffer_Out
	1264	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
	1265	#D-* Examples:
	1266	#D-
	1267	function IGCM_sys_PutBuffer_Out {
	1268	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
	1269	if ( $DEBUG_sys ) ; then
	1270	echo "IGCM_sys_PutBuffer_Out :" $@
	1271	fi
	1272	if [ $DRYRUN = 0 ]; then
	1273	if [ ! -f ${1} ] ; then
	1274	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
	1275	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1276	return 1
	1277	fi
	1278	#
	1279	IGCM_sys_Mkdir $( dirname $2 )
	1280	#
	1281	typeset RET
	1282
	1283	if [ X${JobType} = XRUN ] ; then
	1284	if [ X${3} = X ] ; then
	1285	IGCM_sys_Chmod 444 ${1}
	1286	fi
	1287	fi
	1288	#
	1289	# USUAL WAY
	1290	\cp $1 $2 > out_rsync 2>&1
	1291	RET=$?
	1292
	1293	if [ ${RET} -gt 0 ] ; then
	1294	echo "IGCM_sys_PutBuffer_Out : error."
	1295	cat out_rsync
	1296	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
	1297	else
	1298	\rm out_rsync
	1299	fi
	1300	else
	1301	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1302	fi
	1303	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
	1304	return 0
	1305	}
	1306
	1307	#D-#==================================================
[222]	1308	#D-function IGCM_sys_Get
	1309	#D-* Purpose: Get a file from ${ARCHIVE}
	1310	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	1311	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	1312	function IGCM_sys_Get {
[571]	1313	IGCM_debug_PushStack "IGCM_sys_Get" $@
[222]	1314
[571]	1315	typeset DEST RET dm_liste ifile target file_work
[222]	1316
[571]	1317	if ( $DEBUG_sys ) ; then
	1318	echo "IGCM_sys_Get :" $@
	1319	fi
	1320	if [ $DRYRUN -le 2 ]; then
	1321	if [ X${1} = X'/l' ] ; then
	1322	# test if the first file is present in the old computation :
	1323	eval set +A dm_liste \${${2}}
	1324	else
	1325	eval set +A dm_liste ${1}
[222]	1326	fi
[571]	1327	eval DEST=\${${#}}
[222]	1328
[571]	1329	#=====================================================
	1330	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
	1331	#=====================================================
[222]	1332
[571]	1333	# Is it an R_OUT file (not R_IN) ?
	1334	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
	1335	#if [ $? -eq 0 ] ; then
	1336	# # Yes ? then we try to get it in SCRATCHDIR
	1337	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
	1338	# if [ -f ${file_work[0]} ] ; then
	1339	# IGCM_sys_Cp ${file_work[*]} ${DEST}
	1340	# IGCM_debug_PopStack "IGCM_sys_Get"
	1341	# return
	1342	# fi
	1343	#fi
[222]	1344
[571]	1345	# test if the (first) file is present in the old computation :
	1346	IGCM_sys_TestFileArchive ${dm_liste[0]}
	1347	RET=$?
	1348	if [ ${RET} -gt 0 ] ; then
	1349	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
	1350	IGCM_debug_Exit "IGCM_sys_Get"
	1351	fi
[222]	1352
[571]	1353	dmget ${dm_liste[*]} > out_rsync 2>&1
	1354	RET=$?
	1355	if [ ${RET} -gt 0 ] ; then
	1356	echo "WARNING IGCM_sys_Get : demigration error."
	1357	cat out_rsync
	1358	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	1359	fi
[222]	1360
[571]	1361	#if [ ${RET} -gt 0 ] ; then
	1362	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
	1363	# cat out_rsync
	1364	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	1365	# sleep 30
	1366	# echo "We try another time"
	1367	# dmget ${dm_liste[*]} > out_rsync 2>&1
	1368	# RET=$?
	1369	# if [ ${RET} -gt 0 ] ; then
	1370	# echo "ERROR IGCM_sys_Get : again demigration error :"
	1371	# cat out_rsync
	1372	# IGCM_debug_Exit "IGCM_sys_Get"
	1373	# fi
	1374	# else
	1375	# echo "ERROR IGCM_sys_Get : demigration error :"
	1376	# cat out_rsync
	1377	# IGCM_debug_Exit "IGCM_sys_Get"
	1378	# fi
	1379	#fi
[521]	1380
[571]	1381	#USUAL WAY
	1382	if [ X${1} = X'/l' ] ; then
	1383	(( RET=0 ))
	1384	for target in ${dm_liste[*]} ; do
	1385	local_file=$( basename ${target} )
	1386	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1387	(( RET = RET + $? ))
	1388	done
	1389	else
	1390	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
	1391	RET=$?
	1392	fi
[521]	1393
[245]	1394	# #RSYNC WITH NETWORK SSH CALL
[571]	1395	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
	1396	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
[222]	1397
	1398	# #RSYNC WITH NFS USE
	1399	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
	1400	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
	1401
	1402	# RET=$?
	1403	# IGCM_sys_Rsync_out $RET
	1404
	1405	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
	1406	# (( RET=RET+$? ))
	1407
[571]	1408	if [ ${RET} -gt 0 ] ; then
	1409	echo "IGCM_sys_Get : copy error."
	1410	cat out_rsync
	1411	IGCM_debug_Exit "IGCM_sys_Get"
[222]	1412	else
[571]	1413	\rm out_rsync
[222]	1414	fi
[571]	1415	else
	1416	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1417	fi
	1418	IGCM_debug_PopStack "IGCM_sys_Get"
[222]	1419	}
	1420
[571]	1421	#D-#==================================================
	1422	#D-function IGCM_sys_GetBuffer
	1423	#D-* Purpose: Get a file from ${SCRATCHDIR}
	1424	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
	1425	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
	1426	function IGCM_sys_GetBuffer {
	1427	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
[222]	1428
[571]	1429	typeset DEST RET buf_liste ifile target file_work
	1430
	1431	if ( $DEBUG_sys ) ; then
	1432	echo "IGCM_sys_GetBuffer :" $@
	1433	fi
	1434	if [ $DRYRUN -le 2 ]; then
	1435	if [ X${1} = X'/l' ] ; then
	1436	# test if the first file is present in the old computation :
	1437	eval set +A buf_liste \${${2}}
	1438	else
	1439	eval set +A buf_liste ${1}
	1440	fi
	1441	eval DEST=\${${#}}
	1442
	1443	#USUAL WAY
	1444	if [ X${1} = X'/l' ] ; then
	1445	(( RET=0 ))
	1446	for target in ${buf_liste[*]} ; do
	1447	local_file=$( basename ${target} )
	1448	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
	1449	(( RET = RET + $? ))
	1450	done
	1451	else
	1452	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
	1453	RET=$?
	1454	fi
	1455
	1456	if [ ${RET} -gt 0 ] ; then
	1457	echo "IGCM_sys_GetBuffer : copy error."
	1458	cat out_rsync
	1459	IGCM_debug_Exit "IGCM_sys_GetBuffer"
	1460	else
	1461	\rm out_rsync
	1462	fi
	1463	else
	1464	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1465	fi
	1466	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
	1467	}
	1468
[222]	1469	#D-#==================================================
	1470	#D-function IGCM_sys_GetDate_FichWork
	1471	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
	1472	#D-* Examples:
	1473	#D-
	1474	function IGCM_sys_GetDate_FichWork {
[571]	1475	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
	1476	if ( $DEBUG_sys ) ; then
	1477	echo "IGCM_sys_GetDate_FichWork :" $@
	1478	fi
	1479	typeset dateF
	1480	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
	1481	eval ${2}=${dateF[5]}
	1482
[222]	1483	# donne la date filesys d'un fichier sur la machine work
[571]	1484	IGCM_debug_PopStack "IGCM_sys_FichWork"
[222]	1485	}
	1486
	1487	#D-#==================================================
	1488	#D-function IGCM_sys_GetDate_FichArchive
	1489	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
	1490	#D-* Examples:
	1491	#D-
	1492	function IGCM_sys_GetDate_FichArchive {
[571]	1493	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
	1494	if ( $DEBUG_sys ) ; then
	1495	echo "IGCM_sys_GetDate_FichArchive :" $@
	1496	fi
	1497	typeset dateF
	1498	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
	1499	eval ${2}=${dateF[5]}
	1500
	1501	IGCM_debug_PopStack "IGCM_sys_FichArchive"
	1502	}
	1503
	1504	#D-#==================================================
	1505	#D-function IGCM_sys_Dods_Rm
	1506	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	1507	#D-* Examples:
	1508	#D-
	1509	function IGCM_sys_Dods_Rm {
	1510	if ( $DEBUG_sys ) ; then
	1511	echo "IGCM_sys_Dods_Rm :" $@
	1512	fi
	1513	typeset RET
	1514	RET=0
	1515	if [ $DRYRUN = 0 ]; then
	1516	if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	1517	echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	1518	echo "Nothing has been done."
	1519	return
[222]	1520	fi
[571]	1521	dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
	1522	RET=$?
	1523
	1524	# if [ ${RET} -gt 0 ] ; then
	1525	# echo "IGCM_sys_Dods_Rm : error."
	1526	# cat out_dods_rm
	1527	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	1528	# else
	1529	# rm out_dods_rm
	1530	# fi
	1531
	1532	else
	1533	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1534	fi
	1535	return $RET
[222]	1536	}
	1537
[571]	1538	#D-#==================================================
	1539	#D-function IGCM_sys_Dods_Cp
	1540	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	1541	#D-* Examples:
	1542	#D-
	1543	function IGCM_sys_Dods_Cp {
	1544	if ( $DEBUG_sys ) ; then
	1545	echo "IGCM_sys_Dods_Cp :" $@
	1546	fi
	1547	typeset RET
	1548	RET=0
	1549	if [ $DRYRUN = 0 ]; then
	1550	if [ ! -d ${R_SAVE}/${1} ] ; then
	1551	echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	1552	echo "Nothing has been done."
	1553	return
	1554	fi
	1555	#
	1556	dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
	1557	RET=$?
	1558
	1559	# if [ ${RET} -gt 0 ] ; then
	1560	# echo "IGCM_sys_Dods_Cp : error."
	1561	# cat out_dods_cp
	1562	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	1563	# else
	1564	# rm out_dods_cp
	1565	# fi
	1566
	1567	else
	1568	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1569	fi
	1570	return $RET
	1571	}
	1572
	1573	#D-#==================================================
	1574	#D-function IGCM_sys_Put_Dods
	1575	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
	1576	#D-* Examples:
	1577	#D-
	1578	function IGCM_sys_Put_Dods {
	1579	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	1580	if ( $DEBUG_sys ) ; then
	1581	echo "IGCM_sys_Put_Dods :" $@
	1582	fi
	1583	if [ $DRYRUN = 0 ]; then
	1584	if [ ! -d ${R_SAVE}/${1} ] ; then
	1585	echo "WARNING : IGCM_sys_Put_Dods ${R_SAVE}/${1} DOES NOT EXIST ."
	1586	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1587	return
	1588	fi
	1589
	1590	typeset RET
	1591	#
	1592	cd ${R_SAVE}
	1593	IGCM_sys_Dods_Rm ${1}
	1594	IGCM_sys_Dods_Cp ${1}
	1595	RET=0
	1596
	1597	if [ ${RET} -gt 0 ] ; then
	1598	echo "IGCM_sys_Put_Dods : error."
	1599	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1600	fi
	1601	else
	1602	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
	1603	fi
	1604	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	1605	}
	1606
[222]	1607	##############################################################
	1608	# REBUILD OPERATOR
	1609
	1610	function IGCM_sys_rebuild {
[571]	1611	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
	1612	if ( $DEBUG_sys ) ; then
	1613	echo "IGCM_sys_rebuild :" $@
	1614	fi
	1615	/home/cont003/p86ipsl/X64/bin/rebuild -f -o $@
	1616	if [ $? -gt 0 ] ; then
	1617	echo "IGCM_sys_rebuild : erreur ${@}."
	1618	IGCM_debug_Exit "rebuild"
	1619	fi
[222]	1620
[571]	1621	IGCM_debug_PopStack "IGCM_sys_rebuild"
[222]	1622	}
	1623
	1624	############################################################
	1625	# Activate Running Environnment Variables
	1626
	1627	function IGCM_sys_activ_variables {
[571]	1628	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1629	if ( $DEBUG_sys ) ; then
	1630	echo "IGCM_sys_activ_variables"
	1631	fi
[490]	1632
[571]	1633	typeset max_omp
[490]	1634
[571]	1635	ulimit -s unlimited
[490]	1636
[571]	1637	(( max_omp = 0 ))
	1638	for comp in ${config_ListOfComponents[*]} ; do
[490]	1639
[571]	1640	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1641	eval ExeNameOut=\${config_Executable_${comp}[1]}
[490]	1642
[571]	1643	# Only if we really have an executable for the component :
	1644	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
	1645	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1646	if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
	1647	(( max_omp = comp_proc_omp_loc ))
	1648	fi
[490]	1649	fi
[571]	1650	done
	1651	if [ ${max_omp} -gt 1 ] ; then
	1652	module load openmp/${max_omp}thds
	1653	fi
[490]	1654
[571]	1655	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[222]	1656	}
	1657
	1658	############################################################
	1659	# Desactivate Running Environnment Variables
	1660
	1661	function IGCM_sys_desactiv_variables {
[571]	1662	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1663	if ( $DEBUG_sys ) ; then
	1664	echo "IGCM_sys_desactiv_variables"
	1665	fi
	1666	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[222]	1667	}
	1668
	1669	############################################################
[570]	1670	# Build MPI/OMP scripts run file (dummy function)
[222]	1671
[570]	1672	function IGCM_sys_build_run_file {
[490]	1673
[570]	1674	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[490]	1675
[570]	1676	}
[490]	1677
	1678	############################################################
	1679	# Build MPI/OMP scripts
	1680	function IGCM_sys_build_execution_scripts
	1681	{
[571]	1682	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1683	if ( $DEBUG_sys ) ; then
	1684	echo "IGCM_sys_build_execution_scripts " $@
	1685	fi
	1686	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
	1687	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
	1688	typeset num_corempi nombre_restant_node nombre_restant_comp
[491]	1689
[571]	1690	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
	1691	IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
	1692	fi
[490]	1693
[571]	1694	if ( ${OK_PARA_MPMD} ) ; then
[490]	1695
[571]	1696	if [ -f run_file ] ; then
	1697	IGCM_sys_Rm -f run_file
	1698	fi
	1699	touch run_file
[570]	1700
[571]	1701	if ( ${OK_PARA_OMP} ) ; then
[570]	1702
[571]	1703	# Hosts treatment
[490]	1704
[571]	1705	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
[490]	1706
[571]	1707	i=0
	1708	rm -f hosts
	1709	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
	1710	for nodes in `cat hosts.tmp` ; do
	1711	host[$i]=$nodes
	1712	echo "${host[$i]} slots=8 max_slots=8" >> hosts
	1713	IGCM_debug_Print 1 ${host[$i]}
	1714	i=$((i+1))
	1715	done
	1716	rm -f hosts.tmp
[570]	1717
[571]	1718	listnodes=${host[*]}
[570]	1719
[571]	1720	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
[570]	1721
[571]	1722	# Initialisation
[490]	1723
[571]	1724	init_node=y
	1725	node_num_current=0
	1726	start_num=0
	1727	init_exec=n
[490]	1728
[571]	1729	# Test : if oasis is there, we put it at the first position
[570]	1730
[571]	1731	for comp in ${config_ListOfComponents[*]} ; do
[490]	1732
[571]	1733	if [ "X${comp}" = "XCPL" ] ; then
[570]	1734
[571]	1735	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1736	eval ExeNameOut=\${config_Executable_${comp}[1]}
[570]	1737
[571]	1738	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1739	echo "" >> script_${ExeNameOut}.ksh
	1740	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1741	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1742	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1743	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1744	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[570]	1745
[571]	1746	init_node=n
[570]	1747
[571]	1748	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
	1749	node_num_current=0
	1750	node_current=${host[${node_num_current}]}
[570]	1751
[571]	1752	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
[570]	1753
[571]	1754	init_exec=y
	1755	start_num=1
[570]	1756
[571]	1757	fi
[570]	1758
[571]	1759	done
[490]	1760
[571]	1761	# Then loop on the components (except for oasis)
[490]	1762
[571]	1763	for comp in ${config_ListOfComponents[*]} ; do
[570]	1764
[571]	1765	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1766	eval ExeNameOut=\${config_Executable_${comp}[1]}
[570]	1767
[571]	1768	# Only if we really have an executable for the component :
	1769	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[490]	1770
[571]	1771	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1772	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[570]	1773
[571]	1774	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1775	# echo "set -vx" >> script_${ExeNameOut}.ksh
	1776	echo "" >> script_${ExeNameOut}.ksh
	1777	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1778	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1779	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1780	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1781	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
	1782	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
	1783	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[570]	1784
[571]	1785	node_num=0
[570]	1786
[571]	1787	# We define the number of MPI process to be assigned for the component
[490]	1788
[571]	1789	nombre_restant_comp=${comp_proc_mpi_loc}
[570]	1790
[571]	1791	# Loop on the allocated nodes
[490]	1792
[571]	1793	for node in ${listnodes} ; do
[490]	1794
[571]	1795	# We go to the current node
	1796	if [ ${node_num} = ${node_num_current} ] ; then
[490]	1797
[571]	1798	node_current=${host[${node_num_current}]}
[490]	1799
[571]	1800	# If first time on the node : initialisation
[490]	1801
[571]	1802	if [ ${init_node} = y ] ; then
	1803	nombre_restant_node=${NUM_COREPERNODE}
	1804	fi
[570]	1805
[571]	1806	# Test on the number of OMP threads
[490]	1807
[571]	1808	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
	1809	(( node_num = node_num + 1 ))
	1810	node_num_current=${node_num}
	1811	init_node=y
	1812	continue
	1813	fi
[570]	1814
[571]	1815	# Number of MPI process to assign
[570]	1816
[571]	1817	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
[570]	1818
[571]	1819	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
	1820	num_corempi=${nombre_restant_comp}
	1821	fi
[570]	1822
[571]	1823	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
	1824	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
[570]	1825
[571]	1826	if [ ${init_exec} = y ] ; then
	1827	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
	1828	else
	1829	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
	1830	init_exec=y
	1831	fi
[570]	1832
[571]	1833	(( start_num = num_corempi + start_num ))
[570]	1834
[571]	1835	else
[570]	1836
[571]	1837	(( node_num = node_num + 1 ))
	1838	continue
	1839	fi
[570]	1840
[571]	1841	# Test on the number of core/process remaining on the node/component
[570]	1842
[571]	1843	if [ ${nombre_restant_node} = 0 ] ; then
	1844	(( node_num = node_num + 1 ))
	1845	node_num_current=${node_num}
	1846	init_node=y
[490]	1847
[571]	1848	if [ ${nombre_restant_comp} = 0 ] ; then
	1849	break 1
	1850	fi
	1851	else
[490]	1852
[571]	1853	node_num_current=${node_num}
	1854	init_node=n
[570]	1855
[571]	1856	if [ ${nombre_restant_comp} = 0 ] ; then
	1857	break 1
	1858	fi
	1859	fi
	1860	done
	1861	fi
	1862	done
[570]	1863
[571]	1864	else
[490]	1865
[571]	1866	# Then first loop on the components for the coupler ie oasis
[570]	1867
[571]	1868	### the coupler ie oasis must be the first one
	1869	for comp in ${config_ListOfComponents[*]} ; do
[570]	1870
[571]	1871	eval ExeNameOut=\${config_Executable_${comp}[1]}
[570]	1872
[571]	1873	# for CPL component only
	1874	if [ "X${comp}" = "XCPL" ] ; then
[570]	1875
[571]	1876	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[490]	1877
[571]	1878	if ( ${OK_PARA_MPI} ) ; then
	1879	(( mpi_count = 1 ))
	1880	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
	1881	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1882	(( mpi_count = mpi_count + 1 ))
	1883	done
	1884	else
	1885	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
	1886	fi
	1887	fi
	1888	done
[570]	1889
[571]	1890	# Then second loop on the components
[570]	1891
[571]	1892	for comp in ${config_ListOfComponents[*]} ; do
[490]	1893
[571]	1894	eval ExeNameOut=\${config_Executable_${comp}[1]}
[490]	1895
[571]	1896	# Only if we really have an executable for the component and not the coupler ie oasis:
	1897	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[490]	1898
[571]	1899	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[570]	1900
[571]	1901	if ( ${OK_PARA_MPI} ) ; then
[570]	1902
[571]	1903	(( mpi_count = 1 ))
	1904	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
	1905	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1906	(( mpi_count = mpi_count + 1 ))
	1907	done
	1908	else
	1909	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
	1910	fi
	1911	fi
	1912	done
	1913	IGCM_sys_Chmod u+x run_file
[490]	1914
[571]	1915	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
[570]	1916
[571]	1917	fi
[490]	1918
[571]	1919	else # Only one executable. launch it.
[490]	1920
[571]	1921	for comp in ${config_ListOfComponents[*]} ; do
[570]	1922
[571]	1923	# Only if we really have an executable for the component :
	1924	eval ExeNameOut=\${config_Executable_${comp}[1]}
	1925	if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
[570]	1926
[571]	1927	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1928	echo "" >> script_${ExeNameOut}.ksh
	1929	if ( ${OK_PARA_OMP} ) ; then
	1930	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1931	fi
	1932	if ( ${OK_PARA_MPI} ) ; then
	1933	# Attention : a voir si sur 1 proc mais non //
	1934	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
	1935	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1936	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
	1937	else
	1938	EXECUTION="time ./${ExeNameOut}"
	1939	fi
	1940	fi
	1941	done
[570]	1942
[571]	1943	fi
[570]	1944
	1945	IGCM_debug_Print 1 "sys Titane : La commande d execution est "
	1946	IGCM_debug_Print 1 $EXECUTION
	1947
	1948	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[222]	1949	}
[451]	1950
[490]	1951
	1952
[451]	1953	##############################################################
	1954	# NCO OPERATOR
	1955
[571]	1956	function IGCM_sys_ncap2 {
	1957	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
	1958	if ( $DEBUG_sys ) ; then
	1959	echo "IGCM_sys_ncap2 :" $@
	1960	fi
	1961	ncap2 "$@"
	1962	if [ $? -gt 0 ] ; then
	1963	echo "IGCM_sys_ncap2 : erreur ${@}."
	1964	IGCM_debug_Exit "ncap2"
	1965	fi
	1966
	1967	IGCM_debug_PopStack "IGCM_sys_ncap2"
	1968	}
	1969
[451]	1970	function IGCM_sys_ncatted {
[571]	1971	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
	1972	if ( $DEBUG_sys ) ; then
	1973	echo "IGCM_sys_ncatted :" $@
	1974	fi
	1975	ncatted "$@"
	1976	if [ $? -gt 0 ] ; then
	1977	echo "IGCM_sys_ncatted : erreur ${@}."
	1978	IGCM_debug_Exit "ncatted"
	1979	fi
[451]	1980
[571]	1981	IGCM_debug_PopStack "IGCM_sys_ncatted"
[451]	1982	}
	1983
	1984	function IGCM_sys_ncbo {
[571]	1985	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
	1986	if ( $DEBUG_sys ) ; then
	1987	echo "IGCM_sys_ncbo :" $@
	1988	fi
	1989	ncbo $@
	1990	if [ $? -gt 0 ] ; then
	1991	echo "IGCM_sys_ncbo : erreur ${@}."
	1992	IGCM_debug_Exit "ncbo"
	1993	fi
[451]	1994
[571]	1995	IGCM_debug_PopStack "IGCM_sys_ncbo"
[451]	1996	}
	1997
	1998	function IGCM_sys_ncdiff {
[571]	1999	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
	2000	if ( $DEBUG_sys ) ; then
	2001	echo "IGCM_sys_ncdiff :" $@
	2002	fi
	2003	ncdiff $@
	2004	if [ $? -gt 0 ] ; then
	2005	echo "IGCM_sys_ncdiff : erreur ${@}."
	2006	IGCM_debug_Exit "ncdiff"
	2007	fi
[451]	2008
[571]	2009	IGCM_debug_PopStack "IGCM_sys_ncdiff"
[451]	2010	}
	2011
	2012	function IGCM_sys_ncea {
[571]	2013	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
	2014	if ( $DEBUG_sys ) ; then
	2015	echo "IGCM_sys_ncea :" $@
	2016	fi
	2017	ncea $@
	2018	if [ $? -gt 0 ] ; then
	2019	echo "IGCM_sys_ncea : erreur ${@}."
	2020	IGCM_debug_Exit "ncea"
	2021	fi
[451]	2022
[571]	2023	IGCM_debug_PopStack "IGCM_sys_ncea"
[451]	2024	}
	2025
	2026	function IGCM_sys_ncecat {
[571]	2027	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
	2028	if ( $DEBUG_sys ) ; then
	2029	echo "IGCM_sys_ncecat :" $@
	2030	fi
	2031	ncecat $@
	2032	if [ $? -gt 0 ] ; then
	2033	echo "IGCM_sys_ncecat : erreur ${@}."
	2034	IGCM_debug_Exit "ncecat"
	2035	fi
[451]	2036
[571]	2037	IGCM_debug_PopStack "IGCM_sys_ncecat"
[451]	2038	}
	2039
	2040	function IGCM_sys_ncflint {
[571]	2041	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
	2042	if ( $DEBUG_sys ) ; then
	2043	echo "IGCM_sys_ncflint :" $@
	2044	fi
	2045	ncflint $@
	2046	if [ $? -gt 0 ] ; then
	2047	echo "IGCM_sys_ncflint : erreur ${@}."
	2048	IGCM_debug_Exit "ncflint"
	2049	fi
[451]	2050
[571]	2051	IGCM_debug_PopStack "IGCM_sys_ncflint"
[451]	2052	}
	2053
	2054	function IGCM_sys_ncks {
[571]	2055	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
	2056	if ( $DEBUG_sys ) ; then
	2057	echo "IGCM_sys_ncks :" $@
	2058	fi
	2059	ncks $@
	2060	if [ $? -gt 0 ] ; then
	2061	echo "IGCM_sys_ncks : erreur ${@}."
	2062	IGCM_debug_Exit "ncks"
	2063	fi
[451]	2064
[571]	2065	IGCM_debug_PopStack "IGCM_sys_ncks"
[451]	2066	}
	2067
	2068	function IGCM_sys_ncpdq {
[571]	2069	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
	2070	if ( $DEBUG_sys ) ; then
	2071	echo "IGCM_sys_ncpdq :" $@
	2072	fi
	2073	ncpdq $@
	2074	if [ $? -gt 0 ] ; then
	2075	echo "IGCM_sys_ncpdq : erreur ${@}."
	2076	IGCM_debug_Exit "ncpdq"
	2077	fi
[451]	2078
[571]	2079	IGCM_debug_PopStack "IGCM_sys_ncpdq"
[451]	2080	}
	2081
	2082	function IGCM_sys_ncra {
[571]	2083	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
	2084	if ( $DEBUG_sys ) ; then
	2085	echo "IGCM_sys_ncra :" $@
	2086	fi
	2087	ncra $@
	2088	if [ $? -gt 0 ] ; then
	2089	echo "IGCM_sys_ncra : erreur ${@}."
	2090	IGCM_debug_Exit "ncra"
	2091	fi
[451]	2092
[571]	2093	IGCM_debug_PopStack "IGCM_sys_ncra"
[451]	2094	}
	2095
	2096	function IGCM_sys_ncrcat {
[571]	2097	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
	2098	if ( $DEBUG_sys ) ; then
	2099	echo "IGCM_sys_ncrcat :" $@
	2100	fi
	2101	ncrcat $@
	2102	if [ $? -gt 0 ] ; then
	2103	echo "IGCM_sys_ncrcat : erreur ${@}."
[451]	2104	# IGCM_debug_Exit "ncrcat"
[571]	2105	fi
[451]	2106
[571]	2107	IGCM_debug_PopStack "IGCM_sys_ncrcat"
[451]	2108	}
	2109
	2110	function IGCM_sys_ncrename {
[571]	2111	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
	2112	if ( $DEBUG_sys ) ; then
	2113	echo "IGCM_sys_ncrename :" $@
	2114	fi
	2115	ncrename $@
	2116	if [ $? -gt 0 ] ; then
	2117	echo "IGCM_sys_ncrename : erreur ${@}."
	2118	IGCM_debug_Exit "ncrename"
	2119	fi
[451]	2120
[571]	2121	IGCM_debug_PopStack "IGCM_sys_ncrename"
[451]	2122	}
	2123
	2124	function IGCM_sys_ncwa {
[571]	2125	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
	2126	if ( $DEBUG_sys ) ; then
	2127	echo "IGCM_sys_ncwa :" $@
	2128	fi
	2129	ncwa $@
	2130	if [ $? -gt 0 ] ; then
	2131	echo "IGCM_sys_ncwa : erreur ${@}."
	2132	IGCM_debug_Exit "ncwa"
	2133	fi
[451]	2134
[571]	2135	IGCM_debug_PopStack "IGCM_sys_ncwa"
[451]	2136	}
[571]	2137
	2138	##############################################################
	2139	# CDO OPERATOR
	2140
	2141	function IGCM_sys_cdo {
	2142	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
	2143	if ( $DEBUG_sys ) ; then
	2144	echo "IGCM_sys_cdo :" $@
	2145	fi
	2146	\cdo $@
	2147	if [ $? -gt 0 ] ; then
	2148	echo "IGCM_sys_cdo : erreur ${@}."
	2149	IGCM_debug_PopStack "IGCM_sys_cdo"
	2150	return 1
	2151	else
	2152	IGCM_debug_PopStack "IGCM_sys_cdo"
	2153	return 0
	2154	fi
	2155
	2156	IGCM_debug_PopStack "IGCM_sys_cdo"
	2157	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: