Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1385

Last change on this file since 1385 was 1385, checked in by mafoipsl, 7 years ago
On curie, use advised method for MPMD/MPI/OpenMP execution.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 54.9 KB

Rev	Line
[622]	1	#!/bin/ksh
	2
	3	#**************************************************************
	4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
	5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#=========================================================
	15	# The documentation of this file can be automatically generated
[623]	16	# if you use the prefix #D- for comments to be extracted.
[622]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#=========================================================
	19
	20	#D-#==================================================
[623]	21	#D-LibIGCM_sys for Curie
[622]	22	#D-#==================================================
	23	#D-
[623]	24	#D- This ksh library if a layer under some usefull
[622]	25	#D-environment variables and shell commands.
	26	#D-All those definitions depend on host particularities.
	27	#D-It manages a stack mechanism and test validity of operations.
	28	#D-All function described bellow must be prefixed by IGCM_sys.
	29
	30	#====================================================
	31	# libIGCM_sys PARAMETERS
	32	#====================================================
	33
	34	#====================================================
	35	# set DEBUG_sys to true to output calls of function
	36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
	37
	38	#====================================================
	39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
	40	typeset -r DRYRUN=${DRYRUN:=0}
	41
	42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
	43	# -------------------------------------------------------------------------------------
	44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
[809]	45	# \| \| Cp/Exe/param/files \| \| \| \|
	46	# \| \| Chmod Qsub \| \| \| \|
[622]	47	# -------------------------------------------------------------------------------------
	48	# \| 0 \| yes \| yes \| yes \| yes \|
	49	# -------------------------------------------------------------------------------------
	50	# \| 1 \| yes \| yes \| yes \| no \|
	51	# -------------------------------------------------------------------------------------
	52	# \| 2 \| yes \| yes \| no \| no \|
	53	# -------------------------------------------------------------------------------------
	54	# \| 3 \| yes \| no \| no \| no \|
	55	# -------------------------------------------------------------------------------------
	56
	57	#=====================================================
	58	# Global Variables :
	59	#=====================================================
	60	# Language : "fr" or "en"
	61	typeset -r MYLANG="fr"
	62
	63	#=====================================================
[1201]	64	# Host user names project maxCpuTime
[622]	65	# $hostname ou hostname
[1158]	66	typeset HOST=${HOST:=$( hostname )}
[622]	67	# $username ou whoami
[1158]	68	typeset LOGIN=${LOGIN:=$( whoami )}
[622]	69	# $hostname of the MASTER job
	70	typeset MASTER=curie
[880]	71	# add default project on curie
[1201]	72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} \| cut -d@ -f1 )
	73	# jobWarningDelay in seconds
	74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
[622]	75
	76	#D-
	77	#D-#==================================================
	78	#D-Program used in libIGCM
	79	#D-#==================================================
	80
[742]	81	# Submit command
[747]	82	typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]	83	# rsync with path
	84	typeset -r RSYNC=/usr/bin/rsync
	85	# RSYNC_opt args to rsync
	86	typeset -r RSYNC_opt="-va"
	87	# ie storage filesystem
[623]	88	typeset -r STOREHOST=${MASTER}
[1180]	89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
[622]	90
	91	#====================================================
[1180]	92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
[622]	93	#====================================================
[996]	94	if [ X${TaskType} = Xcomputing ] ; then
[1358]	95	module load python/2.7.8 > /dev/null 2>&1
[1193]	96	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
[996]	97	# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
	98	# uncomment 2 lines :
	99	# module unload netcdf
	100	# module load netcdf/3.6.3
	101	else
[1196]	102	. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
[1358]	103	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
[1364]	104	export UVCDAT_ANONYMOUS_LOG=FALSE
[996]	105	fi
[1315]	106
	107	# Use CMIP6 storage space when using CMIP6 cpu ressources
[1330]	108	[[ "X${PROJECT}" = "Xdevcmip6" && ! $(module list --terse 2>&1 \| grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
[1315]	109
	110	# FYI
[1210]	111	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
	112	[ ! X${TaskType} = Xchecking ] && module list
[1208]	113
[1179]	114	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
	115	export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
	116	export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
[622]	117
	118	#====================================================
	119	# Specific for ocean additionnal diagnostic
	120	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
	121	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
	122
	123	#====================================================
	124	# Host specific DIRECTORIES
	125	#====================================================
	126
[623]	127	# ============ CESIUM START ============ #
	128
[622]	129	#====================================================
[623]	130	#- Mirror libIGCM from titane to cesium if needed
	131	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
	132	#if [ ! ${ROOTSYS} = "home" ] ; then
	133	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
	134	#else
	135	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	136	#fi
	137
	138	#====================================================
	139	#- libIGCM_POST
	140	#if ( ${MirrorlibIGCM} ) ; then
	141	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
	142	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
	143	#else
	144	# typeset -r libIGCM_POST=${libIGCM}
	145	#fi
	146
	147	# ============ CESIUM END ============ #
	148
	149	#====================================================
	150	#- MirrorlibIGCM for frontend
	151	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
	152
	153	#====================================================
	154	#- libIGCM_POST for frontend
	155	typeset -r libIGCM_POST=${libIGCM}
	156
	157	#====================================================
[622]	158	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
	159	typeset -r R_EXE="${MODIPSL}/bin"
	160
	161	#====================================================
	162	#- SUBMIT_DIR : submission dir
[1010]	163	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
[622]	164
	165	#====================================================
	166	#- IN
[1276]	167	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
[622]	168	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
	169
	170	#====================================================
	171	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[850]	172	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[622]	173
	174	#====================================================
[1179]	175	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
[1186]	176	typeset -r OUTCOMMAND_PATH=/tmp
[1179]	177
	178	#====================================================
[622]	179	#- HOST_MPIRUN_COMMAND
[1382]	180	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
[622]	181
	182	#====================================================
	183	#- Max number of arguments passed to nco operator or demigration command
	184	UNIX_MAX_LIMIT=360
	185
[623]	186	#====================================================
[1180]	187	#- set PackDefault to true on curie
[623]	188	PackDefault=true
	189
[664]	190	#====================================================
[623]	191	#- Default number of MPI task for IPSL coupled model
	192	#- required for backward compatibility
	193	#-
	194	DEFAULT_NUM_PROC_OCE=5
	195	DEFAULT_NUM_PROC_CPL=1
[637]	196	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
[623]	197
[725]	198	#D-#==================================================
[1057]	199	#D-function IGCM_sys_defineArchives
	200	#D-* Purpose:
	201	#D-* Define ARCHIVE : Dedicated to large files
	202	#D-* Define STORAGE : Dedicated to small/medium files
	203	#D-* Define R_OUT : Output tree located on ARCHIVE
	204	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
	205	#D-* Define R_BUF : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
	206	#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
[725]	207	#D-* Examples:
	208	#D-
[1057]	209	function IGCM_sys_defineArchives {
	210	IGCM_debug_PushStack "IGCM_sys_defineArchives"
[676]	211
[1060]	212	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]	213	#====================================================
	214	#- ARCHIVE (dedicated to large files)
[1063]	215	ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]	216	else
	217	#====================================================
	218	#- ARCHIVE (dedicated to large files)
[1063]	219	ARCHIVE=${CCCSTOREDIR}
[1057]	220	fi
[725]	221
[1060]	222	if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]	223	#====================================================
	224	#- STORAGE (dedicated to small/medium files)
[1063]	225	STORAGE=${config_UserChoices_STORAGE}
[1057]	226	else
	227	#====================================================
	228	#- STORAGE (dedicated to small/medium files)
[1063]	229	STORAGE=${CCCWORKDIR}
[1057]	230	fi
[725]	231
[1057]	232	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
	233	#====================================================
	234	#- R_OUT
[1063]	235	R_OUT=${SCRATCHDIR}/IGCM_OUT
[1057]	236
	237	#====================================================
	238	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	239	R_FIG=${SCRATCHDIR}/IGCM_OUT
[1057]	240
	241	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
	242
	243	else
	244	#====================================================
	245	#- R_OUT
[1063]	246	R_OUT=${ARCHIVE}/IGCM_OUT
[1057]	247
	248	#====================================================
	249	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]	250	R_FIG=${STORAGE}/IGCM_OUT
[1057]	251	fi
	252
	253	#====================================================
	254	#- R_BUF (ONLY FOR double copy an scratch)
[1063]	255	R_BUF=${SCRATCHDIR}/IGCM_OUT
[1057]	256
	257	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
	258	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
	259	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
	260
	261	IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]	262	}
	263
[622]	264	#D-#==================================================
	265	#D-function IGCM_sys_RshArchive
	266	#D-* Purpose: Archive rsh command
	267	#D-* Examples:
	268	#D-
	269	function IGCM_sys_RshArchive {
[623]	270	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
	271	/bin/ksh <<-EOF
[622]	272	${@}
	273	EOF
[811]	274	status=$?
	275	if [ ${status} -gt 0 ] ; then
	276	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]	277	IGCM_debug_Exit "IGCM_sys_RshArchive"
	278	fi
	279	IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]	280	}
	281
	282	#D-#==================================================
[845]	283	#D-function IGCM_sys_RshArchive_NoError
	284	#D-* Purpose: Archive rsh command, without error
	285	#D-* used only in monitoring.job
	286	#D-* Examples:
	287	#D-
	288	function IGCM_sys_RshArchive_NoError {
	289	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
	290	/bin/ksh <<-EOF
[1181]	291	${@} 2> /dev/null
[845]	292	EOF
	293	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
	294	}
	295
	296	#D-#==================================================
[622]	297	#D-function IGCM_sys_MkdirArchive
	298	#D-* Purpose: Mkdir on Archive
	299	#D-* Examples:
	300	#D-
	301	function IGCM_sys_MkdirArchive {
[623]	302	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
	303	if ( $DEBUG_sys ) ; then
	304	echo "IGCM_sys_MkdirArchive :" $@
	305	fi
	306	#- creation de repertoire sur le serveur fichier
	307	if [ ! -d ${1} ]; then
	308	\mkdir -p $1
[811]	309	status=$?
	310
	311	if [ ${status} -gt 0 ] ; then
	312	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]	313	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]	314	fi
[623]	315	fi
	316	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]	317	}
	318
	319	#D-#==================================================
	320	#D-function IGCM_sys_TestDirArchive
	321	#D-* Purpose: Test Directory that must exists on Archive
	322	#D-* Examples:
	323	#D-
	324	function IGCM_sys_TestDirArchive {
[623]	325	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
	326	if ( $DEBUG_sys ) ; then
	327	echo "IGCM_sys_TestDirArchive :" $@
	328	fi
	329	typeset ExistFlag
	330	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
	331	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
	332	return ${ExistFlag}
[622]	333	}
	334
	335	#D-#==================================================
[623]	336	#D-function IGCM_sys_IsFileArchived
[1179]	337	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
[623]	338	#D-* Examples:
	339	#D-
	340	function IGCM_sys_IsFileArchived {
	341	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
	342	if ( $DEBUG_sys ) ; then
	343	echo "IGCM_sys_IsFileArchived :" $@
	344	fi
	345	typeset IsArchivedFlag
[818]	346	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
[623]	347	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
	348
	349	return ${IsArchivedFlag}
	350	}
	351
	352	#D-#==================================================
[622]	353	#D-function IGCM_sys_TestFileArchive
	354	#D-* Purpose: Test file that must NOT EXISTS on Archive
	355	#D-* Examples:
	356	#D-
	357	function IGCM_sys_TestFileArchive {
[623]	358	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
	359	typeset ExistFlag
	360	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
	361	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]	362
[623]	363	return ${ExistFlag}
[622]	364	}
	365
	366	#D-#==================================================
	367	#D-function IGCM_sys_CountFileArchive
	368	#D-* Purpose: Count files on Archive filesystem
	369	#D-* Examples:
	370	#D-
	371	function IGCM_sys_CountFileArchive {
[623]	372	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
	373	ls ${@} 2>/dev/null \| wc -l
	374	if [ $? -gt 0 ] ; then
	375	echo "IGCM_sys_CountFileArchive : erreur."
	376	fi
	377	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]	378	}
	379
	380	#D-#==================================================
	381	#D-function IGCM_sys_Tree
	382	#D-* Purpose: Tree directories with files on ${ARCHIVE}
	383	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
	384	#D-
	385	function IGCM_sys_Tree {
[623]	386	IGCM_debug_PushStack "IGCM_sys_Tree" $@
	387	if ( $DEBUG_sys ) ; then
	388	echo "IGCM_sys_Tree :" $@
	389	fi
[622]	390
[623]	391	\ls -lR ${@}
[622]	392
[623]	393	IGCM_debug_PopStack "IGCM_sys_Tree"
[622]	394	}
	395
	396	#D-#==================================================
	397	#D-function IGCM_sys_Qsub
	398	#D-* Purpose: Qsub new job
	399	#D-* Examples:
	400	#D-
	401	function IGCM_sys_Qsub {
[623]	402	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
	403	if ( $DEBUG_sys ) ; then
	404	echo "IGCM_sys_Qsub :" $@
	405	fi
[811]	406	typeset options status
[923]	407	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]	408
[1179]	409	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	410	status=$?
[948]	411
[1179]	412	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	413	if [ ${status} -gt 0 ] ; then
	414	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]	415	IGCM_debug_Exit "IGCM_sys_Qsub"
[948]	416	else
[1179]	417	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	418	fi
	419	IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]	420	}
	421
	422	#D-#==================================================
	423	#D-function IGCM_sys_QsubPost
	424	#D-* Purpose: Qsub new job on scalaire
	425	#D-* Examples:
	426	#D-
	427	function IGCM_sys_QsubPost {
[623]	428	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
	429	if ( $DEBUG_sys ) ; then
	430	echo "IGCM_sys_QsubPost :" $@
	431	fi
[923]	432	typeset options status
	433
[1062]	434	# EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
	435	case $( hostname -s ) in
	436	curie*)
[1201]	437	options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
[1062]	438	airain*)
	439	options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
	440	esac
	441
[1179]	442	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[811]	443	status=$?
[948]	444
[1179]	445	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[811]	446	if [ ${status} -gt 0 ] ; then
[923]	447	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]	448	IGCM_debug_Exit "IGCM_sys_QsubPost"
[948]	449	else
[1179]	450	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
[623]	451	fi
	452	IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]	453	}
	454
	455	#D-*************************
[623]	456	#D- File transfer functions
[622]	457	#D-*************************
	458	#D-
	459
	460	#D-#==================================================
	461	#D-function IGCM_sys_RmRunDir
	462	#D-* Purpose: rm tmpdir (dummy function most of the time batch
	463	#D- scheduler will do the job)
	464	#D-* Examples:
	465	#D-
	466	function IGCM_sys_RmRunDir {
[1057]	467	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[623]	468	if ( $DEBUG_sys ) ; then
	469	echo "IGCM_sys_RmRunDir :" $@
	470	fi
[622]	471
[715]	472	typeset status
[622]	473
[1179]	474	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	475	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	476	status=$?
[622]	477
[715]	478	if [ ${status} -gt 0 ] ; then
[1183]	479	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
[1179]	480	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	481	IGCM_debug_Exit "IGCM_sys_RmRunDir"
	482	else
[1179]	483	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	484	fi
	485	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]	486	}
	487
	488	#D-#==================================================
	489	#D-function IGCM_sys_Put_Dir
	490	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
	491	#D-* Examples:
	492	#D-
	493	function IGCM_sys_Put_Dir {
[623]	494	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
	495	if ( $DEBUG_sys ) ; then
	496	echo "IGCM_sys_Put_Dir :" $@
	497	fi
	498	if [ $DRYRUN = 0 ]; then
	499	if [ ! -d ${1} ] ; then
	500	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
	501	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
	502	return
[622]	503	fi
	504
[715]	505	typeset status
[622]	506
[623]	507	# Only if we use rsync
	508	#IGCM_sys_TestDirArchive $( dirname $2 )
	509	#
	510	#USUAL WAY
[1179]	511	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	512	status=$?
[622]	513
[715]	514	if [ ${status} -gt 0 ] ; then
[811]	515	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[1179]	516	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	517	IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]	518	else
[1179]	519	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	520	fi
[623]	521	fi
	522	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]	523	}
	524
	525	#D-#==================================================
	526	#D-function IGCM_sys_Get_Dir
[623]	527	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]	528	#D-* Examples:
	529	#D-
	530	function IGCM_sys_Get_Dir {
[623]	531	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
	532	if ( $DEBUG_sys ) ; then
	533	echo "IGCM_sys_Get_Dir :" $@
	534	fi
	535	if [ $DRYRUN = 0 ]; then
[708]	536	typeset NB_ESSAI DELAI status i
	537	# number of tentative
	538	NB_ESSAI=3
	539	# time delay between tentative
	540	DELAI=2
[622]	541
[676]	542	#
[708]	543	# USUAL WAY
	544	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]	545	ccc_hsm get -r $1
[622]	546
[709]	547	i=0
[708]	548	while [ $i -lt $NB_ESSAI ] ; do
[1179]	549	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[708]	550	status=$?
[811]	551	if [ ${status} -gt 0 ] ; then
[713]	552	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]	553	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
	554	sleep $DELAI
	555	else
	556	break
	557	fi
	558	(( i = i + 1 ))
	559	done
	560
	561	if [ ${status} -gt 0 ] ; then
[1178]	562	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
[1179]	563	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	564	IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]	565	else
[1179]	566	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	567	fi
[623]	568	fi
	569	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]	570	}
	571
	572	#D-#==================================================
	573	#D-function IGCM_sys_Put_Rest
[623]	574	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]	575	#D- File and target directory must exist.
	576	#D-* Examples:
	577	#D-
	578	function IGCM_sys_Put_Rest {
[623]	579	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
	580	if ( $DEBUG_sys ) ; then
	581	echo "IGCM_sys_Put_Rest :" $@
	582	fi
	583	if [ $DRYRUN = 0 ]; then
	584	if [ ! -f ${1} ] ; then
	585	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
	586	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	587	fi
	588
[715]	589	typeset status
[623]	590	#
	591	# USUAL WAY
[1179]	592	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	593	status=$?
[622]	594
[623]	595	# #RSYNC WITH NETWORK SSH CALL
[1179]	596	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	597	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	598
[623]	599	# #RSYNC WITH NFS USE
[1179]	600	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	601	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	602
[715]	603	# status=$?
	604	# IGCM_sys_Rsync_out $status
[622]	605
[1179]	606	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[715]	607	# (( status=status+$? ))
[623]	608
[715]	609	if [ ${status} -gt 0 ] ; then
[1179]	610	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]	611	[ -f ${1} ] && ls -l ${1}
[759]	612	[ -f ${2} ] && ls -l ${2}
	613	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	614	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	615	IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]	616	else
[759]	617
	618	if [ X${JobType} = XRUN ] ; then
	619	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	620	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	621	fi
	622
[1179]	623	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	624	fi
[623]	625	fi
	626	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]	627	}
	628
	629	#D-#==================================================
	630	#D-function IGCM_sys_Put_Out
[623]	631	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]	632	#D-* Examples:
	633	#D-
	634	function IGCM_sys_Put_Out {
[623]	635	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
	636	if ( $DEBUG_sys ) ; then
	637	echo "IGCM_sys_Put_Out :" $@
	638	fi
[667]	639
[711]	640	typeset NB_ESSAI DELAI status i exist skip
[1234]	641	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
[667]	642
	643	# number of tentative
[668]	644	NB_ESSAI=3
[667]	645	# time delay between tentative
[668]	646	DELAI=2
[667]	647
[623]	648	if [ $DRYRUN = 0 ]; then
	649	if [ ! -f ${1} ] ; then
	650	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
	651	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	652	return 1
[622]	653	fi
[623]	654	#
	655	IGCM_sys_MkdirArchive $( dirname $2 )
	656	#
[711]	657	exist=false
	658	skip=false
	659	if [ -f $2 ] ; then
	660	IGCM_debug_Print 1 "$2 already exist"
	661	ccc_hsm get $2
	662	exist=true
	663	if [ "X$( diff $1 $2 )" = X ] ; then
	664	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
	665	skip=true
[667]	666	else
[711]	667	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
	668	skip=false
[667]	669	fi
[711]	670	fi
	671	#
	672	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
	673	IGCM_sys_Chmod u+w $2
	674	fi
[622]	675
[711]	676	if [ X${skip} = Xfalse ] ; then
[719]	677	i=0
[711]	678	while [ $i -lt $NB_ESSAI ] ; do
[1234]	679	# Identify file system
	680	fileDeviceNumberInHex=$( stat -c %d $1 )
	681	status=$?
	682	if [ ${status} -gt 0 ] ; then
	683	IGCM_debug_Exit "IGCM_sys_Put_Out"
	684	fi
	685	# Identify file system
	686	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
	687	status=$?
	688	if [ ${status} -gt 0 ] ; then
	689	IGCM_debug_Exit "IGCM_sys_Put_Out"
	690	fi
	691
	692	if [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] ; then
	693	# They are not on the same device. USUAL WAY
[1179]	694	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	695	status=$?
	696	else
[1234]	697	# They are on the same device. NOT SO USUAL WAY
[1179]	698	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	699	status=$?
	700	fi
[711]	701	if [ ${status} -gt 0 ]; then
[713]	702	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]	703	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]	704	[ -f ${1} ] && ls -l ${1}
[759]	705	[ -f ${2} ] && ls -l ${2}
	706	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]	707	sleep $DELAI
	708	else
	709	break
	710	fi
	711	(( i = i + 1 ))
	712	done
	713	fi
	714
[623]	715	# #RSYNC WITH NETWORK SSH CALL
[1179]	716	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	717	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	718
[623]	719	# #RSYNC WITH NFS USE
[1179]	720	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	721	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[622]	722
[711]	723	# status=$?
	724	# IGCM_sys_Rsync_out $status
[622]	725
[1179]	726	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[711]	727	# (( status=status+$? ))
[622]	728
[711]	729	if [ ${status} -gt 0 ] ; then
[1179]	730	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
[765]	731	[ -f ${1} ] && ls -l ${1}
[759]	732	[ -f ${2} ] && ls -l ${2}
	733	[ -f ${2}/${1} ] && ls -l ${2}/${1}
[1179]	734	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	735	IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]	736	else
[759]	737
	738	if [ X${JobType} = XRUN ] ; then
	739	if [ X${3} = X ] ; then
	740	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
	741	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
	742	fi
	743	fi
	744
[1179]	745	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[622]	746	fi
[623]	747	fi
	748	IGCM_debug_PopStack "IGCM_sys_Put_Out"
	749	return 0
[622]	750	}
	751
	752	#D-#==================================================
	753	#D-function IGCM_sys_Get
	754	#D-* Purpose: Get a file from ${ARCHIVE}
	755	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
	756	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
	757	function IGCM_sys_Get {
[623]	758	IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]	759
[773]	760	typeset DEST dm_liste target file_work
[667]	761	typeset NB_ESSAI DELAI status i
[622]	762
[623]	763	if ( $DEBUG_sys ) ; then
	764	echo "IGCM_sys_Get :" $@
	765	fi
[667]	766
	767	# number of tentative
[668]	768	NB_ESSAI=3
[667]	769	# time delay between tentative
[668]	770	DELAI=2
[667]	771
[623]	772	if [ $DRYRUN -le 2 ]; then
	773	if [ X${1} = X'/l' ] ; then
	774	eval set +A dm_liste \${${2}}
	775	else
	776	eval set +A dm_liste ${1}
[622]	777	fi
[623]	778	eval DEST=\${${#}}
[1179]	779	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[715]	780	status=$?
	781	if [ ${status} -gt 0 ] ; then
	782	echo "WARNING IGCM_sys_Get : error code ${status}"
[1179]	783	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[623]	784	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
	785	fi
[622]	786
[715]	787	#if [ ${status} -gt 0 ] ; then
[1179]	788	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
	789	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	790	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
	791	# sleep 30
	792	# echo "We try another time"
[1179]	793	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	794	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[785]	795	# status=$?
	796	# if [ ${status} -gt 0 ] ; then
	797	# echo "ERROR IGCM_sys_Get : again demigration error :"
[1179]	798	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	799	# IGCM_debug_Exit "IGCM_sys_Get"
[623]	800	# fi
[785]	801	# else
	802	# echo "ERROR IGCM_sys_Get : demigration error :"
[1179]	803	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[785]	804	# IGCM_debug_Exit "IGCM_sys_Get"
	805	# fi
[623]	806	#fi
[622]	807
[719]	808	# #RSYNC WITH NETWORK SSH CALL
[1179]	809	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	810	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	811
	812	# #RSYNC WITH NFS USE
[1179]	813	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	814	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	815
	816	# status=$?
	817	# IGCM_sys_Rsync_out $status
	818
[1179]	819	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	820	# (( status=status+$? ))
	821
[623]	822	#USUAL WAY
	823	if [ X${1} = X'/l' ] ; then
	824	for target in ${dm_liste[*]} ; do
	825	local_file=$( basename ${target} )
[668]	826	# test if the target file is present before the loop
	827	IGCM_sys_TestFileArchive ${target}
[715]	828	status=$?
	829	if [ ${status} -gt 0 ] ; then
[701]	830	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]	831	IGCM_debug_Exit "IGCM_sys_Get"
	832	else
[719]	833	i=0
[668]	834	while [ $i -lt $NB_ESSAI ] ; do
[1037]	835	#if [ X${DoLink} = Xtrue ] ; then
[1179]	836	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	837	# status=$?
	838	# else
[1179]	839	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[1037]	840	# status=$?
	841	#fi
[1179]	842	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[668]	843	status=$?
	844	if [ ${status} -gt 0 ]; then
[713]	845	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]	846	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	847	sleep $DELAI
[668]	848	else
[676]	849	break
[668]	850	fi
	851	(( i = i + 1 ))
	852	done
[719]	853	if [ ${status} -gt 0 ] ; then
	854	echo "IGCM_sys_Get : error"
[1179]	855	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	856	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	857	IGCM_debug_Exit "IGCM_sys_Get"
	858	else
[1179]	859	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	860	fi
[668]	861	fi
[623]	862	done
	863	else
[719]	864	i=0
	865	while [ $i -lt $NB_ESSAI ] ; do
[1179]	866	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
[719]	867	status=$?
	868	if [ ${status} -gt 0 ]; then
	869	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
	870	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
	871	sleep $DELAI
	872	else
	873	break
	874	fi
	875	(( i = i + 1 ))
	876	done
	877	if [ ${status} -gt 0 ] ; then
	878	echo "IGCM_sys_Get : error"
[1179]	879	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	880	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	881	IGCM_debug_Exit "IGCM_sys_Get"
	882	else
[1179]	883	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
[719]	884	fi
[623]	885	fi
	886	fi
	887	IGCM_debug_PopStack "IGCM_sys_Get"
	888	}
[622]	889
[623]	890	#D-#==================================================
[835]	891	#D-function IGCM_sys_GetDate_Monitoring
	892	#D-* Purpose: get the last year for which the monitoring has been computed
	893	#D-* Examples:
	894	#D-
	895	function IGCM_sys_GetDate_Monitoring {
	896	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
	897	if ( $DEBUG_sys ) ; then
	898	echo "IGCM_sys_GetDate_Monitoring :" $@
	899	fi
	900
	901	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
	902
	903	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
	904	}
	905
	906	#D-#==================================================
[623]	907	#D-function IGCM_sys_Dods_Rm
	908	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
	909	#D-* Examples:
	910	#D-
	911	function IGCM_sys_Dods_Rm {
	912	if ( $DEBUG_sys ) ; then
	913	echo "IGCM_sys_Dods_Rm :" $@
	914	fi
[715]	915	typeset status
[623]	916	if [ $DRYRUN = 0 ]; then
	917
	918	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
	919	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
	920	# echo "Nothing has been done."
	921	# return
	922	# fi
	923
[649]	924	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
[715]	925	status=$?
[623]	926
[785]	927	# if [ ${status} -gt 0 ] ; then
	928	# echo "IGCM_sys_Dods_Rm : error."
	929	# cat out_dods_rm
	930	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
	931	# else
	932	# rm out_dods_rm
	933	# fi
[623]	934
	935	fi
[715]	936	return $status
[623]	937	}
	938
	939	#D-#==================================================
	940	#D-function IGCM_sys_Dods_Cp
	941	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
	942	#D-* Examples:
	943	#D-
	944	function IGCM_sys_Dods_Cp {
	945	if ( $DEBUG_sys ) ; then
	946	echo "IGCM_sys_Dods_Cp :" $@
	947	fi
[715]	948	typeset status
[623]	949	if [ $DRYRUN = 0 ]; then
	950
	951	# if [ ! -d ${R_SAVE}/${1} ] ; then
	952	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
	953	# echo "Nothing has been done."
	954	# return
	955	# fi
	956
[649]	957	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]	958	status=$?
[623]	959
[715]	960	# if [ ${status} -gt 0 ] ; then
[623]	961	# echo "IGCM_sys_Dods_Cp : error."
	962	# cat out_dods_cp
	963	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
	964	# else
	965	# rm out_dods_cp
	966	# fi
	967
	968	fi
[715]	969	return $status
[623]	970	}
	971
	972	#D-#==================================================
	973	#D-function IGCM_sys_Put_Dods
[1178]	974	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
[623]	975	#D-* Examples:
	976	#D-
	977	function IGCM_sys_Put_Dods {
	978	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
	979	if ( $DEBUG_sys ) ; then
	980	echo "IGCM_sys_Put_Dods :" $@
	981	fi
[715]	982	typeset status
[623]	983	if [ $DRYRUN = 0 ]; then
	984	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
	985	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
	986	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
	987	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
	988	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	989	return
[622]	990	fi
[623]	991
	992	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
	993	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
	994	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
	995	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
	996	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
	997	return
	998	fi
	999	#
	1000	if [ -d ${R_SAVE}/${1} ] ; then
	1001	cd ${R_SAVE}
	1002	elif [ -d ${R_FIGR}/${1} ] ; then
	1003	cd ${R_FIGR}
	1004	fi
	1005
	1006	IGCM_sys_Dods_Rm ${1}
	1007	IGCM_sys_Dods_Cp ${1}
[715]	1008	status=0
[623]	1009
[715]	1010	if [ ${status} -gt 0 ] ; then
[623]	1011	echo "IGCM_sys_Put_Dods : error."
	1012	IGCM_debug_Exit "IGCM_sys_Put_Dods"
	1013	fi
	1014	fi
	1015	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]	1016	}
	1017
	1018	##############################################################
	1019	# REBUILD OPERATOR
	1020
[811]	1021	#D-#==================================================
[1163]	1022	#D-function IGCM_sys_sync
	1023	#D-* Purpose: flush buffer on disk
	1024	#D-* Examples:
	1025	#D-
	1026	function IGCM_sys_sync {
	1027	IGCM_debug_PushStack "IGCM_sys_sync" $@
	1028	if ( $DEBUG_sys ) ; then
	1029	echo "IGCM_sys_sync :" $@
	1030	fi
	1031
	1032	/bin/sync
	1033
	1034	IGCM_debug_PopStack "IGCM_sys_sync"
	1035	}
	1036
[622]	1037	############################################################
	1038	# Activate Running Environnment Variables
	1039
[811]	1040	#D-#==================================================
[1239]	1041	#D-function IGCM_sys_activ_variables
[811]	1042	#D-* Purpose: set environement variables prior to execution
	1043	#D-* Examples:
	1044	#D-
[622]	1045	function IGCM_sys_activ_variables {
[623]	1046	IGCM_debug_PushStack "IGCM_sys_activ_variables"
	1047	if ( $DEBUG_sys ) ; then
	1048	echo "IGCM_sys_activ_variables"
	1049	fi
[622]	1050
[811]	1051	# --------------------------------------------------------------------
	1052	#D- MPI specifications
	1053	# --------------------------------------------------------------------
[1191]	1054
[811]	1055	# --------------------------------------------------------------------
	1056	#D- Other specifications
	1057	# --------------------------------------------------------------------
	1058
[623]	1059	ulimit -s unlimited
[622]	1060
[623]	1061	IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]	1062	}
	1063
	1064	############################################################
	1065	# Desactivate Running Environnment Variables
	1066
[811]	1067	#D-#==================================================
	1068	#D-function IGCM_sys_desactiv_variables
	1069	#D-* Purpose: unset environement variables after execution
	1070	#D-* Examples:
	1071	#D-
[622]	1072	function IGCM_sys_desactiv_variables {
[623]	1073	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
	1074	if ( $DEBUG_sys ) ; then
	1075	echo "IGCM_sys_desactiv_variables"
	1076	fi
[811]	1077	# --------------------------------------------------------------------
	1078	#D- MPI specifications
	1079	# --------------------------------------------------------------------
	1080
	1081	# --------------------------------------------------------------------
	1082	#D- Other specifications
	1083	# --------------------------------------------------------------------
	1084
[623]	1085	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]	1086	}
	1087
[623]	1088	############################################################
[1238]	1089	# Update job headers to be used by the scheduler
	1090
	1091	#D-#==================================================
	1092	#D-function IGCM_sys_updateHeaders
	1093	#D-* Purpose: Update job headers to be used by the scheduler
	1094	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
	1095	#D-
	1096	function IGCM_sys_updateHeaders {
	1097	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
	1098	if ( $DEBUG_sys ) ; then
	1099	echo "IGCM_sys_updateHeaders"
	1100	fi
	1101	typeset file
	1102	file=$1
	1103
	1104	if [ ${executionType} -eq 1 ] ; then
	1105	# MPMD + MPI
	1106	sed -e "/::openMPthreads::/d" \
	1107	-e "s/::JobNumProcTot::/${coreNumber}/" \
[1256]	1108	-e "/#MSUB -x/d" \
	1109	-e "/--cpu_bind=none/d" \
[1238]	1110	${file} > ${file}.tmp
	1111
	1112	elif [ ${executionType} -eq 2 ] ; then
	1113	# MPMD + MPI + OMP
[1245]	1114	sed -e "/::openMPthreads::/d" \
[1238]	1115	-e "s/::JobNumProcTot::/${coreNumber}/" \
	1116	${file} > ${file}.tmp
	1117
	1118	elif [ ${executionType} -eq 3 ] ; then
	1119	# SPMD + MPI/OMP
	1120	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1121	-e "s/::JobNumProcTot::/${mpiTasks}/" \
[1240]	1122	-e "/#MSUB -x/d" \
[1238]	1123	-e "/--cpu_bind=none/d" \
	1124	${file} > ${file}.tmp
	1125
	1126	elif [ ${executionType} -eq 4 ] ; then
	1127	# SPMD + MPI only
	1128	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
	1129	-e "/::openMPthreads::/d" \
[1240]	1130	-e "/#MSUB -x/d" \
[1238]	1131	-e "/--cpu_bind=none/d" \
	1132	${file} > ${file}.tmp
	1133
	1134	elif [ ${executionType} -eq 5 ] ; then
	1135	# SPMD + OMP only
	1136	sed -e "s/::openMPthreads::/${openMPthreads}/" \
	1137	-e "/::JobNumProcTot::/d" \
[1240]	1138	-e "/#MSUB -x/d" \
[1238]	1139	-e "/--cpu_bind=none/d" \
	1140	${file} > ${file}.tmp
	1141
	1142	elif [ ${executionType} -eq 6 ] ; then
	1143	# SEQUENTIAL THEN
	1144	sed -e "s/::JobNumProcTot::/1/" \
	1145	-e "/::openMPthreads::/d" \
[1240]	1146	-e "/#MSUB -x/d" \
[1238]	1147	-e "/--cpu_bind=none/d" \
	1148	${file} > ${file}.tmp
	1149
	1150	fi
	1151
	1152	IGCM_sys_Mv ${file}.tmp ${file}
	1153
	1154	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
	1155	}
	1156
	1157	############################################################
[623]	1158	# Build MPI/OMP scripts run file (dummy function)
[622]	1159
[811]	1160	#D-#==================================================
	1161	#D-function IGCM_sys_build_run_file
	1162	#D-* Purpose: build run file (deprecated)
	1163	#D-* Examples:
	1164	#D-
[622]	1165	function IGCM_sys_build_run_file {
[811]	1166	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]	1167	}
	1168
	1169	############################################################
	1170	# Build MPI/OMP scripts
[811]	1171
	1172	#D-#==================================================
	1173	#D-function IGCM_sys_build_execution_scripts
	1174	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
	1175	#D-* Examples:
	1176	#D-
[623]	1177	function IGCM_sys_build_execution_scripts
[622]	1178	{
[623]	1179	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
	1180	if ( $DEBUG_sys ) ; then
	1181	echo "IGCM_sys_build_execution_scripts " $@
	1182	fi
[622]	1183
[623]	1184	EXECUTION=${HOST_MPIRUN_COMMAND}
[622]	1185
[1078]	1186	# MPMD mode
[623]	1187	if ( ${OK_PARA_MPMD} ) ; then
[622]	1188
[1381]	1189	if [ -f run_file ] ; then
	1190	IGCM_sys_Rm -f run_file
	1191	fi
	1192	touch run_file
	1193
[1078]	1194	# Only MPI (MPMD)
	1195	if ( ! ${OK_PARA_OMP} ) ; then
[622]	1196
[1078]	1197	# Build run_file
[622]	1198
[1078]	1199	# First loop on the components for the coupler ie oasis (only if oasis3)
	1200	# the coupler ie oasis3 must be the first one
	1201	for comp in ${config_ListOfComponents[*]} ; do
[622]	1202
[1078]	1203	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1204	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1205
[1078]	1206	# for CPL component only
	1207	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1208
[1078]	1209	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1210	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1211	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1212	fi
	1213	done
[622]	1214
[1078]	1215	# Then second loop on the components
	1216	for comp in ${config_ListOfComponents[*]} ; do
[622]	1217
[1078]	1218	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1219	eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]	1220
[1078]	1221	# Only if we really have an executable for the component and not the coupler ie oasis:
	1222	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]	1223
[1078]	1224	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1225	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1226	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
	1227	fi
	1228	done
[622]	1229
[1381]	1230	# MPI-OpenMP (MPMD)
	1231	# example of run_file
	1232	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
	1233	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
	1234	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
[622]	1235
[1078]	1236	else
[622]	1237
[1078]	1238	# Loop on the components
	1239	for comp in ${config_ListOfComponents[*]} ; do
[1000]	1240
[1078]	1241	eval ExeNameIn=\${config_Executable_${comp}[0]}
	1242	eval ExeNameOut=\${config_Executable_${comp}[1]}
[1000]	1243
[1078]	1244	# Not possible if oasis has an executable (i.e old version of oasis3)
	1245	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
	1246	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
	1247	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
	1248	IGCM_debug_Verif_Exit
	1249	fi
[1000]	1250
[1078]	1251	# Only if we really have an executable for the component :
	1252	if [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]	1253
[1078]	1254	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
	1255	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1000]	1256
[1381]	1257	# Build script files
	1258
[1078]	1259	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1260	echo "" >> script_${ExeNameOut}.ksh
	1261	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
	1262
	1263	# Check if the number of threads is correct
	1264	case ${comp_proc_omp_loc} in
	1265	2\|4\|8\|16)
	1266	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
	1267	;;
	1268	*)
	1269	IGCM_debug_Exit "ERROR with OMP parameters !"
	1270	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
	1271	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
	1272	IGCM_debug_Verif_Exit
	1273	;;
	1274	esac
[1381]	1275	### only for LMDZ?
[1078]	1276	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1277	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1278	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1279	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
[1381]	1280
[1078]	1281	fi
[1318]	1282
[1381]	1283	# to have out/err per process on different files
	1284	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
[1317]	1285	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[1078]	1286
[1381]	1287	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
[1078]	1288
[1381]	1289	# Complete run_file
[1000]	1290
[1381]	1291	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
	1292
[1078]	1293	fi
	1294	done
[1381]	1295
[1385]	1296	## variable added to stop after 60s instead of 600s by default.
	1297	## This is used when no error comes from executables and when something stopped an executable without notice.
	1298	export SLURM_WAIT=60
[1381]	1299
[1385]	1300	## module advised by TGCC (instead of 2 variables)
	1301	module load feature/bridge/heterogenous_mpmd
[1381]	1302
[1078]	1303	fi
[1000]	1304
[1381]	1305	# for both MPMD/MPI and MPMD/MPI/OpenMP mode
	1306
	1307	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
	1308
	1309	IGCM_sys_Chmod u+x run_file
	1310	if ( $DEBUG_sys ) ; then
	1311	echo "run_file contains : "
	1312	cat run_file
	1313	fi
	1314
[1078]	1315	# Only one executable (SPMD mode).
	1316	else
[1000]	1317
[622]	1318	for comp in ${config_ListOfComponents[*]} ; do
	1319
[685]	1320	# Only if we really have an executable for the component :
[623]	1321	eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]	1322	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]	1323
[1381]	1324	# Build script files
	1325
[685]	1326	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
	1327	echo "" >> script_${ExeNameOut}.ksh
	1328	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
	1329
	1330	if ( ${OK_PARA_OMP} ) ; then
	1331	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
	1332	echo "" >> script_${ExeNameOut}.ksh
	1333	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
	1334	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
	1335	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
	1336	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
	1337	fi
	1338
	1339	if ( ${OK_PARA_MPI} ) ; then
[623]	1340	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]	1341	# Default : ccc_mprun used if nb_proc gt 1
	1342	# to have out/err per process on different files
[1317]	1343	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
	1344	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1345	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]	1346	else
[685]	1347	# Default : ccc_mprun is NOT used if nb_proc eq 1
	1348	# to have out/err per process on different files
[1317]	1349	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
	1350	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
[685]	1351	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]	1352	fi
[685]	1353
	1354	IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
	1355	cat script_${ExeNameOut}.ksh
	1356
[623]	1357	fi
[622]	1358
	1359	done
	1360
[623]	1361	fi
[622]	1362
[623]	1363	IGCM_debug_Print 1 "sys Curie : execution command is "
	1364	IGCM_debug_Print 1 "$EXECUTION"
[622]	1365
[623]	1366	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]	1367	}
	1368
[933]	1369	#D-#==================================================
	1370	#D-function IGCM_sys_check_path
	1371	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
	1372	#D-* do not point to an important use directory. Stop immediately in that case.
	1373	#D-* Examples:
	1374	#D-
	1375	function IGCM_sys_check_path {
	1376	IGCM_debug_PushStack "IGCM_sys_check_path"
	1377	if ( $DEBUG_sys ) ; then
	1378	echo "IGCM_sys_check_path"
	1379	fi
[811]	1380
[933]	1381	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
	1382	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
	1383	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]	1384	IGCM_debug_Exit "This will stop the job"
[933]	1385	fi
	1386	IGCM_debug_PopStack "IGCM_sys_check_path"
	1387	}
	1388
[811]	1389	#D-#==================================================
	1390	#D-function IGCM_sys_check_quota
	1391	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
	1392	#D-* Examples:
	1393	#D-
[714]	1394	function IGCM_sys_check_quota {
[776]	1395	IGCM_debug_PushStack "IGCM_sys_check_quota"
	1396	if ( $DEBUG_sys ) ; then
	1397	echo "IGCM_sys_check_quota"
	1398	fi
	1399	# Limit of quota (in %)
	1400	limit_quota=90
	1401
	1402	# Check of the volume
[912]	1403	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
	1404	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
[776]	1405
	1406	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
	1407
	1408	unit_avail=${volume_avail: -1}
	1409	unit_quota=${volume_quota: -1}
	1410
	1411	if [ "${unit_quota}" = "*" ] ; then
[811]	1412	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1413	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1414	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1415	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1416	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1417	IGCM_debug_Verif_Exit
[714]	1418	fi
[776]	1419
	1420	temp_avail=${volume_avail%%${volume_avail: -1}*}
	1421	temp_quota=${volume_quota%%${volume_quota: -1}*}
	1422
	1423	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1424
	1425	# Convertion
	1426	if [ ${volume_avail: -1} = "T" ] ; then
	1427	(( temp_avail = temp_avail * 1000000000000 ))
	1428	elif [ ${volume_avail: -1} = "G" ] ; then
	1429	(( temp_avail = temp_avail * 1000000000 ))
	1430	elif [ ${volume_avail: -1} = "M" ] ; then
	1431	(( temp_avail = temp_avail * 1000000 ))
	1432	elif [ ${volume_avail: -1} = "k" ] ; then
	1433	(( temp_avail = temp_avail * 1000 ))
	1434	else
	1435	(( temp_avail = volume_avail ))
	1436	fi
	1437	if [ ${volume_quota: -1} = "T" ] ; then
	1438	(( temp_quota = temp_quota * 1000000000000 ))
	1439	elif [ ${volume_quota: -1} = "G" ] ; then
	1440	(( temp_quota = temp_quota * 1000000000 ))
	1441	elif [ ${volume_quota: -1} = "M" ] ; then
	1442	(( temp_quota = temp_quota * 1000000 ))
	1443	elif [ ${volume_quota: -1} = "k" ] ; then
	1444	(( temp_quota = temp_quota * 1000 ))
	1445	else
	1446	(( temp_quota = volume_quota ))
	1447	fi
	1448	fi
	1449
	1450	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1451	# echo "volume ratio is " $quota_volume
	1452
	1453	if [ ${quota_volume} -ge ${limit_quota} ] ; then
	1454	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
	1455	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
	1456	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1457	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1458	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1459	IGCM_debug_Verif_Exit
	1460	fi
	1461
	1462	fi
	1463
	1464	# Check of the number of inodes
	1465
[912]	1466	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
	1467	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
[776]	1468
	1469	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
	1470
	1471	unit_avail=${inode_avail: -1}
	1472	unit_quota=${inode_quota: -1}
	1473
	1474	if [ "${unit_quota}" = "*" ] ; then
[811]	1475	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1476	IGCM_debug_Print 1 "More than 100% of your quota is used"
	1477	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1478	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1479	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1480	IGCM_debug_Verif_Exit
[776]	1481	fi
	1482
	1483	temp_avail=${inode_avail%%${inode_avail: -1}*}
	1484	temp_quota=${inode_quota%%${inode_quota: -1}*}
	1485
	1486	if [ ! ${unit_avail} = ${unit_quota} ] ; then
	1487
	1488	# Convertion
	1489	if [ ${inode_avail: -1} = "T" ] ; then
	1490	(( temp_avail = temp_avail * 1000000000000 ))
	1491	elif [ ${inode_avail: -1} = "G" ] ; then
	1492	(( temp_avail = temp_avail * 1000000000 ))
	1493	elif [ ${inode_avail: -1} = "M" ] ; then
	1494	(( temp_avail = temp_avail * 1000000 ))
	1495	elif [ ${inode_avail: -1} = "k" ] ; then
	1496	(( temp_avail = temp_avail * 1000 ))
	1497	else
	1498	(( temp_avail = inode_avail ))
	1499	fi
	1500
	1501	if [ ${inode_quota: -1} = "T" ] ; then
	1502	(( temp_quota = temp_quota * 1000000000000 ))
	1503	elif [ ${inode_quota: -1} = "G" ] ; then
	1504	(( temp_quota = temp_quota * 1000000000 ))
	1505	elif [ ${inode_quota: -1} = "M" ] ; then
	1506	(( temp_quota = temp_quota * 1000000 ))
	1507	elif [ ${inode_quota: -1} = "k" ] ; then
	1508	(( temp_quota = temp_quota * 1000 ))
	1509	else
	1510	(( temp_quota = inode_quota ))
	1511	fi
	1512	fi
	1513	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
	1514	# echo "inode ratio is " $quota_inode
	1515
	1516	if [ ${quota_inode} -ge ${limit_quota} ] ; then
	1517	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
	1518	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
	1519	IGCM_debug_Print 1 "Use the ccc_quota command to check"
	1520	IGCM_debug_Print 1 "You must have more than 10% available to run"
	1521	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
	1522	IGCM_debug_Verif_Exit
	1523	fi
	1524	fi
	1525	IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]	1526	}
	1527
[773]	1528	#D-#==================================================
[1286]	1529	#D-function IGCM_sys_projectAccounting
	1530	#D-* Purpose: store project accounting information in a file
	1531	#D-* Examples:
	1532	#D-
	1533	function IGCM_sys_projectAccounting {
	1534	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
	1535	if ( $DEBUG_sys ) ; then
	1536	echo "IGCM_sys_check_quota"
	1537	fi
	1538
[1288]	1539	ssh curie91 /usr/bin/ccc_myproject > $1
[1286]	1540
	1541	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
	1542	}
	1543
	1544	#D-#==================================================
[1244]	1545	#D-function IGCM_sys_getJobSchedulerID
	1546	#D-* Purpose: Get the job ID during execution
	1547	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
	1548	#D-
	1549	function IGCM_sys_getJobSchedulerID {
	1550	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
	1551	if ( $DEBUG_sys ) ; then
	1552	echo "IGCM_sys_getJobSchedulerID"
	1553	fi
	1554
	1555	eval ${1}=${BRIDGE_MSUB_JOBID}
	1556
	1557	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
	1558	}
	1559
	1560	#D-#==================================================
[958]	1561	#D-function IGCM_sys_GetJobID
[1244]	1562	#D-* Purpose: Get the job ID from the JobName
[958]	1563	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
	1564	#D-
	1565	function IGCM_sys_GetJobID {
	1566	IGCM_debug_PushStack "IGCM_sys_GetJobID"
	1567	if ( $DEBUG_sys ) ; then
	1568	echo "IGCM_sys_GetJobID"
	1569	fi
	1570
	1571	# With -f option, the full job name is given in the last column
	1572	ID=$( ccc_mstat -f -u $2 \| \
	1573	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
	1574
	1575	eval ${3}=${ID}
	1576	IGCM_debug_PopStack "IGCM_sys_GetJobID"
	1577	}
	1578
	1579	#D-#==================================================
[773]	1580	#D-function IGCM_sys_CountJobInQueue
[1244]	1581	#D-* Purpose: Count number of users job
[773]	1582	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
	1583	#D-
	1584	function IGCM_sys_CountJobInQueue {
	1585	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
	1586	if ( $DEBUG_sys ) ; then
	1587	echo "IGCM_sys_CountJobInQueue"
	1588	fi
	1589
	1590	# With -f option, the full job name is given in the last column
[834]	1591	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]	1592
	1593	eval ${2}=${NbRun}
	1594
	1595	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
	1596	}
	1597
[955]	1598	#D-#==================================================
	1599	#D-function IGCM_sys_ListJobInQueue
[1244]	1600	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
[955]	1601	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
	1602	#D-
	1603	function IGCM_sys_ListJobInQueue {
	1604	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
	1605	if ( $DEBUG_sys ) ; then
	1606	echo "IGCM_sys_ListJobInQueue"
	1607	fi
	1608
	1609	# With -f option, the full job name is given in the last column
	1610	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
	1611	'( $2 == User && \
	1612	$NF != /TS/ && \
	1613	$NF !~ /PACK/ && \
	1614	$NF !~ /REBUILD/ && \
	1615	$NF !~ /pack/ ) \
	1616	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
	1617
	1618	eval set -A ${2} ${JobList[*]}
	1619
	1620	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
	1621	}
[1272]	1622
	1623	#D-#==================================================
	1624	#D-function IGCM_sys_atlas
	1625	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
	1626	#D-* Examples:
	1627	#D-
	1628	function IGCM_sys_atlas {
	1629	IGCM_debug_PushStack "IGCM_sys_atlas" $@
	1630	if ( $DEBUG_sys ) ; then
	1631	echo "IGCM_sys_atlas :" $@
	1632	fi
	1633
	1634	typeset status
	1635
	1636	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
	1637	status=$?
	1638	if [ ${status} -gt 0 ] ; then
	1639	echo "IGCM_sys_atlas : error code ${status}"
	1640	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1641	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
	1642	IGCM_debug_PopStack "IGCM_sys_atlas"
	1643	return 1
	1644	else
	1645	IGCM_debug_PopStack "IGCM_sys_atlas"
	1646	return 0
	1647	fi
	1648
	1649	IGCM_debug_PopStack "IGCM_sys_atlas"
	1650	}
	1651
[1380]	1652	#D-#==================================================
	1653	#D-function IGCM_sys_rebuild_nemo
	1654	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
	1655	#D-* Examples:
	1656	#D-
	1657
	1658	function IGCM_sys_rebuild_nemo {
	1659	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
	1660	if ( $DEBUG_sys ) ; then
	1661	echo "IGCM_sys_rebuild_nemo :" $@
	1662	fi
	1663
	1664	$( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
	1665
	1666	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
	1667	}
	1668

Note: See TracBrowser for help on using the repository browser.

Download in other formats: