Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1547

Last change on this file since 1547 was 1544, checked in by acosce, 3 years ago
Add case of simulation without IOS component so when IOS_OK_DEDICATED is not defined by libIGCM_config
File size: 65.7 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: 1512 $ Revision of last commit
7	# $Author:: acosce $ Author of last commit
8	# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Irene-amd
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host user names project maxCpuTime
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=irene-amd
71	# add default project on irene-amd
72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} \| cut -d@ -f1 )
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=ccc_msub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	if [ X${TaskType} = Xcomputing ] ; then
95	IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
96	else
97	module purge > /dev/null 2>&1
98	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
99	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_atlas_irene-amd > /dev/null 2>&1
100
101	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
102	export UVCDAT_ANONYMOUS_LOG=FALSE
103	fi
104
105	# FYI
106	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
107	[ ! X${TaskType} = Xchecking ] && module list
108
109	# For AddNoise and AddPertu3DOCE
110	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/bin
111
112	# For rebuild
113	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene-amd/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
114
115	#====================================================
116	# Host specific DIRECTORIES
117	#====================================================
118
119	# ============ CESIUM START ============ #
120
121	#====================================================
122	#- Mirror libIGCM from titane to cesium if needed
123	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
124	#if [ ! ${ROOTSYS} = "home" ] ; then
125	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
126	#else
127	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
128	#fi
129
130	#====================================================
131	#- libIGCM_POST
132	#if ( ${MirrorlibIGCM} ) ; then
133	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
134	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
135	#else
136	# typeset -r libIGCM_POST=${libIGCM}
137	#fi
138
139	# ============ CESIUM END ============ #
140
141	#====================================================
142	#- MirrorlibIGCM for frontend
143	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
144
145	#====================================================
146	#- libIGCM_POST for frontend
147	typeset -r libIGCM_POST=${libIGCM}
148
149	#====================================================
150	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
151	typeset -r R_EXE="${MODIPSL}/bin"
152
153	#====================================================
154	#- SUBMIT_DIR : submission dir
155	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
156
157	#====================================================
158	#- IN
159	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
160
161	#====================================================
162	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
163	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
164
165	#====================================================
166	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
167	typeset -r OUTCOMMAND_PATH=/tmp
168
169	#====================================================
170	#- HOST_MPIRUN_COMMAND
171	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
172
173	#====================================================
174	#- Max number of arguments passed to nco operator or demigration command
175	UNIX_MAX_LIMIT=360
176
177	#====================================================
178	#- set PackDefault to true on irene-amd
179	PackDefault=true
180
181	#====================================================
182	#- Default number of MPI task for IPSL coupled model
183	#- required for backward compatibility
184	#-
185	DEFAULT_NUM_PROC_OCE=5
186	DEFAULT_NUM_PROC_CPL=1
187	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
188
189	#====================================================
190	#- Number of computing cores per node
191	#-
192	NB_CORE_PER_NODE=128
193
194	#D-#==================================================
195	#D-function IGCM_sys_defineArchives
196	#D-* Purpose:
197	#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
198	#D-* Define ARCHIVE : Dedicated to large files
199	#D-* Define STORAGE : Dedicated to small/medium files
200	#D-* Define R_OUT : Output tree located on ARCHIVE
201	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
202	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
203	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
204	#D-* Examples:
205	#D-
206	function IGCM_sys_defineArchives {
207	IGCM_debug_PushStack "IGCM_sys_defineArchives"
208
209	# Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
210	if [ X${config_UserChoices_DataProject} = X ] \|\| [ X${config_UserChoices_DataProject} = DEFAULT ]; then
211	# Default option: Change the dfldatadir according to the project used for submission
212	# The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
213	if [ X"$( echo ${PROJECT} \| grep cmip6 )" != "X" ] ; then
214	module switch dfldatadir dfldatadir/gencmip6
215	IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
216	else
217	module switch dfldatadir dfldatadir/${PROJECT}
218	IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
219	fi
220	else
221	# Use the project set by the variable DataProject in section UserChoices in config.card
222	module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
223	IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
224	fi
225
226	#====================================================
227	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
228	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
229
230	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
231	#====================================================
232	#- ARCHIVE (dedicated to large files)
233	ARCHIVE=${config_UserChoices_ARCHIVE}
234	else
235	#====================================================
236	#- ARCHIVE (dedicated to large files)
237	ARCHIVE=${CCCSTOREDIR}
238	fi
239
240	if [ ! X${config_UserChoices_STORAGE} = X ]; then
241	#====================================================
242	#- STORAGE (dedicated to small/medium files)
243	STORAGE=${config_UserChoices_STORAGE}
244	else
245	#====================================================
246	#- STORAGE (dedicated to small/medium files)
247	STORAGE=${CCCWORKDIR}
248	fi
249
250	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
251	#====================================================
252	#- R_OUT
253	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
254
255	#====================================================
256	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
257	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
258
259	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
260
261	else
262	#====================================================
263	#- R_OUT
264	R_OUT=${ARCHIVE}/IGCM_OUT
265
266	#====================================================
267	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
268	R_FIG=${STORAGE}/IGCM_OUT
269	fi
270
271	#====================================================
272	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
273	CMIP6_BUF=${STORAGE}/IGCM_OUT
274
275	#====================================================
276	#- R_BUF (ONLY FOR double copy an scratch)
277	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
278
279	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
280	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
281	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
282
283	IGCM_debug_PopStack "IGCM_sys_defineArchives"
284	}
285
286	#D-#==================================================
287	#D-function IGCM_sys_RshArchive
288	#D-* Purpose: Archive rsh command
289	#D-* Examples:
290	#D-
291	function IGCM_sys_RshArchive {
292	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
293	/bin/ksh <<-EOF
294	${@}
295	EOF
296	status=$?
297	if [ ${status} -gt 0 ] ; then
298	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
299	IGCM_debug_Exit "IGCM_sys_RshArchive"
300	fi
301	IGCM_debug_PopStack "IGCM_sys_RshArchive"
302	}
303
304	#D-#==================================================
305	#D-function IGCM_sys_RshArchive_NoError
306	#D-* Purpose: Archive rsh command, without error
307	#D-* used only in monitoring.job
308	#D-* Examples:
309	#D-
310	function IGCM_sys_RshArchive_NoError {
311	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
312	/bin/ksh <<-EOF
313	${@} 2> /dev/null
314	EOF
315	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
316	}
317
318	#D-#==================================================
319	#D-function IGCM_sys_MkdirArchive
320	#D-* Purpose: Mkdir on Archive
321	#D-* Examples:
322	#D-
323	function IGCM_sys_MkdirArchive {
324	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
325	if ( $DEBUG_sys ) ; then
326	echo "IGCM_sys_MkdirArchive :" $@
327	fi
328	#- creation de repertoire sur le serveur fichier
329	if [ ! -d ${1} ]; then
330	\mkdir -p $1
331	status=$?
332
333	if [ ${status} -gt 0 ] ; then
334	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
335	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
336	fi
337	fi
338	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
339	}
340
341	#D-#==================================================
342	#D-function IGCM_sys_TestDirArchive
343	#D-* Purpose: Test Directory that must exists on Archive
344	#D-* Examples:
345	#D-
346	function IGCM_sys_TestDirArchive {
347	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
348	if ( $DEBUG_sys ) ; then
349	echo "IGCM_sys_TestDirArchive :" $@
350	fi
351	typeset ExistFlag
352	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
353	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
354	return ${ExistFlag}
355	}
356
357	#D-#==================================================
358	#D-function IGCM_sys_IsFileArchived
359	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
360	#D-* Examples:
361	#D-
362	function IGCM_sys_IsFileArchived {
363	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
364	if ( $DEBUG_sys ) ; then
365	echo "IGCM_sys_IsFileArchived :" $@
366	fi
367	typeset IsArchivedFlag
368	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
369	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
370
371	return ${IsArchivedFlag}
372	}
373
374	#D-#==================================================
375	#D-function IGCM_sys_TestFileArchive
376	#D-* Purpose: Test file that must NOT EXISTS on Archive
377	#D-* Examples:
378	#D-
379	function IGCM_sys_TestFileArchive {
380	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
381	typeset ExistFlag
382	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
383	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
384
385	return ${ExistFlag}
386	}
387
388	#D-#==================================================
389	#D-function IGCM_sys_CountFileArchive
390	#D-* Purpose: Count files on Archive filesystem
391	#D-* Examples:
392	#D-
393	function IGCM_sys_CountFileArchive {
394	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
395	ls ${@} 2>/dev/null \| wc -l
396	if [ $? -gt 0 ] ; then
397	echo "IGCM_sys_CountFileArchive : erreur."
398	fi
399	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
400	}
401
402	#D-#==================================================
403	#D-function IGCM_sys_Tree
404	#D-* Purpose: Tree directories with files on ${ARCHIVE}
405	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
406	#D-
407	function IGCM_sys_Tree {
408	IGCM_debug_PushStack "IGCM_sys_Tree" $@
409	if ( $DEBUG_sys ) ; then
410	echo "IGCM_sys_Tree :" $@
411	fi
412
413	\ls -lR ${@}
414
415	IGCM_debug_PopStack "IGCM_sys_Tree"
416	}
417
418	#D-#==================================================
419	#D-function IGCM_sys_Qsub
420	#D-* Purpose: Qsub new job
421	#D-* Examples:
422	#D-
423	function IGCM_sys_Qsub {
424	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
425	if ( $DEBUG_sys ) ; then
426	echo "IGCM_sys_Qsub :" $@
427	fi
428	typeset options status
429	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
430
431	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
432	status=$?
433
434	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
435	if [ ${status} -gt 0 ] ; then
436	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
437	IGCM_debug_Exit "IGCM_sys_Qsub"
438	else
439	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
440	fi
441	IGCM_debug_PopStack "IGCM_sys_Qsub"
442	}
443
444	#D-#==================================================
445	#D-function IGCM_sys_QsubPost
446	#D-* Purpose: Qsub new job on scalaire
447	#D-* Examples:
448	#D-
449	function IGCM_sys_QsubPost {
450	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
451	if ( $DEBUG_sys ) ; then
452	echo "IGCM_sys_QsubPost :" $@
453	fi
454	typeset options status
455	options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
456
457	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
458	status=$?
459
460	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
461	if [ ${status} -gt 0 ] ; then
462	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
463	IGCM_debug_Exit "IGCM_sys_QsubPost"
464	else
465	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
466	fi
467	IGCM_debug_PopStack "IGCM_sys_QsubPost"
468	}
469
470	#D-*************************
471	#D- File transfer functions
472	#D-*************************
473	#D-
474
475	#D-#==================================================
476	#D-function IGCM_sys_RmRunDir
477	#D-* Purpose: rm tmpdir (dummy function most of the time batch
478	#D- scheduler will do the job)
479	#D-* Examples:
480	#D-
481	function IGCM_sys_RmRunDir {
482	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
483	if ( $DEBUG_sys ) ; then
484	echo "IGCM_sys_RmRunDir :" $@
485	fi
486
487	typeset status
488
489	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
490	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
491	status=$?
492
493	if [ ${status} -gt 0 ] ; then
494	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
495	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
496	IGCM_debug_Exit "IGCM_sys_RmRunDir"
497	else
498	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
499	fi
500	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
501	}
502
503	#D-#==================================================
504	#D-function IGCM_sys_Put_Dir
505	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
506	#D-* Examples:
507	#D-
508	function IGCM_sys_Put_Dir {
509	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
510	if ( $DEBUG_sys ) ; then
511	echo "IGCM_sys_Put_Dir :" $@
512	fi
513	if [ $DRYRUN = 0 ]; then
514	if [ ! -d ${1} ] ; then
515	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
516	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
517	return
518	fi
519
520	typeset status
521
522	# Only if we use rsync
523	#IGCM_sys_TestDirArchive $( dirname $2 )
524	#
525	#USUAL WAY
526	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
527	status=$?
528
529	if [ ${status} -gt 0 ] ; then
530	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
531	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
532	IGCM_debug_Exit "IGCM_sys_Put_Dir"
533	else
534	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
535	fi
536	fi
537	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
538	}
539
540	#D-#==================================================
541	#D-function IGCM_sys_Get_Dir
542	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
543	#D-* Examples:
544	#D-
545	function IGCM_sys_Get_Dir {
546	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
547	if ( $DEBUG_sys ) ; then
548	echo "IGCM_sys_Get_Dir :" $@
549	fi
550	if [ $DRYRUN = 0 ]; then
551	typeset NB_ESSAI DELAI status i
552	# number of tentative
553	NB_ESSAI=3
554	# time delay between tentative
555	DELAI=2
556
557	#
558	# USUAL WAY
559	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
560	ccc_hsm get -r $1
561
562	i=0
563	while [ $i -lt $NB_ESSAI ] ; do
564	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
565	status=$?
566	if [ ${status} -gt 0 ] ; then
567	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
568	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
569	sleep $DELAI
570	else
571	break
572	fi
573	(( i = i + 1 ))
574	done
575
576	if [ ${status} -gt 0 ] ; then
577	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
578	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
579	IGCM_debug_Exit "IGCM_sys_Get_Dir"
580	else
581	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
582	fi
583	fi
584	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
585	}
586
587	#D-#==================================================
588	#D-function IGCM_sys_Put_Rest
589	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
590	#D- File and target directory must exist.
591	#D-* Examples:
592	#D-
593	function IGCM_sys_Put_Rest {
594	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
595	if ( $DEBUG_sys ) ; then
596	echo "IGCM_sys_Put_Rest :" $@
597	fi
598	if [ $DRYRUN = 0 ]; then
599	if [ ! -f ${1} ] ; then
600	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
601	IGCM_debug_Exit "IGCM_sys_Put_Rest"
602	fi
603
604	typeset status
605	#
606	# USUAL WAY
607	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
608	status=$?
609
610	# #RSYNC WITH NETWORK SSH CALL
611	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
612	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
613
614	# #RSYNC WITH NFS USE
615	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617
618	# status=$?
619	# IGCM_sys_Rsync_out $status
620
621	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
622	# (( status=status+$? ))
623
624	if [ ${status} -gt 0 ] ; then
625	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
626	[ -f ${1} ] && ls -l ${1}
627	[ -f ${2} ] && ls -l ${2}
628	[ -f ${2}/${1} ] && ls -l ${2}/${1}
629	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
630	IGCM_debug_Exit "IGCM_sys_Put_Rest"
631	else
632
633	if [ X${JobType} = XRUN ] ; then
634	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
635	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
636	fi
637
638	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
639	fi
640	fi
641	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
642	}
643
644	#D-#==================================================
645	#D-function IGCM_sys_Put_Out
646	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
647	#D-* Examples:
648	#D-
649	function IGCM_sys_Put_Out {
650	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
651	if ( $DEBUG_sys ) ; then
652	echo "IGCM_sys_Put_Out :" $@
653	fi
654
655	typeset NB_ESSAI DELAI status i exist skip
656	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
657
658	# number of tentative
659	NB_ESSAI=3
660	# time delay between tentative
661	DELAI=2
662
663	if [ $DRYRUN = 0 ]; then
664	if [ ! -f ${1} ] ; then
665	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
666	IGCM_debug_PopStack "IGCM_sys_Put_Out"
667	return 1
668	fi
669	#
670	IGCM_sys_MkdirArchive $( dirname $2 )
671	#
672	exist=false
673	skip=false
674	if [ -f $2 ] ; then
675	IGCM_debug_Print 1 "$2 already exist"
676	ccc_hsm get $2
677	exist=true
678	if [ "X$( diff $1 $2 )" = X ] ; then
679	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
680	skip=true
681	else
682	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
683	skip=false
684	fi
685	fi
686	#
687	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
688	IGCM_sys_Chmod u+w $2
689	fi
690
691	if [ X${skip} = Xfalse ] ; then
692	i=0
693	while [ $i -lt $NB_ESSAI ] ; do
694	# Identify file system
695	fileDeviceNumberInHex=$( stat -c %d $1 )
696	status=$?
697	if [ ${status} -gt 0 ] ; then
698	IGCM_debug_Exit "IGCM_sys_Put_Out"
699	fi
700	# Identify file system
701	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
702	status=$?
703	if [ ${status} -gt 0 ] ; then
704	IGCM_debug_Exit "IGCM_sys_Put_Out"
705	fi
706
707	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
708	# They are not on the same device. USUAL WAY
709	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
710	status=$?
711	else
712	# They are on the same device. NOT SO USUAL WAY
713	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
714	status=$?
715	fi
716	if [ ${status} -gt 0 ]; then
717	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
718	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
719	[ -f ${1} ] && ls -l ${1}
720	[ -f ${2} ] && ls -l ${2}
721	[ -f ${2}/${1} ] && ls -l ${2}/${1}
722	sleep $DELAI
723	else
724	break
725	fi
726	(( i = i + 1 ))
727	done
728	fi
729
730	# #RSYNC WITH NETWORK SSH CALL
731	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
732	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
733
734	# #RSYNC WITH NFS USE
735	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
736	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
737
738	# status=$?
739	# IGCM_sys_Rsync_out $status
740
741	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
742	# (( status=status+$? ))
743
744	if [ ${status} -gt 0 ] ; then
745	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
746	[ -f ${1} ] && ls -l ${1}
747	[ -f ${2} ] && ls -l ${2}
748	[ -f ${2}/${1} ] && ls -l ${2}/${1}
749	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
750	IGCM_debug_Exit "IGCM_sys_Put_Out"
751	else
752
753	if [ X${JobType} = XRUN ] ; then
754	if [ X${3} = X ] ; then
755	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
756	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
757	fi
758	fi
759
760	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
761	fi
762	fi
763	IGCM_debug_PopStack "IGCM_sys_Put_Out"
764	return 0
765	}
766
767	#D-#==================================================
768	#D-function IGCM_sys_Get
769	#D-* Purpose: Get a file from ${ARCHIVE}
770	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
771	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
772	function IGCM_sys_Get {
773	IGCM_debug_PushStack "IGCM_sys_Get" $@
774
775	typeset DEST dm_liste target file_work
776	typeset NB_ESSAI DELAI status i
777
778	if ( $DEBUG_sys ) ; then
779	echo "IGCM_sys_Get :" $@
780	fi
781
782	# number of tentative
783	NB_ESSAI=3
784	# time delay between tentative
785	DELAI=2
786
787	if [ $DRYRUN -le 2 ]; then
788	if [ X${1} = X'/l' ] ; then
789	eval set +A dm_liste \${${2}}
790	else
791	eval set +A dm_liste ${1}
792	fi
793	eval DEST=\${${#}}
794	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
795	status=$?
796	if [ ${status} -gt 0 ] ; then
797	echo "WARNING IGCM_sys_Get : error code ${status}"
798	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
799	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
800	fi
801
802	#if [ ${status} -gt 0 ] ; then
803	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
804	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
805	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
806	# sleep 30
807	# echo "We try another time"
808	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
809	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
810	# status=$?
811	# if [ ${status} -gt 0 ] ; then
812	# echo "ERROR IGCM_sys_Get : again demigration error :"
813	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
814	# IGCM_debug_Exit "IGCM_sys_Get"
815	# fi
816	# else
817	# echo "ERROR IGCM_sys_Get : demigration error :"
818	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
819	# IGCM_debug_Exit "IGCM_sys_Get"
820	# fi
821	#fi
822
823	# #RSYNC WITH NETWORK SSH CALL
824	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
825	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
826
827	# #RSYNC WITH NFS USE
828	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
829	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
830
831	# status=$?
832	# IGCM_sys_Rsync_out $status
833
834	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
835	# (( status=status+$? ))
836
837	#USUAL WAY
838	if [ X${1} = X'/l' ] ; then
839	for target in ${dm_liste[*]} ; do
840	local_file=$( basename ${target} )
841	# test if the target file is present before the loop
842	IGCM_sys_TestFileArchive ${target}
843	status=$?
844	if [ ${status} -gt 0 ] ; then
845	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
846	IGCM_debug_Exit "IGCM_sys_Get"
847	else
848	i=0
849	while [ $i -lt $NB_ESSAI ] ; do
850	#if [ X${DoLink} = Xtrue ] ; then
851	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
852	# status=$?
853	# else
854	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
855	# status=$?
856	#fi
857	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
858	status=$?
859	if [ ${status} -gt 0 ]; then
860	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
861	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
862	sleep $DELAI
863	else
864	break
865	fi
866	(( i = i + 1 ))
867	done
868	if [ ${status} -gt 0 ] ; then
869	echo "IGCM_sys_Get : error"
870	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
871	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
872	IGCM_debug_Exit "IGCM_sys_Get"
873	else
874	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
875	fi
876	fi
877	done
878	else
879	i=0
880	while [ $i -lt $NB_ESSAI ] ; do
881	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
882	status=$?
883	if [ ${status} -gt 0 ]; then
884	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
885	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
886	sleep $DELAI
887	else
888	break
889	fi
890	(( i = i + 1 ))
891	done
892	if [ ${status} -gt 0 ] ; then
893	echo "IGCM_sys_Get : error"
894	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
895	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
896	IGCM_debug_Exit "IGCM_sys_Get"
897	else
898	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899	fi
900	fi
901	fi
902	IGCM_debug_PopStack "IGCM_sys_Get"
903	}
904
905	#D-#==================================================
906	#D-function IGCM_sys_GetDate_Monitoring
907	#D-* Purpose: get the last year for which the monitoring has been computed
908	#D-* Examples:
909	#D-
910	function IGCM_sys_GetDate_Monitoring {
911	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
912	if ( $DEBUG_sys ) ; then
913	echo "IGCM_sys_GetDate_Monitoring :" $@
914	fi
915
916	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
917
918	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
919	}
920
921	#D-#==================================================
922	#D-function IGCM_sys_Dods_Rm
923	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
924	#D-* Examples:
925	#D-
926	function IGCM_sys_Dods_Rm {
927	if ( $DEBUG_sys ) ; then
928	echo "IGCM_sys_Dods_Rm :" $@
929	fi
930	typeset status
931	if [ $DRYRUN = 0 ]; then
932
933	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
934	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
935	# echo "Nothing has been done."
936	# return
937	# fi
938
939	if [ "$#" -eq 1 ]; then
940	$(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
941	status=$?
942	else
943	$(ccc_home -u igcmg)/Tools/irene-amd/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
944	status=$?
945	fi
946
947	# if [ ${status} -gt 0 ] ; then
948	# echo "IGCM_sys_Dods_Rm : error."
949	# cat out_dods_rm
950	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
951	# else
952	# rm out_dods_rm
953	# fi
954
955	fi
956	return $status
957	}
958
959	#D-#==================================================
960	#D-function IGCM_sys_Dods_Cp
961	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
962	#D-* Examples:
963	#D-
964	function IGCM_sys_Dods_Cp {
965	if ( $DEBUG_sys ) ; then
966	echo "IGCM_sys_Dods_Cp :" $@
967	fi
968	typeset status
969	if [ $DRYRUN = 0 ]; then
970
971	# if [ ! -d ${R_SAVE}/${1} ] ; then
972	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
973	# echo "Nothing has been done."
974	# return
975	# fi
976
977	$(ccc_home -u igcmg)/Tools/irene-amd/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
978	status=$?
979
980	# if [ ${status} -gt 0 ] ; then
981	# echo "IGCM_sys_Dods_Cp : error."
982	# cat out_dods_cp
983	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
984	# else
985	# rm out_dods_cp
986	# fi
987
988	fi
989	return $status
990	}
991
992	#D-#==================================================
993	#D-function IGCM_sys_Put_Dods
994	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
995	#D-* Examples:
996	#D-
997	function IGCM_sys_Put_Dods {
998	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
999	if ( $DEBUG_sys ) ; then
1000	echo "IGCM_sys_Put_Dods :" $@
1001	fi
1002	typeset status
1003	if [ $DRYRUN = 0 ]; then
1004	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1005	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1006	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1007	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1008	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1009	return
1010	fi
1011
1012	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1013	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1014	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1015	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1016	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1017	return
1018	fi
1019	#
1020	if [ -d ${R_SAVE}/${1} ] ; then
1021	cd ${R_SAVE}
1022	elif [ -d ${R_FIGR}/${1} ] ; then
1023	cd ${R_FIGR}
1024	fi
1025
1026	IGCM_sys_Dods_Rm ${1}
1027	IGCM_sys_Dods_Cp ${1}
1028	status=0
1029
1030	if [ ${status} -gt 0 ] ; then
1031	echo "IGCM_sys_Put_Dods : error."
1032	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1033	fi
1034	fi
1035	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1036	}
1037
1038	##############################################################
1039	# REBUILD OPERATOR
1040
1041	#D-#==================================================
1042	#D-function IGCM_sys_sync
1043	#D-* Purpose: flush buffer on disk
1044	#D-* Examples:
1045	#D-
1046	function IGCM_sys_sync {
1047	IGCM_debug_PushStack "IGCM_sys_sync" $@
1048	if ( $DEBUG_sys ) ; then
1049	echo "IGCM_sys_sync :" $@
1050	fi
1051
1052	/bin/sync
1053
1054	IGCM_debug_PopStack "IGCM_sys_sync"
1055	}
1056
1057	############################################################
1058	# Activate Running Environnment Variables
1059
1060	#D-#==================================================
1061	#D-function IGCM_sys_activ_variables
1062	#D-* Purpose: set environement variables prior to execution
1063	#D-* Examples:
1064	#D-
1065	function IGCM_sys_activ_variables {
1066	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1067	if ( $DEBUG_sys ) ; then
1068	echo "IGCM_sys_activ_variables"
1069	fi
1070
1071	# --------------------------------------------------------------------
1072	#D- MODULE specifications
1073	# --------------------------------------------------------------------
1074	# Source the file EnvFile containing all module load needed to run the model.
1075	# EnvFile can be specified in config.card. If this is the case and if the file exists,
1076	# it will be used. Otherwise a default file will be used.
1077	if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1078	# EnvFile is set in config.card and the file exists
1079	IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1080	EnvFile=${config_UserChoices_EnvFile}
1081	else
1082	IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1083	module purge > /dev/null 2>&1
1084	if [ X${compilerVersion} = XV19 ] ; then
1085	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1086	EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1087	else
1088	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1089	fi
1090	fi
1091	IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"
1092	IGCM_debug_Print 1 ${EnvFile}
1093	. ${EnvFile}
1094
1095	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1096	module list
1097
1098	# --------------------------------------------------------------------
1099	#D- MPI specifications
1100	# --------------------------------------------------------------------
1101
1102	# --------------------------------------------------------------------
1103	#D- Other specifications
1104	# --------------------------------------------------------------------
1105
1106	ulimit -s unlimited
1107
1108	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1109	}
1110
1111	############################################################
1112	# Desactivate Running Environnment Variables
1113
1114	#D-#==================================================
1115	#D-function IGCM_sys_desactiv_variables
1116	#D-* Purpose: unset environement variables after execution
1117	#D-* Examples:
1118	#D-
1119	function IGCM_sys_desactiv_variables {
1120	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1121	if ( $DEBUG_sys ) ; then
1122	echo "IGCM_sys_desactiv_variables"
1123	fi
1124	# --------------------------------------------------------------------
1125	#D- MPI specifications
1126	# --------------------------------------------------------------------
1127
1128	# --------------------------------------------------------------------
1129	#D- Other specifications
1130	# --------------------------------------------------------------------
1131
1132	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1133	}
1134
1135	############################################################
1136	# Update job headers to be used by the scheduler
1137
1138	#D-#==================================================
1139	#D-function IGCM_sys_updateHeaders
1140	#D-* Purpose: Update job headers to be used by the scheduler
1141	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1142	#D-
1143	function IGCM_sys_updateHeaders {
1144	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1145	if ( $DEBUG_sys ) ; then
1146	echo "IGCM_sys_updateHeaders"
1147	fi
1148	typeset file
1149	file=$1
1150
1151	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1152	# Compute the number of core to add to force the use of a new node for xios servers
1153
1154	if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1155	(( IOS_OK_DEDICATED = 0 ))
1156	fi
1157
1158
1159	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1160	(( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1161	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1162	(( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1163	(( coreNumber = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1164	fi
1165
1166	if [ ${executionType} -eq 1 ] ; then
1167	# MPMD + MPI
1168	sed -e "/::openMPthreads::/d" \
1169	-e "s/::JobNumProcTot::/${coreNumber}/" \
1170	${file} > ${file}.tmp
1171
1172	elif [ ${executionType} -eq 2 ] ; then
1173	# MPMD + MPI + OMP :ccc_mprun
1174	sed -e "/::openMPthreads::/d" \
1175	-e "s/::JobNumProcTot::/${coreNumber}/" \
1176	${file} > ${file}.tmp
1177
1178	elif [ ${executionType} -eq 3 ] ; then
1179	# SPMD + MPI/OMP
1180	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1181	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1182	-e "/#MSUB -x/d" \
1183	${file} > ${file}.tmp
1184
1185	elif [ ${executionType} -eq 4 ] ; then
1186	# SPMD + MPI only
1187	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1188	-e "/::openMPthreads::/d" \
1189	-e "/#MSUB -x/d" \
1190	${file} > ${file}.tmp
1191
1192	elif [ ${executionType} -eq 5 ] ; then
1193	# SPMD + OMP only
1194	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1195	-e "/::JobNumProcTot::/d" \
1196	-e "/#MSUB -x/d" \
1197	${file} > ${file}.tmp
1198
1199	elif [ ${executionType} -eq 6 ] ; then
1200	# SEQUENTIAL THEN
1201	sed -e "s/::JobNumProcTot::/1/" \
1202	-e "/::openMPthreads::/d" \
1203	-e "/#MSUB -x/d" \
1204	${file} > ${file}.tmp
1205
1206	fi
1207
1208	IGCM_sys_Mv ${file}.tmp ${file}
1209
1210	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1211	}
1212
1213	############################################################
1214	# Build MPI/OMP scripts run file (dummy function)
1215
1216	#D-#==================================================
1217	#D-function IGCM_sys_build_run_file
1218	#D-* Purpose: build run file (deprecated)
1219	#D-* Examples:
1220	#D-
1221	function IGCM_sys_build_run_file {
1222	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1223	}
1224
1225	############################################################
1226	# Build MPI/OMP scripts
1227
1228	#D-#==================================================
1229	#D-function IGCM_sys_build_execution_scripts
1230	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1231	#D-* Examples:
1232	#D-
1233	function IGCM_sys_build_execution_scripts
1234	{
1235	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1236	if ( $DEBUG_sys ) ; then
1237	echo "IGCM_sys_build_execution_scripts " $@
1238	fi
1239
1240	EXECUTION=${HOST_MPIRUN_COMMAND}
1241
1242	if ( ${OK_PARA_MPMD} ) ; then
1243
1244	# MPMD mode
1245	# 1 MPI only
1246	# 2 MPI/OpenMP mpirun method
1247	# 3 MPI/OpenMP ccc_mprun method
1248
1249	if [ -f run_file ] ; then
1250	IGCM_sys_Rm -f run_file
1251	fi
1252	touch run_file
1253
1254	# case 1 : Only MPI (MPMD)
1255	if ( ! ${OK_PARA_OMP} ) ; then
1256
1257	# Build run_file
1258
1259	# First loop on the components for the coupler ie oasis (only if oasis3)
1260	# the coupler ie oasis3 must be the first one
1261	for comp in ${config_ListOfComponents[*]} ; do
1262
1263	eval ExeNameIn=\${config_Executable_${comp}[0]}
1264	eval ExeNameOut=\${config_Executable_${comp}[1]}
1265
1266	# for CPL component only
1267	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1268
1269	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1270	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1271	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1272	fi
1273	done
1274
1275	# Then second loop on the components
1276	for comp in ${config_ListOfComponents[*]} ; do
1277
1278	eval ExeNameIn=\${config_Executable_${comp}[0]}
1279	eval ExeNameOut=\${config_Executable_${comp}[1]}
1280
1281	# Only if we really have an executable for the component and not the coupler ie oasis:
1282	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1283
1284	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1285	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1286	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1287
1288	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1289	# Compute the number of core to add to force the use of a new node for xios servers
1290	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1291	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1292	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1293	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1294	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1295	if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1296	echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1297	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1298	echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1299	fi
1300	else
1301	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1302	fi
1303	else
1304	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1305	fi
1306	else
1307	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1308	fi
1309
1310
1311	fi
1312	done
1313
1314	## module advised by TGCC (instead of 2 variables)
1315	module load feature/bridge/heterogenous_mpmd
1316
1317	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1318
1319	IGCM_sys_Chmod u+x run_file
1320	if ( $DEBUG_sys ) ; then
1321	echo "run_file contains : "
1322	cat run_file
1323	fi
1324
1325	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1326
1327	# 2 MPI/OpenMP mpirun method
1328
1329	# Use of mpirun instead of ccc_mprun
1330	EXECUTION="time mpirun"
1331
1332	# Hosts treatment
1333	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1334
1335	i=0
1336	rm -f hosts rankfile
1337	IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1338	for nodes in `cat hosts.tmp`
1339	do
1340	host[$i]=$nodes
1341	echo "${host[$i]}" >> hosts
1342	IGCM_debug_Print 1 ${host[$i]}
1343	i=$((i+1))
1344	done
1345	rm -f hosts.tmp
1346
1347	listnodes=${host[*]}
1348
1349	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1350
1351	# Initialisation
1352	rank=0
1353	current_core=0
1354	init_exec=n
1355
1356	# Loop on the components
1357	for comp in ${config_ListOfComponents[*]} ; do
1358
1359	eval ExeNameIn=\${config_Executable_${comp}[0]}
1360	eval ExeNameOut=\${config_Executable_${comp}[1]}
1361
1362	# Not possible if oasis has an executable (i.e old version of oasis3)
1363	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1364	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1365	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1366	IGCM_debug_Verif_Exit
1367	fi
1368
1369	# Only if we really have an executable for the component :
1370	if [ "X${ExeNameOut}" != X\"\" ] ; then
1371
1372	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1373	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1374	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1375
1376	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1377	echo "" >> script_${ExeNameOut}.ksh
1378	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1379
1380	# Check if the number of threads is correct
1381	case ${comp_proc_omp_loc} in
1382	2\|4\|8\|16)
1383	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1384	;;
1385	*)
1386	IGCM_debug_Exit "ERROR with OMP parameters !"
1387	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1388	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1389	IGCM_debug_Verif_Exit
1390	;;
1391	esac
1392	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1393	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1394	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1395	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1396	fi
1397
1398	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1399	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1400	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1401	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1402	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1403
1404	if [ ${init_exec} = y ] ; then
1405	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1406	else
1407	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1408	init_exec=y
1409	fi
1410
1411	# Build rankfile : method used to assign cores and nodes for the MPI process
1412	# Ex :
1413	#rank 0=curie5296 slot=0,1,2,3
1414	#rank 1=curie5296 slot=4,5,6,7
1415	# Example of final command :
1416	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1417	# with script_lmdz.x.ksh :
1418	# #!/bin/ksh
1419	#export KMP_STACKSIZE=3g
1420	#export KMP_LIBRARY=turnaround
1421	#export MKL_SERIAL=YES
1422	#OMP_NUM_THREADS=4
1423	#./lmdz.x
1424	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1425	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1426	else
1427	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1428	fi
1429	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1430	(( index_host = current_core / NB_CORE_PER_NODE ))
1431	host_value=${host[${index_host}]}
1432	(( slot = current_core % NB_CORE_PER_NODE ))
1433	virg=","
1434	string_final=""
1435	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1436	string=$index$virg
1437	string_final=$string_final$string
1438	done
1439	string_final=$( echo $string_final \| sed "s/.$//" )
1440	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1441	(( rank = rank + 1 ))
1442	(( current_core = current_core + offset_comp_proc_loc ))
1443	done
1444	fi
1445	done
1446
1447	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1448
1449	# 3 MPI/OpenMP ccc_mprun method
1450
1451	# MPI-OpenMP (MPMD)
1452	# example of run_file
1453	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1454	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1455	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1456
1457	# Loop on the components
1458	for comp in ${config_ListOfComponents[*]} ; do
1459
1460	eval ExeNameIn=\${config_Executable_${comp}[0]}
1461	eval ExeNameOut=\${config_Executable_${comp}[1]}
1462
1463	# Not possible if oasis has an executable (i.e old version of oasis3)
1464	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1465	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1466	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1467	IGCM_debug_Verif_Exit
1468	fi
1469
1470	# Only if we really have an executable for the component :
1471	if [ "X${ExeNameOut}" != X\"\" ] ; then
1472
1473	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1474	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1475	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1476
1477	# Compute number of cores per process
1478	(( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc ))
1479
1480	# Build script files
1481
1482	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1483	echo "" >> script_${ExeNameOut}.ksh
1484	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1485
1486	# Check if the number of threads is correct
1487	case ${comp_proc_omp_loc} in
1488	2\|4\|8\|16)
1489	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1490	;;
1491	*)
1492	IGCM_debug_Exit "ERROR with OMP parameters !"
1493	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1494	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1495	IGCM_debug_Verif_Exit
1496	;;
1497	esac
1498	### only for LMDZ?
1499	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1500	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1501	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1502	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1503
1504	fi
1505
1506	# to have out/err per process on different files
1507	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1508	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1509
1510	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1511
1512	# Complete run_file
1513
1514	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1515	# Compute the number of core to add to force the use of a new node for xios servers
1516	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1517	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1518	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc ))
1519	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1520	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1521	if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1522	echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1523	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1524	echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1525	fi
1526	else
1527	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1528	fi
1529	else
1530	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1531	fi
1532	else
1533	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1534	fi
1535
1536	fi
1537	done
1538
1539	## variable added to stop after 60s instead of 600s by default.
1540	## This is used when no error comes from executables and when something stopped an executable without notice.
1541	export SLURM_WAIT=60
1542
1543	## module advised by TGCC (instead of 2 variables)
1544	module load feature/bridge/heterogenous_mpmd
1545
1546	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1547
1548	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1549
1550	IGCM_sys_Chmod u+x run_file
1551	if ( $DEBUG_sys ) ; then
1552	echo "run_file contains : "
1553	cat run_file
1554	fi
1555
1556	else
1557
1558	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1559
1560	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1561
1562	else
1563	# Only one executable (SPMD mode).
1564
1565	for comp in ${config_ListOfComponents[*]} ; do
1566
1567	# Only if we really have an executable for the component :
1568	eval ExeNameOut=\${config_Executable_${comp}[1]}
1569	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1570
1571	# Build script files
1572
1573	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1574	echo "" >> script_${ExeNameOut}.ksh
1575	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1576
1577	if ( ${OK_PARA_OMP} ) ; then
1578	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1579	echo "" >> script_${ExeNameOut}.ksh
1580	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1581	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1582	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1583	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1584	fi
1585
1586	if ( ${OK_PARA_MPI} ) ; then
1587	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1588	# Default : ccc_mprun used if nb_proc gt 1
1589	# to have out/err per process on different files
1590	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1591	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1592	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1593	else
1594	# Default : ccc_mprun is NOT used if nb_proc eq 1
1595	# to have out/err per process on different files
1596	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1597	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1598	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1599	fi
1600
1601	IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1602	cat script_${ExeNameOut}.ksh
1603
1604	fi
1605
1606	done
1607
1608	fi # ${OK_PARA_MPMD}
1609
1610	IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1611	IGCM_debug_Print 1 "$EXECUTION"
1612
1613	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1614	}
1615
1616	#D-#==================================================
1617	#D-function IGCM_sys_check_path
1618	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1619	#D-* do not point to an important use directory. Stop immediately in that case.
1620	#D-* Examples:
1621	#D-
1622	function IGCM_sys_check_path {
1623	IGCM_debug_PushStack "IGCM_sys_check_path"
1624	if ( $DEBUG_sys ) ; then
1625	echo "IGCM_sys_check_path"
1626	fi
1627
1628	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1629	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1630	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1631	IGCM_debug_Exit "This will stop the job"
1632	fi
1633	IGCM_debug_PopStack "IGCM_sys_check_path"
1634	}
1635
1636	#D-#==================================================
1637	#D-function IGCM_sys_check_quota
1638	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1639	#D-* Examples:
1640	#D-
1641	function IGCM_sys_check_quota {
1642	IGCM_debug_PushStack "IGCM_sys_check_quota"
1643	if ( $DEBUG_sys ) ; then
1644	echo "IGCM_sys_check_quota"
1645	fi
1646	# Limit of quota (in %)
1647	limit_quota=90
1648
1649	# Check of the volume
1650	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1651	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1652
1653	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1654
1655	unit_avail=${volume_avail: -1}
1656	unit_quota=${volume_quota: -1}
1657
1658	if [ "${unit_quota}" = "*" ] ; then
1659	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1660	IGCM_debug_Print 1 "More than 100% of your quota is used"
1661	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1662	IGCM_debug_Print 1 "You must have more than 10% available to run"
1663	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1664	IGCM_debug_Verif_Exit
1665	fi
1666
1667	temp_avail=${volume_avail%%${volume_avail: -1}*}
1668	temp_quota=${volume_quota%%${volume_quota: -1}*}
1669
1670	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1671
1672	# Convertion
1673	if [ ${volume_avail: -1} = "T" ] ; then
1674	(( temp_avail = temp_avail * 1000000000000 ))
1675	elif [ ${volume_avail: -1} = "G" ] ; then
1676	(( temp_avail = temp_avail * 1000000000 ))
1677	elif [ ${volume_avail: -1} = "M" ] ; then
1678	(( temp_avail = temp_avail * 1000000 ))
1679	elif [ ${volume_avail: -1} = "k" ] ; then
1680	(( temp_avail = temp_avail * 1000 ))
1681	else
1682	(( temp_avail = volume_avail ))
1683	fi
1684	if [ ${volume_quota: -1} = "T" ] ; then
1685	(( temp_quota = temp_quota * 1000000000000 ))
1686	elif [ ${volume_quota: -1} = "G" ] ; then
1687	(( temp_quota = temp_quota * 1000000000 ))
1688	elif [ ${volume_quota: -1} = "M" ] ; then
1689	(( temp_quota = temp_quota * 1000000 ))
1690	elif [ ${volume_quota: -1} = "k" ] ; then
1691	(( temp_quota = temp_quota * 1000 ))
1692	else
1693	(( temp_quota = volume_quota ))
1694	fi
1695	fi
1696
1697	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1698	# echo "volume ratio is " $quota_volume
1699
1700	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1701	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1702	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1703	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1704	IGCM_debug_Print 1 "You must have more than 10% available to run"
1705	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1706	IGCM_debug_Verif_Exit
1707	fi
1708
1709	fi
1710
1711	# Check of the number of inodes
1712
1713	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1714	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1715
1716	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1717
1718	unit_avail=${inode_avail: -1}
1719	unit_quota=${inode_quota: -1}
1720
1721	if [ "${unit_quota}" = "*" ] ; then
1722	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1723	IGCM_debug_Print 1 "More than 100% of your quota is used"
1724	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1725	IGCM_debug_Print 1 "You must have more than 10% available to run"
1726	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1727	IGCM_debug_Verif_Exit
1728	fi
1729
1730	temp_avail=${inode_avail%%${inode_avail: -1}*}
1731	temp_quota=${inode_quota%%${inode_quota: -1}*}
1732
1733	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1734
1735	# Convertion
1736	if [ ${inode_avail: -1} = "T" ] ; then
1737	(( temp_avail = temp_avail * 1000000000000 ))
1738	elif [ ${inode_avail: -1} = "G" ] ; then
1739	(( temp_avail = temp_avail * 1000000000 ))
1740	elif [ ${inode_avail: -1} = "M" ] ; then
1741	(( temp_avail = temp_avail * 1000000 ))
1742	elif [ ${inode_avail: -1} = "k" ] ; then
1743	(( temp_avail = temp_avail * 1000 ))
1744	else
1745	(( temp_avail = inode_avail ))
1746	fi
1747
1748	if [ ${inode_quota: -1} = "T" ] ; then
1749	(( temp_quota = temp_quota * 1000000000000 ))
1750	elif [ ${inode_quota: -1} = "G" ] ; then
1751	(( temp_quota = temp_quota * 1000000000 ))
1752	elif [ ${inode_quota: -1} = "M" ] ; then
1753	(( temp_quota = temp_quota * 1000000 ))
1754	elif [ ${inode_quota: -1} = "k" ] ; then
1755	(( temp_quota = temp_quota * 1000 ))
1756	else
1757	(( temp_quota = inode_quota ))
1758	fi
1759	fi
1760	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1761	# echo "inode ratio is " $quota_inode
1762
1763	if [ ${quota_inode} -ge ${limit_quota} ] ; then
1764	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1765	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1766	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1767	IGCM_debug_Print 1 "You must have more than 10% available to run"
1768	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1769	IGCM_debug_Verif_Exit
1770	fi
1771	fi
1772	IGCM_debug_PopStack "IGCM_sys_check_quota"
1773	}
1774
1775	#D-#==================================================
1776	#D-function IGCM_sys_projectAccounting
1777	#D-* Purpose: store project accounting information in a file
1778	#D-* Examples:
1779	#D-
1780	function IGCM_sys_projectAccounting {
1781	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1782	if ( $DEBUG_sys ) ; then
1783	echo "IGCM_sys_check_quota"
1784	fi
1785
1786	ssh irene194 /usr/bin/ccc_myproject > $1
1787
1788	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1789	}
1790
1791	#D-#==================================================
1792	#D-function IGCM_sys_getJobSchedulerID
1793	#D-* Purpose: Get the job ID during execution
1794	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1795	#D-
1796	function IGCM_sys_getJobSchedulerID {
1797	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1798	if ( $DEBUG_sys ) ; then
1799	echo "IGCM_sys_getJobSchedulerID"
1800	fi
1801	eval ${1}=${BRIDGE_MSUB_JOBID}
1802
1803	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1804	}
1805
1806	#D-#==================================================
1807	#D-function IGCM_sys_GetJobID
1808	#D-* Purpose: Get the job ID from the JobName
1809	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1810	#D-
1811	function IGCM_sys_GetJobID {
1812	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1813	if ( $DEBUG_sys ) ; then
1814	echo "IGCM_sys_GetJobID"
1815	fi
1816
1817	# With -f option, the full job name is given in the last column
1818	ID=$( ccc_mstat -f -u $2 \| \
1819	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1820
1821	eval ${3}=${ID}
1822	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1823	}
1824
1825	#D-#==================================================
1826	#D-function IGCM_sys_CountJobInQueue
1827	#D-* Purpose: Count number of users job
1828	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1829	#D-
1830	function IGCM_sys_CountJobInQueue {
1831	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1832	if ( $DEBUG_sys ) ; then
1833	echo "IGCM_sys_CountJobInQueue"
1834	fi
1835
1836	# With -f option, the full job name is given in the last column
1837	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1838
1839	eval ${2}=${NbRun}
1840
1841	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1842	}
1843
1844	#D-#==================================================
1845	#D-function IGCM_sys_ListJobInQueue
1846	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1847	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1848	#D-
1849	function IGCM_sys_ListJobInQueue {
1850	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1851	if ( $DEBUG_sys ) ; then
1852	echo "IGCM_sys_ListJobInQueue"
1853	fi
1854
1855	# With -f option, the full job name is given in the last column
1856	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1857	'( $2 == User && \
1858	$NF != /TS/ && \
1859	$NF !~ /PACK/ && \
1860	$NF !~ /REBUILD/ && \
1861	$NF !~ /pack/ ) \
1862	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1863
1864	eval set -A ${2} ${JobList[*]}
1865
1866	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1867	}
1868
1869	#D-#==================================================
1870	#D-function IGCM_sys_atlas
1871	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1872	#D-* Examples:
1873	#D-
1874	function IGCM_sys_atlas {
1875	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1876	if ( $DEBUG_sys ) ; then
1877	echo "IGCM_sys_atlas :" $@
1878	fi
1879
1880	typeset status
1881
1882	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1883	status=$?
1884	if [ ${status} -gt 0 ] ; then
1885	echo "IGCM_sys_atlas : error code ${status}"
1886	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1887	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1888	IGCM_debug_PopStack "IGCM_sys_atlas"
1889	return 1
1890	else
1891	IGCM_debug_PopStack "IGCM_sys_atlas"
1892	return 0
1893	fi
1894
1895	IGCM_debug_PopStack "IGCM_sys_atlas"
1896	}
1897
1898	#D-#==================================================
1899	#D-function IGCM_sys_rebuild_nemo
1900	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1901	#D-* Examples:
1902	#D-
1903
1904	function IGCM_sys_rebuild_nemo {
1905	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1906	if ( $DEBUG_sys ) ; then
1907	echo "IGCM_sys_rebuild_nemo :" $@
1908	fi
1909
1910	$( ccc_home -u igcmg)/Tools/irene-amd/bin/rebuild_nemo ${1} ${2}
1911
1912	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1913	}
1914

Note: See TracBrowser for help on using the repository browser.

Download in other formats: