Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1582

Last change on this file since 1582 was 1582, checked in by aclsce, 15 months ago
Use of irene tools (instead of irene-amd tools) because of RH8 change.
File size: 66.2 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: 1512 $ Revision of last commit
7	# $Author:: acosce $ Author of last commit
8	# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Irene-amd
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41	typeset -r DRYRUN=0
42	typeset -r DRYRUN_DEBUG=4
43	else
44	typeset -r DRYRUN=${DRYRUN:=0}
45	typeset -r DRYRUN_DEBUG=0
46	fi
47
48	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49	# -------------------------------------------------------------------------------------
50	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
51	# \| \| Cp/Exe/param/files \| \| \| \|
52	# \| \| Chmod Qsub \| \| \| \|
53	# -------------------------------------------------------------------------------------
54	# \| 0 \| yes \| yes \| yes \| yes \|
55	# -------------------------------------------------------------------------------------
56	# \| 1 \| yes \| yes \| yes \| no \|
57	# -------------------------------------------------------------------------------------
58	# \| 2 \| yes \| yes \| no \| no \|
59	# -------------------------------------------------------------------------------------
60	# \| 3 \| yes \| no \| no \| no \|
61	# -------------------------------------------------------------------------------------
62	# \| 4 \| yes \| yes \| creation of RUN_DIR and Job_debug \|
63	# -------------------------------------------------------------------------------------
64
65
66	#=====================================================
67	# Global Variables :
68	#=====================================================
69	# Language : "fr" or "en"
70	typeset -r MYLANG="fr"
71
72	#=====================================================
73	# Host user names project maxCpuTime
74	# $hostname ou hostname
75	typeset HOST=${HOST:=$( hostname )}
76	# $username ou whoami
77	typeset LOGIN=${LOGIN:=$( whoami )}
78	# $hostname of the MASTER job
79	typeset MASTER=irene-amd
80	# add default project on irene-amd
81	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} \| cut -d@ -f1 )
82	# jobWarningDelay in seconds
83	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
84
85	#D-
86	#D-#==================================================
87	#D-Program used in libIGCM
88	#D-#==================================================
89
90	# Submit command
91	typeset SUBMIT=${SUBMIT:=ccc_msub}
92	# rsync with path
93	typeset -r RSYNC=/usr/bin/rsync
94	# RSYNC_opt args to rsync
95	typeset -r RSYNC_opt="-va"
96	# ie storage filesystem
97	typeset -r STOREHOST=${MASTER}
98	typeset -r REMOTE_RSYNC=/usr/bin/rsync
99
100	#====================================================
101	# Set environment tools (ferret, nco, cdo, rebuild, ...)
102	#====================================================
103	typeset CCCOS=${CCCOS:=$( ccc_os )}
104
105	if [ X${TaskType} = Xcomputing ] ; then
106	IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
107	else
108	module purge > /dev/null 2>&1
109	if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
110	. $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
111	else
112	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_irene-amd > /dev/null 2>&1
113	. $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_atlas_irene-amd > /dev/null 2>&1
114	fi
115
116	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
117	export UVCDAT_ANONYMOUS_LOG=FALSE
118	fi
119
120	# FYI
121	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
122	[ ! X${TaskType} = Xchecking ] && module list
123
124	# For AddNoise and AddPertu3DOCE
125	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
126
127	# For rebuild
128	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl/bin
129
130	#====================================================
131	# Host specific DIRECTORIES
132	#====================================================
133
134	# ============ CESIUM START ============ #
135
136	#====================================================
137	#- Mirror libIGCM from titane to cesium if needed
138	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
139	#if [ ! ${ROOTSYS} = "home" ] ; then
140	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
141	#else
142	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
143	#fi
144
145	#====================================================
146	#- libIGCM_POST
147	#if ( ${MirrorlibIGCM} ) ; then
148	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
149	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
150	#else
151	# typeset -r libIGCM_POST=${libIGCM}
152	#fi
153
154	# ============ CESIUM END ============ #
155
156	#====================================================
157	#- MirrorlibIGCM for frontend
158	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
159
160	#====================================================
161	#- libIGCM_POST for frontend
162	typeset -r libIGCM_POST=${libIGCM}
163
164	#====================================================
165	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
166	typeset -r R_EXE="${MODIPSL}/bin"
167
168	#====================================================
169	#- SUBMIT_DIR : submission dir
170	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
171
172	#====================================================
173	#- IN
174	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
175
176	#====================================================
177	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
178	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
179
180	#====================================================
181	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
182	typeset -r OUTCOMMAND_PATH=/tmp
183
184	#====================================================
185	#- HOST_MPIRUN_COMMAND
186	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
187
188	#====================================================
189	#- Max number of arguments passed to nco operator or demigration command
190	UNIX_MAX_LIMIT=360
191
192	#====================================================
193	#- set PackDefault to true on irene-amd
194	PackDefault=true
195
196	#====================================================
197	#- Default number of MPI task for IPSL coupled model
198	#- required for backward compatibility
199	#-
200	DEFAULT_NUM_PROC_OCE=5
201	DEFAULT_NUM_PROC_CPL=1
202	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
203
204	#====================================================
205	#- Number of computing cores per node
206	#-
207	NB_CORE_PER_NODE=128
208
209	#D-#==================================================
210	#D-function IGCM_sys_defineArchives
211	#D-* Purpose:
212	#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
213	#D-* Define ARCHIVE : Dedicated to large files
214	#D-* Define STORAGE : Dedicated to small/medium files
215	#D-* Define R_OUT : Output tree located on ARCHIVE
216	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
217	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
218	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
219	#D-* Examples:
220	#D-
221	function IGCM_sys_defineArchives {
222	IGCM_debug_PushStack "IGCM_sys_defineArchives"
223
224	# Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
225	if [ X${config_UserChoices_DataProject} = X ] \|\| [ X${config_UserChoices_DataProject} = DEFAULT ]; then
226	# Default option: Change the dfldatadir according to the project used for submission
227	# The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
228	if [ X"$( echo ${PROJECT} \| grep cmip6 )" != "X" ] ; then
229	module switch dfldatadir dfldatadir/gencmip6
230	IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
231	else
232	module switch dfldatadir dfldatadir/${PROJECT}
233	IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
234	fi
235	else
236	# Use the project set by the variable DataProject in section UserChoices in config.card
237	module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
238	IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
239	fi
240
241	#====================================================
242	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
243	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
244
245	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
246	#====================================================
247	#- ARCHIVE (dedicated to large files)
248	ARCHIVE=${config_UserChoices_ARCHIVE}
249	else
250	#====================================================
251	#- ARCHIVE (dedicated to large files)
252	ARCHIVE=${CCCSTOREDIR}
253	fi
254
255	if [ ! X${config_UserChoices_STORAGE} = X ]; then
256	#====================================================
257	#- STORAGE (dedicated to small/medium files)
258	STORAGE=${config_UserChoices_STORAGE}
259	else
260	#====================================================
261	#- STORAGE (dedicated to small/medium files)
262	STORAGE=${CCCWORKDIR}
263	fi
264
265	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
266	#====================================================
267	#- R_OUT
268	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
269
270	#====================================================
271	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
273
274	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
275
276	else
277	#====================================================
278	#- R_OUT
279	R_OUT=${ARCHIVE}/IGCM_OUT
280
281	#====================================================
282	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
283	R_FIG=${STORAGE}/IGCM_OUT
284	fi
285
286	#====================================================
287	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
288	CMIP6_BUF=${STORAGE}/IGCM_OUT
289
290	#====================================================
291	#- R_BUF (ONLY FOR double copy an scratch)
292	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
293
294	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
295	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
296	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
297
298	IGCM_debug_PopStack "IGCM_sys_defineArchives"
299	}
300
301	#D-#==================================================
302	#D-function IGCM_sys_RshArchive
303	#D-* Purpose: Archive rsh command
304	#D-* Examples:
305	#D-
306	function IGCM_sys_RshArchive {
307	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
308	/bin/ksh <<-EOF
309	${@}
310	EOF
311	status=$?
312	if [ ${status} -gt 0 ] ; then
313	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
314	IGCM_debug_Exit "IGCM_sys_RshArchive"
315	fi
316	IGCM_debug_PopStack "IGCM_sys_RshArchive"
317	}
318
319	#D-#==================================================
320	#D-function IGCM_sys_RshArchive_NoError
321	#D-* Purpose: Archive rsh command, without error
322	#D-* used only in monitoring.job
323	#D-* Examples:
324	#D-
325	function IGCM_sys_RshArchive_NoError {
326	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
327	/bin/ksh <<-EOF
328	${@} 2> /dev/null
329	EOF
330	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
331	}
332
333	#D-#==================================================
334	#D-function IGCM_sys_MkdirArchive
335	#D-* Purpose: Mkdir on Archive
336	#D-* Examples:
337	#D-
338	function IGCM_sys_MkdirArchive {
339	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
340	if ( $DEBUG_sys ) ; then
341	echo "IGCM_sys_MkdirArchive :" $@
342	fi
343	#- creation de repertoire sur le serveur fichier
344	if [ ! -d ${1} ]; then
345	\mkdir -p $1
346	status=$?
347
348	if [ ${status} -gt 0 ] ; then
349	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
350	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
351	fi
352	fi
353	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
354	}
355
356	#D-#==================================================
357	#D-function IGCM_sys_TestDirArchive
358	#D-* Purpose: Test Directory that must exists on Archive
359	#D-* Examples:
360	#D-
361	function IGCM_sys_TestDirArchive {
362	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
363	if ( $DEBUG_sys ) ; then
364	echo "IGCM_sys_TestDirArchive :" $@
365	fi
366	typeset ExistFlag
367	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
368	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
369	return ${ExistFlag}
370	}
371
372	#D-#==================================================
373	#D-function IGCM_sys_IsFileArchived
374	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
375	#D-* Examples:
376	#D-
377	function IGCM_sys_IsFileArchived {
378	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
379	if ( $DEBUG_sys ) ; then
380	echo "IGCM_sys_IsFileArchived :" $@
381	fi
382	typeset IsArchivedFlag
383	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
384	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
385
386	return ${IsArchivedFlag}
387	}
388
389	#D-#==================================================
390	#D-function IGCM_sys_TestFileArchive
391	#D-* Purpose: Test file that must NOT EXISTS on Archive
392	#D-* Examples:
393	#D-
394	function IGCM_sys_TestFileArchive {
395	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
396	typeset ExistFlag
397	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
398	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
399
400	return ${ExistFlag}
401	}
402
403	#D-#==================================================
404	#D-function IGCM_sys_CountFileArchive
405	#D-* Purpose: Count files on Archive filesystem
406	#D-* Examples:
407	#D-
408	function IGCM_sys_CountFileArchive {
409	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
410	ls ${@} 2>/dev/null \| wc -l
411	if [ $? -gt 0 ] ; then
412	echo "IGCM_sys_CountFileArchive : erreur."
413	fi
414	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
415	}
416
417	#D-#==================================================
418	#D-function IGCM_sys_Tree
419	#D-* Purpose: Tree directories with files on ${ARCHIVE}
420	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
421	#D-
422	function IGCM_sys_Tree {
423	IGCM_debug_PushStack "IGCM_sys_Tree" $@
424	if ( $DEBUG_sys ) ; then
425	echo "IGCM_sys_Tree :" $@
426	fi
427
428	\ls -lR ${@}
429
430	IGCM_debug_PopStack "IGCM_sys_Tree"
431	}
432
433	#D-#==================================================
434	#D-function IGCM_sys_Qsub
435	#D-* Purpose: Qsub new job
436	#D-* Examples:
437	#D-
438	function IGCM_sys_Qsub {
439	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
440	if ( $DEBUG_sys ) ; then
441	echo "IGCM_sys_Qsub :" $@
442	fi
443	typeset options status
444	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
445
446	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
447	status=$?
448
449	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
450	if [ ${status} -gt 0 ] ; then
451	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
452	IGCM_debug_Exit "IGCM_sys_Qsub"
453	else
454	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
455	fi
456	IGCM_debug_PopStack "IGCM_sys_Qsub"
457	}
458
459	#D-#==================================================
460	#D-function IGCM_sys_QsubPost
461	#D-* Purpose: Qsub new job on scalaire
462	#D-* Examples:
463	#D-
464	function IGCM_sys_QsubPost {
465	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
466	if ( $DEBUG_sys ) ; then
467	echo "IGCM_sys_QsubPost :" $@
468	fi
469	typeset options status
470	options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
471
472	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
473	status=$?
474
475	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
476	if [ ${status} -gt 0 ] ; then
477	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
478	IGCM_debug_Exit "IGCM_sys_QsubPost"
479	else
480	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
481	fi
482	IGCM_debug_PopStack "IGCM_sys_QsubPost"
483	}
484
485	#D-*************************
486	#D- File transfer functions
487	#D-*************************
488	#D-
489
490	#D-#==================================================
491	#D-function IGCM_sys_RmRunDir
492	#D-* Purpose: rm tmpdir (dummy function most of the time batch
493	#D- scheduler will do the job)
494	#D-* Examples:
495	#D-
496	function IGCM_sys_RmRunDir {
497	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
498	if ( $DEBUG_sys ) ; then
499	echo "IGCM_sys_RmRunDir :" $@
500	fi
501
502	typeset status
503
504	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
505	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
506	status=$?
507
508	if [ ${status} -gt 0 ] ; then
509	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
510	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
511	IGCM_debug_Exit "IGCM_sys_RmRunDir"
512	else
513	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514	fi
515	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
516	}
517
518	#D-#==================================================
519	#D-function IGCM_sys_Put_Dir
520	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
521	#D-* Examples:
522	#D-
523	function IGCM_sys_Put_Dir {
524	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
525	if ( $DEBUG_sys ) ; then
526	echo "IGCM_sys_Put_Dir :" $@
527	fi
528	if [ $DRYRUN = 0 ]; then
529	if [ ! -d ${1} ] ; then
530	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
531	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
532	return
533	fi
534
535	typeset status
536
537	# Only if we use rsync
538	#IGCM_sys_TestDirArchive $( dirname $2 )
539	#
540	#USUAL WAY
541	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
542	status=$?
543
544	if [ ${status} -gt 0 ] ; then
545	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
546	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
547	IGCM_debug_Exit "IGCM_sys_Put_Dir"
548	else
549	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550	fi
551	fi
552	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
553	}
554
555	#D-#==================================================
556	#D-function IGCM_sys_Get_Dir
557	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
558	#D-* Examples:
559	#D-
560	function IGCM_sys_Get_Dir {
561	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
562	if ( $DEBUG_sys ) ; then
563	echo "IGCM_sys_Get_Dir :" $@
564	fi
565	if [ $DRYRUN = 0 ]; then
566	typeset NB_ESSAI DELAI status i
567	# number of tentative
568	NB_ESSAI=3
569	# time delay between tentative
570	DELAI=2
571
572	#
573	# USUAL WAY
574	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
575	ccc_hsm get -r $1
576
577	i=0
578	while [ $i -lt $NB_ESSAI ] ; do
579	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
580	status=$?
581	if [ ${status} -gt 0 ] ; then
582	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
583	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
584	sleep $DELAI
585	else
586	break
587	fi
588	(( i = i + 1 ))
589	done
590
591	if [ ${status} -gt 0 ] ; then
592	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
593	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
594	IGCM_debug_Exit "IGCM_sys_Get_Dir"
595	else
596	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
597	fi
598	fi
599	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
600	}
601
602	#D-#==================================================
603	#D-function IGCM_sys_Put_Rest
604	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
605	#D- File and target directory must exist.
606	#D-* Examples:
607	#D-
608	function IGCM_sys_Put_Rest {
609	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
610	if ( $DEBUG_sys ) ; then
611	echo "IGCM_sys_Put_Rest :" $@
612	fi
613	if [ $DRYRUN = 0 ]; then
614	if [ ! -f ${1} ] ; then
615	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
616	IGCM_debug_Exit "IGCM_sys_Put_Rest"
617	fi
618
619	typeset status
620	#
621	# USUAL WAY
622	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
623	status=$?
624
625	# #RSYNC WITH NETWORK SSH CALL
626	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
628
629	# #RSYNC WITH NFS USE
630	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
632
633	# status=$?
634	# IGCM_sys_Rsync_out $status
635
636	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
637	# (( status=status+$? ))
638
639	if [ ${status} -gt 0 ] ; then
640	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
641	[ -f ${1} ] && ls -l ${1}
642	[ -f ${2} ] && ls -l ${2}
643	[ -f ${2}/${1} ] && ls -l ${2}/${1}
644	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645	IGCM_debug_Exit "IGCM_sys_Put_Rest"
646	else
647
648	if [ X${JobType} = XRUN ] ; then
649	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
650	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
651	fi
652
653	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
654	fi
655	fi
656	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
657	}
658
659	#D-#==================================================
660	#D-function IGCM_sys_Put_Out
661	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
662	#D-* Examples:
663	#D-
664	function IGCM_sys_Put_Out {
665	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
666	if ( $DEBUG_sys ) ; then
667	echo "IGCM_sys_Put_Out :" $@
668	fi
669
670	typeset NB_ESSAI DELAI status i exist skip
671	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
672
673	# number of tentative
674	NB_ESSAI=3
675	# time delay between tentative
676	DELAI=2
677
678	if [ $DRYRUN = 0 ]; then
679	if [ ! -f ${1} ] ; then
680	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
681	IGCM_debug_PopStack "IGCM_sys_Put_Out"
682	return 1
683	fi
684	#
685	IGCM_sys_MkdirArchive $( dirname $2 )
686	#
687	exist=false
688	skip=false
689	if [ -f $2 ] ; then
690	IGCM_debug_Print 1 "$2 already exist"
691	ccc_hsm get $2
692	exist=true
693	if [ "X$( diff $1 $2 )" = X ] ; then
694	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
695	skip=true
696	else
697	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
698	skip=false
699	fi
700	fi
701	#
702	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
703	IGCM_sys_Chmod u+w $2
704	fi
705
706	if [ X${skip} = Xfalse ] ; then
707	i=0
708	while [ $i -lt $NB_ESSAI ] ; do
709	# Identify file system
710	fileDeviceNumberInHex=$( stat -c %d $1 )
711	status=$?
712	if [ ${status} -gt 0 ] ; then
713	IGCM_debug_Exit "IGCM_sys_Put_Out"
714	fi
715	# Identify file system
716	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
717	status=$?
718	if [ ${status} -gt 0 ] ; then
719	IGCM_debug_Exit "IGCM_sys_Put_Out"
720	fi
721
722	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
723	# They are not on the same device. USUAL WAY
724	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
725	status=$?
726	else
727	# They are on the same device. NOT SO USUAL WAY
728	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
729	status=$?
730	fi
731	if [ ${status} -gt 0 ]; then
732	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
733	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
734	[ -f ${1} ] && ls -l ${1}
735	[ -f ${2} ] && ls -l ${2}
736	[ -f ${2}/${1} ] && ls -l ${2}/${1}
737	sleep $DELAI
738	else
739	break
740	fi
741	(( i = i + 1 ))
742	done
743	fi
744
745	# #RSYNC WITH NETWORK SSH CALL
746	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
747	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
748
749	# #RSYNC WITH NFS USE
750	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
751	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
752
753	# status=$?
754	# IGCM_sys_Rsync_out $status
755
756	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
757	# (( status=status+$? ))
758
759	if [ ${status} -gt 0 ] ; then
760	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
761	[ -f ${1} ] && ls -l ${1}
762	[ -f ${2} ] && ls -l ${2}
763	[ -f ${2}/${1} ] && ls -l ${2}/${1}
764	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765	IGCM_debug_Exit "IGCM_sys_Put_Out"
766	else
767
768	if [ X${JobType} = XRUN ] ; then
769	if [ X${3} = X ] ; then
770	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
771	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
772	fi
773	fi
774
775	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
776	fi
777	fi
778	IGCM_debug_PopStack "IGCM_sys_Put_Out"
779	return 0
780	}
781
782	#D-#==================================================
783	#D-function IGCM_sys_Get
784	#D-* Purpose: Get a file from ${ARCHIVE}
785	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
786	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
787	function IGCM_sys_Get {
788	IGCM_debug_PushStack "IGCM_sys_Get" $@
789
790	typeset DEST dm_liste target file_work
791	typeset NB_ESSAI DELAI status i
792
793	if ( $DEBUG_sys ) ; then
794	echo "IGCM_sys_Get :" $@
795	fi
796
797	# number of tentative
798	NB_ESSAI=3
799	# time delay between tentative
800	DELAI=2
801
802	if [ $DRYRUN -le 2 ]; then
803	if [ X${1} = X'/l' ] ; then
804	eval set +A dm_liste \${${2}}
805	else
806	eval set +A dm_liste ${1}
807	fi
808	eval DEST=\${${#}}
809	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
810	status=$?
811	if [ ${status} -gt 0 ] ; then
812	echo "WARNING IGCM_sys_Get : error code ${status}"
813	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
814	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
815	fi
816
817	#if [ ${status} -gt 0 ] ; then
818	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
819	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
820	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
821	# sleep 30
822	# echo "We try another time"
823	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
825	# status=$?
826	# if [ ${status} -gt 0 ] ; then
827	# echo "ERROR IGCM_sys_Get : again demigration error :"
828	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829	# IGCM_debug_Exit "IGCM_sys_Get"
830	# fi
831	# else
832	# echo "ERROR IGCM_sys_Get : demigration error :"
833	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
834	# IGCM_debug_Exit "IGCM_sys_Get"
835	# fi
836	#fi
837
838	# #RSYNC WITH NETWORK SSH CALL
839	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
841
842	# #RSYNC WITH NFS USE
843	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
844	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
845
846	# status=$?
847	# IGCM_sys_Rsync_out $status
848
849	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
850	# (( status=status+$? ))
851
852	#USUAL WAY
853	if [ X${1} = X'/l' ] ; then
854	for target in ${dm_liste[*]} ; do
855	local_file=$( basename ${target} )
856	# test if the target file is present before the loop
857	IGCM_sys_TestFileArchive ${target}
858	status=$?
859	if [ ${status} -gt 0 ] ; then
860	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
861	IGCM_debug_Exit "IGCM_sys_Get"
862	else
863	i=0
864	while [ $i -lt $NB_ESSAI ] ; do
865	#if [ X${DoLink} = Xtrue ] ; then
866	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
867	# status=$?
868	# else
869	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
870	# status=$?
871	#fi
872	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
873	status=$?
874	if [ ${status} -gt 0 ]; then
875	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
876	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
877	sleep $DELAI
878	else
879	break
880	fi
881	(( i = i + 1 ))
882	done
883	if [ ${status} -gt 0 ] ; then
884	echo "IGCM_sys_Get : error"
885	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
886	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
887	IGCM_debug_Exit "IGCM_sys_Get"
888	else
889	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
890	fi
891	fi
892	done
893	else
894	i=0
895	while [ $i -lt $NB_ESSAI ] ; do
896	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
897	status=$?
898	if [ ${status} -gt 0 ]; then
899	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
900	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
901	sleep $DELAI
902	else
903	break
904	fi
905	(( i = i + 1 ))
906	done
907	if [ ${status} -gt 0 ] ; then
908	echo "IGCM_sys_Get : error"
909	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
910	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
911	IGCM_debug_Exit "IGCM_sys_Get"
912	else
913	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
914	fi
915	fi
916	fi
917	IGCM_debug_PopStack "IGCM_sys_Get"
918	}
919
920	#D-#==================================================
921	#D-function IGCM_sys_GetDate_Monitoring
922	#D-* Purpose: get the last year for which the monitoring has been computed
923	#D-* Examples:
924	#D-
925	function IGCM_sys_GetDate_Monitoring {
926	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
927	if ( $DEBUG_sys ) ; then
928	echo "IGCM_sys_GetDate_Monitoring :" $@
929	fi
930
931	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
932
933	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
934	}
935
936	#D-#==================================================
937	#D-function IGCM_sys_Dods_Rm
938	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
939	#D-* Examples:
940	#D-
941	function IGCM_sys_Dods_Rm {
942	if ( $DEBUG_sys ) ; then
943	echo "IGCM_sys_Dods_Rm :" $@
944	fi
945	typeset status
946	if [ $DRYRUN = 0 ]; then
947
948	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
949	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
950	# echo "Nothing has been done."
951	# return
952	# fi
953
954	if [ "$#" -eq 1 ]; then
955	$(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
956	status=$?
957	else
958	$(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
959	status=$?
960	fi
961
962	# if [ ${status} -gt 0 ] ; then
963	# echo "IGCM_sys_Dods_Rm : error."
964	# cat out_dods_rm
965	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
966	# else
967	# rm out_dods_rm
968	# fi
969
970	fi
971	return $status
972	}
973
974	#D-#==================================================
975	#D-function IGCM_sys_Dods_Cp
976	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
977	#D-* Examples:
978	#D-
979	function IGCM_sys_Dods_Cp {
980	if ( $DEBUG_sys ) ; then
981	echo "IGCM_sys_Dods_Cp :" $@
982	fi
983	typeset status
984	if [ $DRYRUN = 0 ]; then
985
986	# if [ ! -d ${R_SAVE}/${1} ] ; then
987	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
988	# echo "Nothing has been done."
989	# return
990	# fi
991
992	$(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
993	status=$?
994
995	# if [ ${status} -gt 0 ] ; then
996	# echo "IGCM_sys_Dods_Cp : error."
997	# cat out_dods_cp
998	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
999	# else
1000	# rm out_dods_cp
1001	# fi
1002
1003	fi
1004	return $status
1005	}
1006
1007	#D-#==================================================
1008	#D-function IGCM_sys_Put_Dods
1009	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1010	#D-* Examples:
1011	#D-
1012	function IGCM_sys_Put_Dods {
1013	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1014	if ( $DEBUG_sys ) ; then
1015	echo "IGCM_sys_Put_Dods :" $@
1016	fi
1017	typeset status
1018	if [ $DRYRUN = 0 ]; then
1019	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1020	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1021	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1022	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1023	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1024	return
1025	fi
1026
1027	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1028	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1029	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1030	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1031	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1032	return
1033	fi
1034	#
1035	if [ -d ${R_SAVE}/${1} ] ; then
1036	cd ${R_SAVE}
1037	elif [ -d ${R_FIGR}/${1} ] ; then
1038	cd ${R_FIGR}
1039	fi
1040
1041	IGCM_sys_Dods_Rm ${1}
1042	IGCM_sys_Dods_Cp ${1}
1043	status=0
1044
1045	if [ ${status} -gt 0 ] ; then
1046	echo "IGCM_sys_Put_Dods : error."
1047	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1048	fi
1049	fi
1050	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1051	}
1052
1053	##############################################################
1054	# REBUILD OPERATOR
1055
1056	#D-#==================================================
1057	#D-function IGCM_sys_sync
1058	#D-* Purpose: flush buffer on disk
1059	#D-* Examples:
1060	#D-
1061	function IGCM_sys_sync {
1062	IGCM_debug_PushStack "IGCM_sys_sync" $@
1063	if ( $DEBUG_sys ) ; then
1064	echo "IGCM_sys_sync :" $@
1065	fi
1066
1067	/bin/sync
1068
1069	IGCM_debug_PopStack "IGCM_sys_sync"
1070	}
1071
1072	############################################################
1073	# Activate Running Environnment Variables
1074
1075	#D-#==================================================
1076	#D-function IGCM_sys_activ_variables
1077	#D-* Purpose: set environement variables prior to execution
1078	#D-* Examples:
1079	#D-
1080	function IGCM_sys_activ_variables {
1081	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1082	if ( $DEBUG_sys ) ; then
1083	echo "IGCM_sys_activ_variables"
1084	fi
1085
1086	# --------------------------------------------------------------------
1087	#D- MODULE specifications
1088	# --------------------------------------------------------------------
1089	# Source the file EnvFile containing all module load needed to run the model.
1090	# EnvFile can be specified in config.card. If this is the case and if the file exists,
1091	# it will be used. Otherwise a default file will be used.
1092	if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1093	# EnvFile is set in config.card and the file exists
1094	IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1095	EnvFile=${config_UserChoices_EnvFile}
1096	else
1097	IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1098	module purge > /dev/null 2>&1
1099	if [ X${compilerVersion} = XV19 ] ; then
1100	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1101	EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1102	else
1103	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1104	fi
1105	fi
1106	IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"
1107	IGCM_debug_Print 1 ${EnvFile}
1108	. ${EnvFile}
1109
1110	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1111	module list
1112
1113	# --------------------------------------------------------------------
1114	#D- MPI specifications
1115	# --------------------------------------------------------------------
1116
1117	# --------------------------------------------------------------------
1118	#D- Other specifications
1119	# --------------------------------------------------------------------
1120
1121	ulimit -s unlimited
1122
1123	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1124	}
1125
1126	############################################################
1127	# Desactivate Running Environnment Variables
1128
1129	#D-#==================================================
1130	#D-function IGCM_sys_desactiv_variables
1131	#D-* Purpose: unset environement variables after execution
1132	#D-* Examples:
1133	#D-
1134	function IGCM_sys_desactiv_variables {
1135	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1136	if ( $DEBUG_sys ) ; then
1137	echo "IGCM_sys_desactiv_variables"
1138	fi
1139	# --------------------------------------------------------------------
1140	#D- MPI specifications
1141	# --------------------------------------------------------------------
1142
1143	# --------------------------------------------------------------------
1144	#D- Other specifications
1145	# --------------------------------------------------------------------
1146
1147	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1148	}
1149
1150	############################################################
1151	# Update job headers to be used by the scheduler
1152
1153	#D-#==================================================
1154	#D-function IGCM_sys_updateHeaders
1155	#D-* Purpose: Update job headers to be used by the scheduler
1156	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1157	#D-
1158	function IGCM_sys_updateHeaders {
1159	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1160	if ( $DEBUG_sys ) ; then
1161	echo "IGCM_sys_updateHeaders"
1162	fi
1163	typeset file
1164	file=$1
1165
1166	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1167	# Compute the number of core to add to force the use of a new node for xios servers
1168
1169	if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1170	(( IOS_OK_DEDICATED = 0 ))
1171	fi
1172
1173
1174	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1175	(( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1176	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1177	(( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1178	(( coreNumber = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1179	fi
1180
1181	if [ ${executionType} -eq 1 ] ; then
1182	# MPMD + MPI
1183	sed -e "/::openMPthreads::/d" \
1184	-e "s/::JobNumProcTot::/${coreNumber}/" \
1185	${file} > ${file}.tmp
1186
1187	elif [ ${executionType} -eq 2 ] ; then
1188	# MPMD + MPI + OMP :ccc_mprun
1189	sed -e "/::openMPthreads::/d" \
1190	-e "s/::JobNumProcTot::/${coreNumber}/" \
1191	${file} > ${file}.tmp
1192
1193	elif [ ${executionType} -eq 3 ] ; then
1194	# SPMD + MPI/OMP
1195	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1196	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1197	-e "/#MSUB -x/d" \
1198	${file} > ${file}.tmp
1199
1200	elif [ ${executionType} -eq 4 ] ; then
1201	# SPMD + MPI only
1202	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1203	-e "/::openMPthreads::/d" \
1204	-e "/#MSUB -x/d" \
1205	${file} > ${file}.tmp
1206
1207	elif [ ${executionType} -eq 5 ] ; then
1208	# SPMD + OMP only
1209	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1210	-e "/::JobNumProcTot::/d" \
1211	-e "/#MSUB -x/d" \
1212	${file} > ${file}.tmp
1213
1214	elif [ ${executionType} -eq 6 ] ; then
1215	# SEQUENTIAL THEN
1216	sed -e "s/::JobNumProcTot::/1/" \
1217	-e "/::openMPthreads::/d" \
1218	-e "/#MSUB -x/d" \
1219	${file} > ${file}.tmp
1220
1221	fi
1222
1223	IGCM_sys_Mv ${file}.tmp ${file}
1224
1225	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1226	}
1227
1228	############################################################
1229	# Build MPI/OMP scripts run file (dummy function)
1230
1231	#D-#==================================================
1232	#D-function IGCM_sys_build_run_file
1233	#D-* Purpose: build run file (deprecated)
1234	#D-* Examples:
1235	#D-
1236	function IGCM_sys_build_run_file {
1237	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1238	}
1239
1240	############################################################
1241	# Build MPI/OMP scripts
1242
1243	#D-#==================================================
1244	#D-function IGCM_sys_build_execution_scripts
1245	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1246	#D-* Examples:
1247	#D-
1248	function IGCM_sys_build_execution_scripts
1249	{
1250	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1251	if ( $DEBUG_sys ) ; then
1252	echo "IGCM_sys_build_execution_scripts " $@
1253	fi
1254
1255	EXECUTION=${HOST_MPIRUN_COMMAND}
1256
1257	if ( ${OK_PARA_MPMD} ) ; then
1258
1259	# MPMD mode
1260	# 1 MPI only
1261	# 2 MPI/OpenMP mpirun method
1262	# 3 MPI/OpenMP ccc_mprun method
1263
1264	if [ -f run_file ] ; then
1265	IGCM_sys_Rm -f run_file
1266	fi
1267	touch run_file
1268
1269	# case 1 : Only MPI (MPMD)
1270	if ( ! ${OK_PARA_OMP} ) ; then
1271
1272	# Build run_file
1273
1274	# First loop on the components for the coupler ie oasis (only if oasis3)
1275	# the coupler ie oasis3 must be the first one
1276	for comp in ${config_ListOfComponents[*]} ; do
1277
1278	eval ExeNameIn=\${config_Executable_${comp}[0]}
1279	eval ExeNameOut=\${config_Executable_${comp}[1]}
1280
1281	# for CPL component only
1282	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1283
1284	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1285	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1286	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1287	fi
1288	done
1289
1290	# Then second loop on the components
1291	for comp in ${config_ListOfComponents[*]} ; do
1292
1293	eval ExeNameIn=\${config_Executable_${comp}[0]}
1294	eval ExeNameOut=\${config_Executable_${comp}[1]}
1295
1296	# Only if we really have an executable for the component and not the coupler ie oasis:
1297	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1298
1299	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1300	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1301	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1302
1303	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1304	# Compute the number of core to add to force the use of a new node for xios servers
1305	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1306	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1307	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1308	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1309	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1310	if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1311	echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1312	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1313	echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1314	fi
1315	else
1316	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1317	fi
1318	else
1319	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1320	fi
1321	else
1322	echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1323	fi
1324
1325
1326	fi
1327	done
1328
1329	## module advised by TGCC (instead of 2 variables)
1330	module load feature/bridge/heterogenous_mpmd
1331
1332	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1333
1334	IGCM_sys_Chmod u+x run_file
1335	if ( $DEBUG_sys ) ; then
1336	echo "run_file contains : "
1337	cat run_file
1338	fi
1339
1340	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1341
1342	# 2 MPI/OpenMP mpirun method
1343
1344	# Use of mpirun instead of ccc_mprun
1345	EXECUTION="time mpirun"
1346
1347	# Hosts treatment
1348	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1349
1350	i=0
1351	rm -f hosts rankfile
1352	IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1353	for nodes in `cat hosts.tmp`
1354	do
1355	host[$i]=$nodes
1356	echo "${host[$i]}" >> hosts
1357	IGCM_debug_Print 1 ${host[$i]}
1358	i=$((i+1))
1359	done
1360	rm -f hosts.tmp
1361
1362	listnodes=${host[*]}
1363
1364	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1365
1366	# Initialisation
1367	rank=0
1368	current_core=0
1369	init_exec=n
1370
1371	# Loop on the components
1372	for comp in ${config_ListOfComponents[*]} ; do
1373
1374	eval ExeNameIn=\${config_Executable_${comp}[0]}
1375	eval ExeNameOut=\${config_Executable_${comp}[1]}
1376
1377	# Not possible if oasis has an executable (i.e old version of oasis3)
1378	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1379	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1380	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1381	IGCM_debug_Verif_Exit
1382	fi
1383
1384	# Only if we really have an executable for the component :
1385	if [ "X${ExeNameOut}" != X\"\" ] ; then
1386
1387	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1388	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1389	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1390
1391	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1392	echo "" >> script_${ExeNameOut}.ksh
1393	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1394
1395	# Check if the number of threads is correct
1396	case ${comp_proc_omp_loc} in
1397	2\|4\|8\|16)
1398	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1399	;;
1400	*)
1401	IGCM_debug_Exit "ERROR with OMP parameters !"
1402	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1403	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1404	IGCM_debug_Verif_Exit
1405	;;
1406	esac
1407	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1408	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1409	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1410	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1411	fi
1412
1413	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1414	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1415	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1416	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1417	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1418
1419	if [ ${init_exec} = y ] ; then
1420	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1421	else
1422	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1423	init_exec=y
1424	fi
1425
1426	# Build rankfile : method used to assign cores and nodes for the MPI process
1427	# Ex :
1428	#rank 0=curie5296 slot=0,1,2,3
1429	#rank 1=curie5296 slot=4,5,6,7
1430	# Example of final command :
1431	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1432	# with script_lmdz.x.ksh :
1433	# #!/bin/ksh
1434	#export KMP_STACKSIZE=3g
1435	#export KMP_LIBRARY=turnaround
1436	#export MKL_SERIAL=YES
1437	#OMP_NUM_THREADS=4
1438	#./lmdz.x
1439	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1440	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1441	else
1442	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1443	fi
1444	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1445	(( index_host = current_core / NB_CORE_PER_NODE ))
1446	host_value=${host[${index_host}]}
1447	(( slot = current_core % NB_CORE_PER_NODE ))
1448	virg=","
1449	string_final=""
1450	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1451	string=$index$virg
1452	string_final=$string_final$string
1453	done
1454	string_final=$( echo $string_final \| sed "s/.$//" )
1455	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1456	(( rank = rank + 1 ))
1457	(( current_core = current_core + offset_comp_proc_loc ))
1458	done
1459	fi
1460	done
1461
1462	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1463
1464	# 3 MPI/OpenMP ccc_mprun method
1465
1466	# MPI-OpenMP (MPMD)
1467	# example of run_file
1468	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1469	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1470	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1471
1472	# Loop on the components
1473	for comp in ${config_ListOfComponents[*]} ; do
1474
1475	eval ExeNameIn=\${config_Executable_${comp}[0]}
1476	eval ExeNameOut=\${config_Executable_${comp}[1]}
1477
1478	# Not possible if oasis has an executable (i.e old version of oasis3)
1479	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1480	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1481	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1482	IGCM_debug_Verif_Exit
1483	fi
1484
1485	# Only if we really have an executable for the component :
1486	if [ "X${ExeNameOut}" != X\"\" ] ; then
1487
1488	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1489	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1490	eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1491
1492	# Compute number of cores per process
1493	(( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc ))
1494
1495	# Build script files
1496
1497	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1498	echo "" >> script_${ExeNameOut}.ksh
1499	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1500
1501	# Check if the number of threads is correct
1502	case ${comp_proc_omp_loc} in
1503	2\|4\|8\|16)
1504	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1505	;;
1506	*)
1507	IGCM_debug_Exit "ERROR with OMP parameters !"
1508	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1509	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1510	IGCM_debug_Verif_Exit
1511	;;
1512	esac
1513	### only for LMDZ?
1514	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1515	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1516	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1517	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1518
1519	fi
1520
1521	# to have out/err per process on different files
1522	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1523	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1524
1525	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1526
1527	# Complete run_file
1528
1529	# In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1530	# Compute the number of core to add to force the use of a new node for xios servers
1531	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XIOS" ] ) ; then
1532	if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1533	(( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc ))
1534	(( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1535	(( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1536	if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1537	echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1538	if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1539	echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1540	fi
1541	else
1542	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1543	fi
1544	else
1545	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1546	fi
1547	else
1548	echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1549	fi
1550
1551	fi
1552	done
1553
1554	## variable added to stop after 60s instead of 600s by default.
1555	## This is used when no error comes from executables and when something stopped an executable without notice.
1556	export SLURM_WAIT=60
1557
1558	## module advised by TGCC (instead of 2 variables)
1559	module load feature/bridge/heterogenous_mpmd
1560
1561	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1562
1563	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1564
1565	IGCM_sys_Chmod u+x run_file
1566	if ( $DEBUG_sys ) ; then
1567	echo "run_file contains : "
1568	cat run_file
1569	fi
1570
1571	else
1572
1573	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1574
1575	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1576
1577	else
1578	# Only one executable (SPMD mode).
1579
1580	for comp in ${config_ListOfComponents[*]} ; do
1581
1582	# Only if we really have an executable for the component :
1583	eval ExeNameOut=\${config_Executable_${comp}[1]}
1584	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1585
1586	# Build script files
1587
1588	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1589	echo "" >> script_${ExeNameOut}.ksh
1590	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1591
1592	if ( ${OK_PARA_OMP} ) ; then
1593	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1594	echo "" >> script_${ExeNameOut}.ksh
1595	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1596	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1597	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1598	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1599	fi
1600
1601	if ( ${OK_PARA_MPI} ) ; then
1602	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1603	# Default : ccc_mprun used if nb_proc gt 1
1604	# to have out/err per process on different files
1605	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1606	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1607	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1608	else
1609	# Default : ccc_mprun is NOT used if nb_proc eq 1
1610	# to have out/err per process on different files
1611	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1612	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1613	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1614	fi
1615
1616	IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1617	cat script_${ExeNameOut}.ksh
1618
1619	fi
1620
1621	done
1622
1623	fi # ${OK_PARA_MPMD}
1624
1625	IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1626	IGCM_debug_Print 1 "$EXECUTION"
1627
1628	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1629	}
1630
1631	#D-#==================================================
1632	#D-function IGCM_sys_check_path
1633	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1634	#D-* do not point to an important use directory. Stop immediately in that case.
1635	#D-* Examples:
1636	#D-
1637	function IGCM_sys_check_path {
1638	IGCM_debug_PushStack "IGCM_sys_check_path"
1639	if ( $DEBUG_sys ) ; then
1640	echo "IGCM_sys_check_path"
1641	fi
1642
1643	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1644	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1645	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1646	IGCM_debug_Exit "This will stop the job"
1647	fi
1648	IGCM_debug_PopStack "IGCM_sys_check_path"
1649	}
1650
1651	#D-#==================================================
1652	#D-function IGCM_sys_check_quota
1653	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1654	#D-* Examples:
1655	#D-
1656	function IGCM_sys_check_quota {
1657	IGCM_debug_PushStack "IGCM_sys_check_quota"
1658	if ( $DEBUG_sys ) ; then
1659	echo "IGCM_sys_check_quota"
1660	fi
1661	# Limit of quota (in %)
1662	limit_quota=90
1663
1664	# Check of the volume
1665	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1666	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1667
1668	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1669
1670	unit_avail=${volume_avail: -1}
1671	unit_quota=${volume_quota: -1}
1672
1673	if [ "${unit_quota}" = "*" ] ; then
1674	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1675	IGCM_debug_Print 1 "More than 100% of your quota is used"
1676	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1677	IGCM_debug_Print 1 "You must have more than 10% available to run"
1678	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1679	IGCM_debug_Verif_Exit
1680	fi
1681
1682	temp_avail=${volume_avail%%${volume_avail: -1}*}
1683	temp_quota=${volume_quota%%${volume_quota: -1}*}
1684
1685	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1686
1687	# Convertion
1688	if [ ${volume_avail: -1} = "T" ] ; then
1689	(( temp_avail = temp_avail * 1000000000000 ))
1690	elif [ ${volume_avail: -1} = "G" ] ; then
1691	(( temp_avail = temp_avail * 1000000000 ))
1692	elif [ ${volume_avail: -1} = "M" ] ; then
1693	(( temp_avail = temp_avail * 1000000 ))
1694	elif [ ${volume_avail: -1} = "k" ] ; then
1695	(( temp_avail = temp_avail * 1000 ))
1696	else
1697	(( temp_avail = volume_avail ))
1698	fi
1699	if [ ${volume_quota: -1} = "T" ] ; then
1700	(( temp_quota = temp_quota * 1000000000000 ))
1701	elif [ ${volume_quota: -1} = "G" ] ; then
1702	(( temp_quota = temp_quota * 1000000000 ))
1703	elif [ ${volume_quota: -1} = "M" ] ; then
1704	(( temp_quota = temp_quota * 1000000 ))
1705	elif [ ${volume_quota: -1} = "k" ] ; then
1706	(( temp_quota = temp_quota * 1000 ))
1707	else
1708	(( temp_quota = volume_quota ))
1709	fi
1710	fi
1711
1712	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1713	# echo "volume ratio is " $quota_volume
1714
1715	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1716	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1717	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1718	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1719	IGCM_debug_Print 1 "You must have more than 10% available to run"
1720	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1721	IGCM_debug_Verif_Exit
1722	fi
1723
1724	fi
1725
1726	# Check of the number of inodes
1727
1728	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1729	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1730
1731	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1732
1733	unit_avail=${inode_avail: -1}
1734	unit_quota=${inode_quota: -1}
1735
1736	if [ "${unit_quota}" = "*" ] ; then
1737	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1738	IGCM_debug_Print 1 "More than 100% of your quota is used"
1739	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1740	IGCM_debug_Print 1 "You must have more than 10% available to run"
1741	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1742	IGCM_debug_Verif_Exit
1743	fi
1744
1745	temp_avail=${inode_avail%%${inode_avail: -1}*}
1746	temp_quota=${inode_quota%%${inode_quota: -1}*}
1747
1748	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1749
1750	# Convertion
1751	if [ ${inode_avail: -1} = "T" ] ; then
1752	(( temp_avail = temp_avail * 1000000000000 ))
1753	elif [ ${inode_avail: -1} = "G" ] ; then
1754	(( temp_avail = temp_avail * 1000000000 ))
1755	elif [ ${inode_avail: -1} = "M" ] ; then
1756	(( temp_avail = temp_avail * 1000000 ))
1757	elif [ ${inode_avail: -1} = "k" ] ; then
1758	(( temp_avail = temp_avail * 1000 ))
1759	else
1760	(( temp_avail = inode_avail ))
1761	fi
1762
1763	if [ ${inode_quota: -1} = "T" ] ; then
1764	(( temp_quota = temp_quota * 1000000000000 ))
1765	elif [ ${inode_quota: -1} = "G" ] ; then
1766	(( temp_quota = temp_quota * 1000000000 ))
1767	elif [ ${inode_quota: -1} = "M" ] ; then
1768	(( temp_quota = temp_quota * 1000000 ))
1769	elif [ ${inode_quota: -1} = "k" ] ; then
1770	(( temp_quota = temp_quota * 1000 ))
1771	else
1772	(( temp_quota = inode_quota ))
1773	fi
1774	fi
1775	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1776	# echo "inode ratio is " $quota_inode
1777
1778	if [ ${quota_inode} -ge ${limit_quota} ] ; then
1779	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1780	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1781	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1782	IGCM_debug_Print 1 "You must have more than 10% available to run"
1783	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1784	IGCM_debug_Verif_Exit
1785	fi
1786	fi
1787	IGCM_debug_PopStack "IGCM_sys_check_quota"
1788	}
1789
1790	#D-#==================================================
1791	#D-function IGCM_sys_projectAccounting
1792	#D-* Purpose: store project accounting information in a file
1793	#D-* Examples:
1794	#D-
1795	function IGCM_sys_projectAccounting {
1796	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1797	if ( $DEBUG_sys ) ; then
1798	echo "IGCM_sys_check_quota"
1799	fi
1800
1801	ssh irene194 /usr/bin/ccc_myproject > $1
1802
1803	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1804	}
1805
1806	#D-#==================================================
1807	#D-function IGCM_sys_getJobSchedulerID
1808	#D-* Purpose: Get the job ID during execution
1809	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1810	#D-
1811	function IGCM_sys_getJobSchedulerID {
1812	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1813	if ( $DEBUG_sys ) ; then
1814	echo "IGCM_sys_getJobSchedulerID"
1815	fi
1816	eval ${1}=${BRIDGE_MSUB_JOBID}
1817
1818	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1819	}
1820
1821	#D-#==================================================
1822	#D-function IGCM_sys_GetJobID
1823	#D-* Purpose: Get the job ID from the JobName
1824	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1825	#D-
1826	function IGCM_sys_GetJobID {
1827	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1828	if ( $DEBUG_sys ) ; then
1829	echo "IGCM_sys_GetJobID"
1830	fi
1831
1832	# With -f option, the full job name is given in the last column
1833	ID=$( ccc_mstat -f -u $2 \| \
1834	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1835
1836	eval ${3}=${ID}
1837	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1838	}
1839
1840	#D-#==================================================
1841	#D-function IGCM_sys_CountJobInQueue
1842	#D-* Purpose: Count number of users job
1843	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1844	#D-
1845	function IGCM_sys_CountJobInQueue {
1846	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1847	if ( $DEBUG_sys ) ; then
1848	echo "IGCM_sys_CountJobInQueue"
1849	fi
1850
1851	# With -f option, the full job name is given in the last column
1852	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1853
1854	eval ${2}=${NbRun}
1855
1856	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1857	}
1858
1859	#D-#==================================================
1860	#D-function IGCM_sys_ListJobInQueue
1861	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1862	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1863	#D-
1864	function IGCM_sys_ListJobInQueue {
1865	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1866	if ( $DEBUG_sys ) ; then
1867	echo "IGCM_sys_ListJobInQueue"
1868	fi
1869
1870	# With -f option, the full job name is given in the last column
1871	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1872	'( $2 == User && \
1873	$NF != /TS/ && \
1874	$NF !~ /PACK/ && \
1875	$NF !~ /REBUILD/ && \
1876	$NF !~ /pack/ ) \
1877	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1878
1879	eval set -A ${2} ${JobList[*]}
1880
1881	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1882	}
1883
1884	#D-#==================================================
1885	#D-function IGCM_sys_atlas
1886	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1887	#D-* Examples:
1888	#D-
1889	function IGCM_sys_atlas {
1890	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1891	if ( $DEBUG_sys ) ; then
1892	echo "IGCM_sys_atlas :" $@
1893	fi
1894
1895	typeset status
1896
1897	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1898	status=$?
1899	if [ ${status} -gt 0 ] ; then
1900	echo "IGCM_sys_atlas : error code ${status}"
1901	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1902	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1903	IGCM_debug_PopStack "IGCM_sys_atlas"
1904	return 1
1905	else
1906	IGCM_debug_PopStack "IGCM_sys_atlas"
1907	return 0
1908	fi
1909
1910	IGCM_debug_PopStack "IGCM_sys_atlas"
1911	}
1912
1913	#D-#==================================================
1914	#D-function IGCM_sys_rebuild_nemo
1915	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1916	#D-* Examples:
1917	#D-
1918
1919	function IGCM_sys_rebuild_nemo {
1920	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1921	if ( $DEBUG_sys ) ; then
1922	echo "IGCM_sys_rebuild_nemo :" $@
1923	fi
1924
1925	$( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1926
1927	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1928	}
1929

Note: See TracBrowser for help on using the repository browser.

Download in other formats: