Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh @ 1482

Last change on this file since 1482 was 1482, checked in by jgipsl, 5 years ago
Add check if the EnvFile? exists if it is set in config.card. Else the default file will be sourced in the same way as if EnvFile? is not set in config.card.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 63.8 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Irene
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host user names project maxCpuTime
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=irene
71	# add default project on irene
72	typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} \| cut -d@ -f1 )
73	# jobWarningDelay in seconds
74	typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=ccc_msub}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo, rebuild, ...)
93	#====================================================
94	if [ X${TaskType} = Xcomputing ] ; then
95	IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
96	else
97	module purge > /dev/null 2>&1
98	##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
99	. $( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
100	##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
101	. $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
102
103	export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
104	export UVCDAT_ANONYMOUS_LOG=FALSE
105	fi
106
107	# FYI
108	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
109	[ ! X${TaskType} = Xchecking ] && module list
110
111	# For AddNoise and AddPertu3DOCE
112	##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
113	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
114
115	# For rebuild
116	##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
117	export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
118
119	#====================================================
120	# Host specific DIRECTORIES
121	#====================================================
122
123	# ============ CESIUM START ============ #
124
125	#====================================================
126	#- Mirror libIGCM from titane to cesium if needed
127	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
128	#if [ ! ${ROOTSYS} = "home" ] ; then
129	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
130	#else
131	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132	#fi
133
134	#====================================================
135	#- libIGCM_POST
136	#if ( ${MirrorlibIGCM} ) ; then
137	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
138	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
139	#else
140	# typeset -r libIGCM_POST=${libIGCM}
141	#fi
142
143	# ============ CESIUM END ============ #
144
145	#====================================================
146	#- MirrorlibIGCM for frontend
147	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
148
149	#====================================================
150	#- libIGCM_POST for frontend
151	typeset -r libIGCM_POST=${libIGCM}
152
153	#====================================================
154	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
155	typeset -r R_EXE="${MODIPSL}/bin"
156
157	#====================================================
158	#- SUBMIT_DIR : submission dir
159	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
160
161	#====================================================
162	#- IN
163	typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
164
165	#====================================================
166	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
167	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
168
169	#====================================================
170	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
171	typeset -r OUTCOMMAND_PATH=/tmp
172
173	#====================================================
174	#- HOST_MPIRUN_COMMAND
175	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
176
177	#====================================================
178	#- Max number of arguments passed to nco operator or demigration command
179	UNIX_MAX_LIMIT=360
180
181	#====================================================
182	#- set PackDefault to true on irene
183	PackDefault=true
184
185	#====================================================
186	#- Default number of MPI task for IPSL coupled model
187	#- required for backward compatibility
188	#-
189	DEFAULT_NUM_PROC_OCE=5
190	DEFAULT_NUM_PROC_CPL=1
191	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
192
193	#====================================================
194	#- Number of computing cores per node
195	#-
196	NB_CORE_PER_NODE=48
197
198	#D-#==================================================
199	#D-function IGCM_sys_defineArchives
200	#D-* Purpose:
201	#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
202	#D-* Define ARCHIVE : Dedicated to large files
203	#D-* Define STORAGE : Dedicated to small/medium files
204	#D-* Define R_OUT : Output tree located on ARCHIVE
205	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
206	#D-* Define R_BUF : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
207	#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
208	#D-* Examples:
209	#D-
210	function IGCM_sys_defineArchives {
211	IGCM_debug_PushStack "IGCM_sys_defineArchives"
212
213	# Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
214	if [ X${config_UserChoices_DataProject} = X ] \|\| [ X${config_UserChoices_DataProject} = DEFAULT ]; then
215	# Default option: Change the dfldatadir according to the project used for submission
216	# The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
217	if [ X"$( echo ${PROJECT} \| grep cmip6 )" != "X" ] ; then
218	module switch dfldatadir dfldatadir/gencmip6
219	IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
220	else
221	module switch dfldatadir dfldatadir/${PROJECT}
222	IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
223	fi
224	else
225	# Use the project set by the variable DataProject in section UserChoices in config.card
226	module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
227	IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
228	fi
229
230	#====================================================
231	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
232	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
233
234	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
235	#====================================================
236	#- ARCHIVE (dedicated to large files)
237	ARCHIVE=${config_UserChoices_ARCHIVE}
238	else
239	#====================================================
240	#- ARCHIVE (dedicated to large files)
241	ARCHIVE=${CCCSTOREDIR}
242	fi
243
244	if [ ! X${config_UserChoices_STORAGE} = X ]; then
245	#====================================================
246	#- STORAGE (dedicated to small/medium files)
247	STORAGE=${config_UserChoices_STORAGE}
248	else
249	#====================================================
250	#- STORAGE (dedicated to small/medium files)
251	STORAGE=${CCCWORKDIR}
252	fi
253
254	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
255	#====================================================
256	#- R_OUT
257	R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
258
259	#====================================================
260	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
261	R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
262
263	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
264
265	else
266	#====================================================
267	#- R_OUT
268	R_OUT=${ARCHIVE}/IGCM_OUT
269
270	#====================================================
271	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272	R_FIG=${STORAGE}/IGCM_OUT
273	fi
274
275	#====================================================
276	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
277	CMIP6_BUF=${STORAGE}/IGCM_OUT
278
279	#====================================================
280	#- R_BUF (ONLY FOR double copy an scratch)
281	R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
282
283	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
284	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
285	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
286
287	IGCM_debug_PopStack "IGCM_sys_defineArchives"
288	}
289
290	#D-#==================================================
291	#D-function IGCM_sys_RshArchive
292	#D-* Purpose: Archive rsh command
293	#D-* Examples:
294	#D-
295	function IGCM_sys_RshArchive {
296	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
297	/bin/ksh <<-EOF
298	${@}
299	EOF
300	status=$?
301	if [ ${status} -gt 0 ] ; then
302	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
303	IGCM_debug_Exit "IGCM_sys_RshArchive"
304	fi
305	IGCM_debug_PopStack "IGCM_sys_RshArchive"
306	}
307
308	#D-#==================================================
309	#D-function IGCM_sys_RshArchive_NoError
310	#D-* Purpose: Archive rsh command, without error
311	#D-* used only in monitoring.job
312	#D-* Examples:
313	#D-
314	function IGCM_sys_RshArchive_NoError {
315	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
316	/bin/ksh <<-EOF
317	${@} 2> /dev/null
318	EOF
319	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
320	}
321
322	#D-#==================================================
323	#D-function IGCM_sys_MkdirArchive
324	#D-* Purpose: Mkdir on Archive
325	#D-* Examples:
326	#D-
327	function IGCM_sys_MkdirArchive {
328	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
329	if ( $DEBUG_sys ) ; then
330	echo "IGCM_sys_MkdirArchive :" $@
331	fi
332	#- creation de repertoire sur le serveur fichier
333	if [ ! -d ${1} ]; then
334	\mkdir -p $1
335	status=$?
336
337	if [ ${status} -gt 0 ] ; then
338	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
339	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
340	fi
341	fi
342	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
343	}
344
345	#D-#==================================================
346	#D-function IGCM_sys_TestDirArchive
347	#D-* Purpose: Test Directory that must exists on Archive
348	#D-* Examples:
349	#D-
350	function IGCM_sys_TestDirArchive {
351	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
352	if ( $DEBUG_sys ) ; then
353	echo "IGCM_sys_TestDirArchive :" $@
354	fi
355	typeset ExistFlag
356	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
357	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
358	return ${ExistFlag}
359	}
360
361	#D-#==================================================
362	#D-function IGCM_sys_IsFileArchived
363	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
364	#D-* Examples:
365	#D-
366	function IGCM_sys_IsFileArchived {
367	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
368	if ( $DEBUG_sys ) ; then
369	echo "IGCM_sys_IsFileArchived :" $@
370	fi
371	typeset IsArchivedFlag
372	IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
373	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
374
375	return ${IsArchivedFlag}
376	}
377
378	#D-#==================================================
379	#D-function IGCM_sys_TestFileArchive
380	#D-* Purpose: Test file that must NOT EXISTS on Archive
381	#D-* Examples:
382	#D-
383	function IGCM_sys_TestFileArchive {
384	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
385	typeset ExistFlag
386	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
387	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
388
389	return ${ExistFlag}
390	}
391
392	#D-#==================================================
393	#D-function IGCM_sys_CountFileArchive
394	#D-* Purpose: Count files on Archive filesystem
395	#D-* Examples:
396	#D-
397	function IGCM_sys_CountFileArchive {
398	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
399	ls ${@} 2>/dev/null \| wc -l
400	if [ $? -gt 0 ] ; then
401	echo "IGCM_sys_CountFileArchive : erreur."
402	fi
403	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
404	}
405
406	#D-#==================================================
407	#D-function IGCM_sys_Tree
408	#D-* Purpose: Tree directories with files on ${ARCHIVE}
409	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
410	#D-
411	function IGCM_sys_Tree {
412	IGCM_debug_PushStack "IGCM_sys_Tree" $@
413	if ( $DEBUG_sys ) ; then
414	echo "IGCM_sys_Tree :" $@
415	fi
416
417	\ls -lR ${@}
418
419	IGCM_debug_PopStack "IGCM_sys_Tree"
420	}
421
422	#D-#==================================================
423	#D-function IGCM_sys_Qsub
424	#D-* Purpose: Qsub new job
425	#D-* Examples:
426	#D-
427	function IGCM_sys_Qsub {
428	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
429	if ( $DEBUG_sys ) ; then
430	echo "IGCM_sys_Qsub :" $@
431	fi
432	typeset options status
433	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
434
435	/usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
436	status=$?
437
438	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
439	if [ ${status} -gt 0 ] ; then
440	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
441	IGCM_debug_Exit "IGCM_sys_Qsub"
442	else
443	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
444	fi
445	IGCM_debug_PopStack "IGCM_sys_Qsub"
446	}
447
448	#D-#==================================================
449	#D-function IGCM_sys_QsubPost
450	#D-* Purpose: Qsub new job on scalaire
451	#D-* Examples:
452	#D-
453	function IGCM_sys_QsubPost {
454	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
455	if ( $DEBUG_sys ) ; then
456	echo "IGCM_sys_QsubPost :" $@
457	fi
458	typeset options status
459	options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
460
461	/usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
462	status=$?
463
464	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
465	if [ ${status} -gt 0 ] ; then
466	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
467	IGCM_debug_Exit "IGCM_sys_QsubPost"
468	else
469	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
470	fi
471	IGCM_debug_PopStack "IGCM_sys_QsubPost"
472	}
473
474	#D-*************************
475	#D- File transfer functions
476	#D-*************************
477	#D-
478
479	#D-#==================================================
480	#D-function IGCM_sys_RmRunDir
481	#D-* Purpose: rm tmpdir (dummy function most of the time batch
482	#D- scheduler will do the job)
483	#D-* Examples:
484	#D-
485	function IGCM_sys_RmRunDir {
486	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
487	if ( $DEBUG_sys ) ; then
488	echo "IGCM_sys_RmRunDir :" $@
489	fi
490
491	typeset status
492
493	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
494	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
495	status=$?
496
497	if [ ${status} -gt 0 ] ; then
498	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
499	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
500	IGCM_debug_Exit "IGCM_sys_RmRunDir"
501	else
502	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
503	fi
504	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
505	}
506
507	#D-#==================================================
508	#D-function IGCM_sys_Put_Dir
509	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
510	#D-* Examples:
511	#D-
512	function IGCM_sys_Put_Dir {
513	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
514	if ( $DEBUG_sys ) ; then
515	echo "IGCM_sys_Put_Dir :" $@
516	fi
517	if [ $DRYRUN = 0 ]; then
518	if [ ! -d ${1} ] ; then
519	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
520	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
521	return
522	fi
523
524	typeset status
525
526	# Only if we use rsync
527	#IGCM_sys_TestDirArchive $( dirname $2 )
528	#
529	#USUAL WAY
530	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
531	status=$?
532
533	if [ ${status} -gt 0 ] ; then
534	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
535	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
536	IGCM_debug_Exit "IGCM_sys_Put_Dir"
537	else
538	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
539	fi
540	fi
541	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
542	}
543
544	#D-#==================================================
545	#D-function IGCM_sys_Get_Dir
546	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
547	#D-* Examples:
548	#D-
549	function IGCM_sys_Get_Dir {
550	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
551	if ( $DEBUG_sys ) ; then
552	echo "IGCM_sys_Get_Dir :" $@
553	fi
554	if [ $DRYRUN = 0 ]; then
555	typeset NB_ESSAI DELAI status i
556	# number of tentative
557	NB_ESSAI=3
558	# time delay between tentative
559	DELAI=2
560
561	#
562	# USUAL WAY
563	# add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
564	ccc_hsm get -r $1
565
566	i=0
567	while [ $i -lt $NB_ESSAI ] ; do
568	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
569	status=$?
570	if [ ${status} -gt 0 ] ; then
571	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
572	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
573	sleep $DELAI
574	else
575	break
576	fi
577	(( i = i + 1 ))
578	done
579
580	if [ ${status} -gt 0 ] ; then
581	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
582	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
583	IGCM_debug_Exit "IGCM_sys_Get_Dir"
584	else
585	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
586	fi
587	fi
588	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
589	}
590
591	#D-#==================================================
592	#D-function IGCM_sys_Put_Rest
593	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
594	#D- File and target directory must exist.
595	#D-* Examples:
596	#D-
597	function IGCM_sys_Put_Rest {
598	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
599	if ( $DEBUG_sys ) ; then
600	echo "IGCM_sys_Put_Rest :" $@
601	fi
602	if [ $DRYRUN = 0 ]; then
603	if [ ! -f ${1} ] ; then
604	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
605	IGCM_debug_Exit "IGCM_sys_Put_Rest"
606	fi
607
608	typeset status
609	#
610	# USUAL WAY
611	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
612	status=$?
613
614	# #RSYNC WITH NETWORK SSH CALL
615	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
616	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
617
618	# #RSYNC WITH NFS USE
619	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
620	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621
622	# status=$?
623	# IGCM_sys_Rsync_out $status
624
625	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
626	# (( status=status+$? ))
627
628	if [ ${status} -gt 0 ] ; then
629	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
630	[ -f ${1} ] && ls -l ${1}
631	[ -f ${2} ] && ls -l ${2}
632	[ -f ${2}/${1} ] && ls -l ${2}/${1}
633	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
634	IGCM_debug_Exit "IGCM_sys_Put_Rest"
635	else
636
637	if [ X${JobType} = XRUN ] ; then
638	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
639	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
640	fi
641
642	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
643	fi
644	fi
645	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
646	}
647
648	#D-#==================================================
649	#D-function IGCM_sys_Put_Out
650	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
651	#D-* Examples:
652	#D-
653	function IGCM_sys_Put_Out {
654	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
655	if ( $DEBUG_sys ) ; then
656	echo "IGCM_sys_Put_Out :" $@
657	fi
658
659	typeset NB_ESSAI DELAI status i exist skip
660	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
661
662	# number of tentative
663	NB_ESSAI=3
664	# time delay between tentative
665	DELAI=2
666
667	if [ $DRYRUN = 0 ]; then
668	if [ ! -f ${1} ] ; then
669	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
670	IGCM_debug_PopStack "IGCM_sys_Put_Out"
671	return 1
672	fi
673	#
674	IGCM_sys_MkdirArchive $( dirname $2 )
675	#
676	exist=false
677	skip=false
678	if [ -f $2 ] ; then
679	IGCM_debug_Print 1 "$2 already exist"
680	ccc_hsm get $2
681	exist=true
682	if [ "X$( diff $1 $2 )" = X ] ; then
683	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
684	skip=true
685	else
686	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
687	skip=false
688	fi
689	fi
690	#
691	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
692	IGCM_sys_Chmod u+w $2
693	fi
694
695	if [ X${skip} = Xfalse ] ; then
696	i=0
697	while [ $i -lt $NB_ESSAI ] ; do
698	# Identify file system
699	fileDeviceNumberInHex=$( stat -c %d $1 )
700	status=$?
701	if [ ${status} -gt 0 ] ; then
702	IGCM_debug_Exit "IGCM_sys_Put_Out"
703	fi
704	# Identify file system
705	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
706	status=$?
707	if [ ${status} -gt 0 ] ; then
708	IGCM_debug_Exit "IGCM_sys_Put_Out"
709	fi
710
711	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
712	# They are not on the same device. USUAL WAY
713	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
714	status=$?
715	else
716	# They are on the same device. NOT SO USUAL WAY
717	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718	status=$?
719	fi
720	if [ ${status} -gt 0 ]; then
721	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
722	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
723	[ -f ${1} ] && ls -l ${1}
724	[ -f ${2} ] && ls -l ${2}
725	[ -f ${2}/${1} ] && ls -l ${2}/${1}
726	sleep $DELAI
727	else
728	break
729	fi
730	(( i = i + 1 ))
731	done
732	fi
733
734	# #RSYNC WITH NETWORK SSH CALL
735	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
736	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
737
738	# #RSYNC WITH NFS USE
739	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
740	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741
742	# status=$?
743	# IGCM_sys_Rsync_out $status
744
745	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
746	# (( status=status+$? ))
747
748	if [ ${status} -gt 0 ] ; then
749	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
750	[ -f ${1} ] && ls -l ${1}
751	[ -f ${2} ] && ls -l ${2}
752	[ -f ${2}/${1} ] && ls -l ${2}/${1}
753	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
754	IGCM_debug_Exit "IGCM_sys_Put_Out"
755	else
756
757	if [ X${JobType} = XRUN ] ; then
758	if [ X${3} = X ] ; then
759	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
760	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
761	fi
762	fi
763
764	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765	fi
766	fi
767	IGCM_debug_PopStack "IGCM_sys_Put_Out"
768	return 0
769	}
770
771	#D-#==================================================
772	#D-function IGCM_sys_Get
773	#D-* Purpose: Get a file from ${ARCHIVE}
774	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
775	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
776	function IGCM_sys_Get {
777	IGCM_debug_PushStack "IGCM_sys_Get" $@
778
779	typeset DEST dm_liste target file_work
780	typeset NB_ESSAI DELAI status i
781
782	if ( $DEBUG_sys ) ; then
783	echo "IGCM_sys_Get :" $@
784	fi
785
786	# number of tentative
787	NB_ESSAI=3
788	# time delay between tentative
789	DELAI=2
790
791	if [ $DRYRUN -le 2 ]; then
792	if [ X${1} = X'/l' ] ; then
793	eval set +A dm_liste \${${2}}
794	else
795	eval set +A dm_liste ${1}
796	fi
797	eval DEST=\${${#}}
798	ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
799	status=$?
800	if [ ${status} -gt 0 ] ; then
801	echo "WARNING IGCM_sys_Get : error code ${status}"
802	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
803	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
804	fi
805
806	#if [ ${status} -gt 0 ] ; then
807	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
808	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
810	# sleep 30
811	# echo "We try another time"
812	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
813	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
814	# status=$?
815	# if [ ${status} -gt 0 ] ; then
816	# echo "ERROR IGCM_sys_Get : again demigration error :"
817	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
818	# IGCM_debug_Exit "IGCM_sys_Get"
819	# fi
820	# else
821	# echo "ERROR IGCM_sys_Get : demigration error :"
822	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823	# IGCM_debug_Exit "IGCM_sys_Get"
824	# fi
825	#fi
826
827	# #RSYNC WITH NETWORK SSH CALL
828	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
829	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
830
831	# #RSYNC WITH NFS USE
832	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
833	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
834
835	# status=$?
836	# IGCM_sys_Rsync_out $status
837
838	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
839	# (( status=status+$? ))
840
841	#USUAL WAY
842	if [ X${1} = X'/l' ] ; then
843	for target in ${dm_liste[*]} ; do
844	local_file=$( basename ${target} )
845	# test if the target file is present before the loop
846	IGCM_sys_TestFileArchive ${target}
847	status=$?
848	if [ ${status} -gt 0 ] ; then
849	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
850	IGCM_debug_Exit "IGCM_sys_Get"
851	else
852	i=0
853	while [ $i -lt $NB_ESSAI ] ; do
854	#if [ X${DoLink} = Xtrue ] ; then
855	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
856	# status=$?
857	# else
858	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
859	# status=$?
860	#fi
861	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862	status=$?
863	if [ ${status} -gt 0 ]; then
864	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
865	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
866	sleep $DELAI
867	else
868	break
869	fi
870	(( i = i + 1 ))
871	done
872	if [ ${status} -gt 0 ] ; then
873	echo "IGCM_sys_Get : error"
874	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
875	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
876	IGCM_debug_Exit "IGCM_sys_Get"
877	else
878	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
879	fi
880	fi
881	done
882	else
883	i=0
884	while [ $i -lt $NB_ESSAI ] ; do
885	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
886	status=$?
887	if [ ${status} -gt 0 ]; then
888	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
889	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
890	sleep $DELAI
891	else
892	break
893	fi
894	(( i = i + 1 ))
895	done
896	if [ ${status} -gt 0 ] ; then
897	echo "IGCM_sys_Get : error"
898	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
900	IGCM_debug_Exit "IGCM_sys_Get"
901	else
902	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
903	fi
904	fi
905	fi
906	IGCM_debug_PopStack "IGCM_sys_Get"
907	}
908
909	#D-#==================================================
910	#D-function IGCM_sys_GetDate_Monitoring
911	#D-* Purpose: get the last year for which the monitoring has been computed
912	#D-* Examples:
913	#D-
914	function IGCM_sys_GetDate_Monitoring {
915	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
916	if ( $DEBUG_sys ) ; then
917	echo "IGCM_sys_GetDate_Monitoring :" $@
918	fi
919
920	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
921
922	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
923	}
924
925	#D-#==================================================
926	#D-function IGCM_sys_Dods_Rm
927	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
928	#D-* Examples:
929	#D-
930	function IGCM_sys_Dods_Rm {
931	if ( $DEBUG_sys ) ; then
932	echo "IGCM_sys_Dods_Rm :" $@
933	fi
934	typeset status
935	if [ $DRYRUN = 0 ]; then
936
937	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
938	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
939	# echo "Nothing has been done."
940	# return
941	# fi
942
943	if [ "$#" -eq 1 ]; then
944	## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
945	$(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
946	status=$?
947	else
948	## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
949	$(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
950	status=$?
951	fi
952
953	# if [ ${status} -gt 0 ] ; then
954	# echo "IGCM_sys_Dods_Rm : error."
955	# cat out_dods_rm
956	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
957	# else
958	# rm out_dods_rm
959	# fi
960
961	fi
962	return $status
963	}
964
965	#D-#==================================================
966	#D-function IGCM_sys_Dods_Cp
967	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
968	#D-* Examples:
969	#D-
970	function IGCM_sys_Dods_Cp {
971	if ( $DEBUG_sys ) ; then
972	echo "IGCM_sys_Dods_Cp :" $@
973	fi
974	typeset status
975	if [ $DRYRUN = 0 ]; then
976
977	# if [ ! -d ${R_SAVE}/${1} ] ; then
978	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
979	# echo "Nothing has been done."
980	# return
981	# fi
982
983	## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
984	$(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
985	status=$?
986
987	# if [ ${status} -gt 0 ] ; then
988	# echo "IGCM_sys_Dods_Cp : error."
989	# cat out_dods_cp
990	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
991	# else
992	# rm out_dods_cp
993	# fi
994
995	fi
996	return $status
997	}
998
999	#D-#==================================================
1000	#D-function IGCM_sys_Put_Dods
1001	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1002	#D-* Examples:
1003	#D-
1004	function IGCM_sys_Put_Dods {
1005	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1006	if ( $DEBUG_sys ) ; then
1007	echo "IGCM_sys_Put_Dods :" $@
1008	fi
1009	typeset status
1010	if [ $DRYRUN = 0 ]; then
1011	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1012	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1013	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1014	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1015	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1016	return
1017	fi
1018
1019	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1020	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1021	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1022	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1023	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1024	return
1025	fi
1026	#
1027	if [ -d ${R_SAVE}/${1} ] ; then
1028	cd ${R_SAVE}
1029	elif [ -d ${R_FIGR}/${1} ] ; then
1030	cd ${R_FIGR}
1031	fi
1032
1033	IGCM_sys_Dods_Rm ${1}
1034	IGCM_sys_Dods_Cp ${1}
1035	status=0
1036
1037	if [ ${status} -gt 0 ] ; then
1038	echo "IGCM_sys_Put_Dods : error."
1039	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1040	fi
1041	fi
1042	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1043	}
1044
1045	##############################################################
1046	# REBUILD OPERATOR
1047
1048	#D-#==================================================
1049	#D-function IGCM_sys_sync
1050	#D-* Purpose: flush buffer on disk
1051	#D-* Examples:
1052	#D-
1053	function IGCM_sys_sync {
1054	IGCM_debug_PushStack "IGCM_sys_sync" $@
1055	if ( $DEBUG_sys ) ; then
1056	echo "IGCM_sys_sync :" $@
1057	fi
1058
1059	/bin/sync
1060
1061	IGCM_debug_PopStack "IGCM_sys_sync"
1062	}
1063
1064	############################################################
1065	# Activate Running Environnment Variables
1066
1067	#D-#==================================================
1068	#D-function IGCM_sys_activ_variables
1069	#D-* Purpose: set environement variables prior to execution
1070	#D-* Examples:
1071	#D-
1072	function IGCM_sys_activ_variables {
1073	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1074	if ( $DEBUG_sys ) ; then
1075	echo "IGCM_sys_activ_variables"
1076	fi
1077
1078	# --------------------------------------------------------------------
1079	#D- MODULE specifications
1080	# --------------------------------------------------------------------
1081	# Source the file EnvFile containing all module load needed to run the model.
1082	# Use default a default file on the shared account or use an optional file specified in config.card.
1083	if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1084	# EnvFile is set in config.card and the file exists
1085	IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1086	EnvFile=${config_UserChoices_EnvFile}
1087	else
1088	IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1089	module purge > /dev/null 2>&1
1090	if [ X${compilerVersion} = XV17 ] ; then
1091	IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1092	## EnvFile=/ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1093	EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1094	else
1095	IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1096	fi
1097	fi
1098	IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"
1099	IGCM_debug_Print 1 ${EnvFile}
1100	. ${EnvFile}
1101
1102	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. "
1103	module list
1104
1105	# Keep track of this
1106	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
1107	[ ! X${TaskType} = Xchecking ] && module list
1108
1109	# --------------------------------------------------------------------
1110	#D- MPI specifications
1111	# --------------------------------------------------------------------
1112
1113	# --------------------------------------------------------------------
1114	#D- Other specifications
1115	# --------------------------------------------------------------------
1116
1117	ulimit -s unlimited
1118
1119	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1120	}
1121
1122	############################################################
1123	# Desactivate Running Environnment Variables
1124
1125	#D-#==================================================
1126	#D-function IGCM_sys_desactiv_variables
1127	#D-* Purpose: unset environement variables after execution
1128	#D-* Examples:
1129	#D-
1130	function IGCM_sys_desactiv_variables {
1131	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1132	if ( $DEBUG_sys ) ; then
1133	echo "IGCM_sys_desactiv_variables"
1134	fi
1135	# --------------------------------------------------------------------
1136	#D- MPI specifications
1137	# --------------------------------------------------------------------
1138
1139	# --------------------------------------------------------------------
1140	#D- Other specifications
1141	# --------------------------------------------------------------------
1142
1143	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1144	}
1145
1146	############################################################
1147	# Update job headers to be used by the scheduler
1148
1149	#D-#==================================================
1150	#D-function IGCM_sys_updateHeaders
1151	#D-* Purpose: Update job headers to be used by the scheduler
1152	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1153	#D-
1154	function IGCM_sys_updateHeaders {
1155	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1156	if ( $DEBUG_sys ) ; then
1157	echo "IGCM_sys_updateHeaders"
1158	fi
1159	typeset file
1160	file=$1
1161
1162	if [ ${executionType} -eq 1 ] ; then
1163	# MPMD + MPI
1164	sed -e "/::openMPthreads::/d" \
1165	-e "s/::JobNumProcTot::/${coreNumber}/" \
1166	-e "/#MSUB -x/d" \
1167	-e "/--cpu_bind=none/d" \
1168	${file} > ${file}.tmp
1169
1170	elif [ ${executionType} -eq 2 ] ; then
1171	# MPMD + MPI + OMP : mpirun/ccc_mprun/error
1172	if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1173	sed -e "/::openMPthreads::/d" \
1174	-e "s/::JobNumProcTot::/${coreNumber}/" \
1175	${file} > ${file}.tmp
1176	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1177	sed -e "/::openMPthreads::/d" \
1178	-e "s/::JobNumProcTot::/${coreNumber}/" \
1179	-e "/--cpu_bind=none/d" \
1180	${file} > ${file}.tmp
1181	else
1182	IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1183	IGCM_debug_Exit "IGCM_sys_updateHeaders"
1184	fi
1185	elif [ ${executionType} -eq 3 ] ; then
1186	# SPMD + MPI/OMP
1187	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1188	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1189	-e "/#MSUB -x/d" \
1190	-e "/--cpu_bind=none/d" \
1191	${file} > ${file}.tmp
1192
1193	elif [ ${executionType} -eq 4 ] ; then
1194	# SPMD + MPI only
1195	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1196	-e "/::openMPthreads::/d" \
1197	-e "/#MSUB -x/d" \
1198	-e "/--cpu_bind=none/d" \
1199	${file} > ${file}.tmp
1200
1201	elif [ ${executionType} -eq 5 ] ; then
1202	# SPMD + OMP only
1203	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1204	-e "/::JobNumProcTot::/d" \
1205	-e "/#MSUB -x/d" \
1206	-e "/--cpu_bind=none/d" \
1207	${file} > ${file}.tmp
1208
1209	elif [ ${executionType} -eq 6 ] ; then
1210	# SEQUENTIAL THEN
1211	sed -e "s/::JobNumProcTot::/1/" \
1212	-e "/::openMPthreads::/d" \
1213	-e "/#MSUB -x/d" \
1214	-e "/--cpu_bind=none/d" \
1215	${file} > ${file}.tmp
1216
1217	fi
1218
1219	IGCM_sys_Mv ${file}.tmp ${file}
1220
1221	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1222	}
1223
1224	############################################################
1225	# Build MPI/OMP scripts run file (dummy function)
1226
1227	#D-#==================================================
1228	#D-function IGCM_sys_build_run_file
1229	#D-* Purpose: build run file (deprecated)
1230	#D-* Examples:
1231	#D-
1232	function IGCM_sys_build_run_file {
1233	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1234	}
1235
1236	############################################################
1237	# Build MPI/OMP scripts
1238
1239	#D-#==================================================
1240	#D-function IGCM_sys_build_execution_scripts
1241	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1242	#D-* Examples:
1243	#D-
1244	function IGCM_sys_build_execution_scripts
1245	{
1246	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1247	if ( $DEBUG_sys ) ; then
1248	echo "IGCM_sys_build_execution_scripts " $@
1249	fi
1250
1251	EXECUTION=${HOST_MPIRUN_COMMAND}
1252
1253	if ( ${OK_PARA_MPMD} ) ; then
1254
1255	# MPMD mode
1256	# 1 MPI only
1257	# 2 MPI/OpenMP mpirun method
1258	# 3 MPI/OpenMP ccc_mprun method
1259
1260	if [ -f run_file ] ; then
1261	IGCM_sys_Rm -f run_file
1262	fi
1263	touch run_file
1264
1265	# case 1 : Only MPI (MPMD)
1266	if ( ! ${OK_PARA_OMP} ) ; then
1267
1268	# Build run_file
1269
1270	# First loop on the components for the coupler ie oasis (only if oasis3)
1271	# the coupler ie oasis3 must be the first one
1272	for comp in ${config_ListOfComponents[*]} ; do
1273
1274	eval ExeNameIn=\${config_Executable_${comp}[0]}
1275	eval ExeNameOut=\${config_Executable_${comp}[1]}
1276
1277	# for CPL component only
1278	if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1279
1280	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1281	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1282	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1283	fi
1284	done
1285
1286	# Then second loop on the components
1287	for comp in ${config_ListOfComponents[*]} ; do
1288
1289	eval ExeNameIn=\${config_Executable_${comp}[0]}
1290	eval ExeNameOut=\${config_Executable_${comp}[1]}
1291
1292	# Only if we really have an executable for the component and not the coupler ie oasis:
1293	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1294
1295	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1296	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1297	echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1298	fi
1299	done
1300
1301	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1302
1303	IGCM_sys_Chmod u+x run_file
1304	if ( $DEBUG_sys ) ; then
1305	echo "run_file contains : "
1306	cat run_file
1307	fi
1308
1309	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1310
1311	# 2 MPI/OpenMP mpirun method
1312
1313	# Use of mpirun instead of ccc_mprun
1314	EXECUTION="time mpirun"
1315
1316	# Hosts treatment
1317	${EXECUTION} hostname \| sort \| uniq > hosts.tmp
1318
1319	i=0
1320	rm -f hosts rankfile
1321	IGCM_debug_Print 1 "sys Irene, Hosts available :"
1322	for nodes in `cat hosts.tmp`
1323	do
1324	host[$i]=$nodes
1325	echo "${host[$i]}" >> hosts
1326	IGCM_debug_Print 1 ${host[$i]}
1327	i=$((i+1))
1328	done
1329	rm -f hosts.tmp
1330
1331	listnodes=${host[*]}
1332
1333	EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1334
1335	# Initialisation
1336	rank=0
1337	current_core=0
1338	init_exec=n
1339
1340	# Loop on the components
1341	for comp in ${config_ListOfComponents[*]} ; do
1342
1343	eval ExeNameIn=\${config_Executable_${comp}[0]}
1344	eval ExeNameOut=\${config_Executable_${comp}[1]}
1345
1346	# Not possible if oasis has an executable (i.e old version of oasis3)
1347	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1348	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1349	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1350	IGCM_debug_Verif_Exit
1351	fi
1352
1353	# Only if we really have an executable for the component :
1354	if [ "X${ExeNameOut}" != X\"\" ] ; then
1355
1356	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1357	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1358	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1359
1360	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1361	echo "" >> script_${ExeNameOut}.ksh
1362	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1363
1364	# Check if the number of threads is correct
1365	case ${comp_proc_omp_loc} in
1366	2\|4\|8\|16)
1367	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1368	;;
1369	*)
1370	IGCM_debug_Exit "ERROR with OMP parameters !"
1371	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1372	IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1373	IGCM_debug_Verif_Exit
1374	;;
1375	esac
1376	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1377	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1378	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1379	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1380	fi
1381
1382	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1383	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1384	echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1385	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1386	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1387
1388	if [ ${init_exec} = y ] ; then
1389	EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1390	else
1391	EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1392	init_exec=y
1393	fi
1394
1395	# Build rankfile : method used to assign cores and nodes for the MPI process
1396	# Ex :
1397	#rank 0=curie5296 slot=0,1,2,3
1398	#rank 1=curie5296 slot=4,5,6,7
1399	# Example of final command :
1400	# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1401	# with script_lmdz.x.ksh :
1402	# #!/bin/ksh
1403	#export KMP_STACKSIZE=3g
1404	#export KMP_LIBRARY=turnaround
1405	#export MKL_SERIAL=YES
1406	#OMP_NUM_THREADS=4
1407	#./lmdz.x
1408	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1409	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1410	else
1411	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1412	fi
1413	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1414	(( index_host = current_core / NB_CORE_PER_NODE ))
1415	host_value=${host[${index_host}]}
1416	(( slot = current_core % NB_CORE_PER_NODE ))
1417	virg=","
1418	string_final=""
1419	for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1420	string=$index$virg
1421	string_final=$string_final$string
1422	done
1423	string_final=$( echo $string_final \| sed "s/.$//" )
1424	echo "rank $rank=$host_value slot=$string_final" >> rankfile
1425	(( rank = rank + 1 ))
1426	(( current_core = current_core + offset_comp_proc_loc ))
1427	done
1428	fi
1429	done
1430
1431	elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] \|\| [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1432
1433	# 3 MPI/OpenMP ccc_mprun method
1434
1435	# MPI-OpenMP (MPMD)
1436	# example of run_file
1437	# 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1438	# 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1439	# 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1440
1441	# Loop on the components
1442	for comp in ${config_ListOfComponents[*]} ; do
1443
1444	eval ExeNameIn=\${config_Executable_${comp}[0]}
1445	eval ExeNameOut=\${config_Executable_${comp}[1]}
1446
1447	# Not possible if oasis has an executable (i.e old version of oasis3)
1448	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1449	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1450	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1451	IGCM_debug_Verif_Exit
1452	fi
1453
1454	# Only if we really have an executable for the component :
1455	if [ "X${ExeNameOut}" != X\"\" ] ; then
1456
1457	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1458	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1459
1460	# Build script files
1461
1462	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1463	echo "" >> script_${ExeNameOut}.ksh
1464	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1465
1466	# Check if the number of threads is correct
1467	case ${comp_proc_omp_loc} in
1468	2\|4\|8\|12\|16\|24)
1469	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1470	;;
1471	*)
1472	IGCM_debug_Exit "ERROR with OMP parameters !"
1473	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1474	IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1475	IGCM_debug_Verif_Exit
1476	;;
1477	esac
1478	### only for LMDZ?
1479	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1480	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1481	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1482	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1483
1484	fi
1485
1486	# to have out/err per process on different files
1487	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1488	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1489
1490	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1491
1492	# Complete run_file
1493
1494	echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1495
1496	fi
1497	done
1498
1499	## variable added to stop after 60s instead of 600s by default.
1500	## This is used when no error comes from executables and when something stopped an executable without notice.
1501	export SLURM_WAIT=60
1502
1503	## module advised by TGCC (instead of 2 variables)
1504	module load feature/bridge/heterogenous_mpmd
1505
1506	# set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1507
1508	EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1509
1510	IGCM_sys_Chmod u+x run_file
1511	if ( $DEBUG_sys ) ; then
1512	echo "run_file contains : "
1513	cat run_file
1514	fi
1515
1516	else
1517
1518	IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1519
1520	fi # 1 2 or 3 if ${OK_PARA_MPMD}
1521
1522	else
1523	# Only one executable (SPMD mode).
1524
1525	for comp in ${config_ListOfComponents[*]} ; do
1526
1527	# Only if we really have an executable for the component :
1528	eval ExeNameOut=\${config_Executable_${comp}[1]}
1529	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1530
1531	# Build script files
1532
1533	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1534	echo "" >> script_${ExeNameOut}.ksh
1535	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1536
1537	if ( ${OK_PARA_OMP} ) ; then
1538	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1539	echo "" >> script_${ExeNameOut}.ksh
1540	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1541	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1542	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1543	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1544	fi
1545
1546	if ( ${OK_PARA_MPI} ) ; then
1547	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1548	# Default : ccc_mprun used if nb_proc gt 1
1549	# to have out/err per process on different files
1550	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1551	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1552	EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1553	else
1554	# Default : ccc_mprun is NOT used if nb_proc eq 1
1555	# to have out/err per process on different files
1556	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1557	#echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1558	EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1559	fi
1560
1561	IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1562	cat script_${ExeNameOut}.ksh
1563
1564	fi
1565
1566	done
1567
1568	fi # ${OK_PARA_MPMD}
1569
1570	IGCM_debug_Print 1 "sys Irene : execution command is "
1571	IGCM_debug_Print 1 "$EXECUTION"
1572
1573	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1574	}
1575
1576	#D-#==================================================
1577	#D-function IGCM_sys_check_path
1578	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1579	#D-* do not point to an important use directory. Stop immediately in that case.
1580	#D-* Examples:
1581	#D-
1582	function IGCM_sys_check_path {
1583	IGCM_debug_PushStack "IGCM_sys_check_path"
1584	if ( $DEBUG_sys ) ; then
1585	echo "IGCM_sys_check_path"
1586	fi
1587
1588	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] \|\| [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1589	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1590	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1591	IGCM_debug_Exit "This will stop the job"
1592	fi
1593	IGCM_debug_PopStack "IGCM_sys_check_path"
1594	}
1595
1596	#D-#==================================================
1597	#D-function IGCM_sys_check_quota
1598	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1599	#D-* Examples:
1600	#D-
1601	function IGCM_sys_check_quota {
1602	IGCM_debug_PushStack "IGCM_sys_check_quota"
1603	if ( $DEBUG_sys ) ; then
1604	echo "IGCM_sys_check_quota"
1605	fi
1606	# Limit of quota (in %)
1607	limit_quota=90
1608
1609	# Check of the volume
1610	volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1611	volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1612
1613	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1614
1615	unit_avail=${volume_avail: -1}
1616	unit_quota=${volume_quota: -1}
1617
1618	if [ "${unit_quota}" = "*" ] ; then
1619	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1620	IGCM_debug_Print 1 "More than 100% of your quota is used"
1621	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1622	IGCM_debug_Print 1 "You must have more than 10% available to run"
1623	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1624	IGCM_debug_Verif_Exit
1625	fi
1626
1627	temp_avail=${volume_avail%%${volume_avail: -1}*}
1628	temp_quota=${volume_quota%%${volume_quota: -1}*}
1629
1630	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1631
1632	# Convertion
1633	if [ ${volume_avail: -1} = "T" ] ; then
1634	(( temp_avail = temp_avail * 1000000000000 ))
1635	elif [ ${volume_avail: -1} = "G" ] ; then
1636	(( temp_avail = temp_avail * 1000000000 ))
1637	elif [ ${volume_avail: -1} = "M" ] ; then
1638	(( temp_avail = temp_avail * 1000000 ))
1639	elif [ ${volume_avail: -1} = "k" ] ; then
1640	(( temp_avail = temp_avail * 1000 ))
1641	else
1642	(( temp_avail = volume_avail ))
1643	fi
1644	if [ ${volume_quota: -1} = "T" ] ; then
1645	(( temp_quota = temp_quota * 1000000000000 ))
1646	elif [ ${volume_quota: -1} = "G" ] ; then
1647	(( temp_quota = temp_quota * 1000000000 ))
1648	elif [ ${volume_quota: -1} = "M" ] ; then
1649	(( temp_quota = temp_quota * 1000000 ))
1650	elif [ ${volume_quota: -1} = "k" ] ; then
1651	(( temp_quota = temp_quota * 1000 ))
1652	else
1653	(( temp_quota = volume_quota ))
1654	fi
1655	fi
1656
1657	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1658	# echo "volume ratio is " $quota_volume
1659
1660	if [ ${quota_volume} -ge ${limit_quota} ] ; then
1661	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1662	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1663	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1664	IGCM_debug_Print 1 "You must have more than 10% available to run"
1665	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1666	IGCM_debug_Verif_Exit
1667	fi
1668
1669	fi
1670
1671	# Check of the number of inodes
1672
1673	inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1674	inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1675
1676	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1677
1678	unit_avail=${inode_avail: -1}
1679	unit_quota=${inode_quota: -1}
1680
1681	if [ "${unit_quota}" = "*" ] ; then
1682	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1683	IGCM_debug_Print 1 "More than 100% of your quota is used"
1684	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1685	IGCM_debug_Print 1 "You must have more than 10% available to run"
1686	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1687	IGCM_debug_Verif_Exit
1688	fi
1689
1690	temp_avail=${inode_avail%%${inode_avail: -1}*}
1691	temp_quota=${inode_quota%%${inode_quota: -1}*}
1692
1693	if [ ! ${unit_avail} = ${unit_quota} ] ; then
1694
1695	# Convertion
1696	if [ ${inode_avail: -1} = "T" ] ; then
1697	(( temp_avail = temp_avail * 1000000000000 ))
1698	elif [ ${inode_avail: -1} = "G" ] ; then
1699	(( temp_avail = temp_avail * 1000000000 ))
1700	elif [ ${inode_avail: -1} = "M" ] ; then
1701	(( temp_avail = temp_avail * 1000000 ))
1702	elif [ ${inode_avail: -1} = "k" ] ; then
1703	(( temp_avail = temp_avail * 1000 ))
1704	else
1705	(( temp_avail = inode_avail ))
1706	fi
1707
1708	if [ ${inode_quota: -1} = "T" ] ; then
1709	(( temp_quota = temp_quota * 1000000000000 ))
1710	elif [ ${inode_quota: -1} = "G" ] ; then
1711	(( temp_quota = temp_quota * 1000000000 ))
1712	elif [ ${inode_quota: -1} = "M" ] ; then
1713	(( temp_quota = temp_quota * 1000000 ))
1714	elif [ ${inode_quota: -1} = "k" ] ; then
1715	(( temp_quota = temp_quota * 1000 ))
1716	else
1717	(( temp_quota = inode_quota ))
1718	fi
1719	fi
1720	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1721	# echo "inode ratio is " $quota_inode
1722
1723	if [ ${quota_inode} -ge ${limit_quota} ] ; then
1724	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1725	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1726	IGCM_debug_Print 1 "Use the ccc_quota command to check"
1727	IGCM_debug_Print 1 "You must have more than 10% available to run"
1728	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1729	IGCM_debug_Verif_Exit
1730	fi
1731	fi
1732	IGCM_debug_PopStack "IGCM_sys_check_quota"
1733	}
1734
1735	#D-#==================================================
1736	#D-function IGCM_sys_projectAccounting
1737	#D-* Purpose: store project accounting information in a file
1738	#D-* Examples:
1739	#D-
1740	function IGCM_sys_projectAccounting {
1741	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1742	if ( $DEBUG_sys ) ; then
1743	echo "IGCM_sys_check_quota"
1744	fi
1745
1746	ssh irene191 /usr/bin/ccc_myproject > $1
1747
1748	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1749	}
1750
1751	#D-#==================================================
1752	#D-function IGCM_sys_getJobSchedulerID
1753	#D-* Purpose: Get the job ID during execution
1754	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1755	#D-
1756	function IGCM_sys_getJobSchedulerID {
1757	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1758	if ( $DEBUG_sys ) ; then
1759	echo "IGCM_sys_getJobSchedulerID"
1760	fi
1761
1762	eval ${1}=${BRIDGE_MSUB_JOBID}
1763
1764	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1765	}
1766
1767	#D-#==================================================
1768	#D-function IGCM_sys_GetJobID
1769	#D-* Purpose: Get the job ID from the JobName
1770	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1771	#D-
1772	function IGCM_sys_GetJobID {
1773	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1774	if ( $DEBUG_sys ) ; then
1775	echo "IGCM_sys_GetJobID"
1776	fi
1777
1778	# With -f option, the full job name is given in the last column
1779	ID=$( ccc_mstat -f -u $2 \| \
1780	gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1781
1782	eval ${3}=${ID}
1783	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1784	}
1785
1786	#D-#==================================================
1787	#D-function IGCM_sys_CountJobInQueue
1788	#D-* Purpose: Count number of users job
1789	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1790	#D-
1791	function IGCM_sys_CountJobInQueue {
1792	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1793	if ( $DEBUG_sys ) ; then
1794	echo "IGCM_sys_CountJobInQueue"
1795	fi
1796
1797	# With -f option, the full job name is given in the last column
1798	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1799
1800	eval ${2}=${NbRun}
1801
1802	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1803	}
1804
1805	#D-#==================================================
1806	#D-function IGCM_sys_ListJobInQueue
1807	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1808	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1809	#D-
1810	function IGCM_sys_ListJobInQueue {
1811	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1812	if ( $DEBUG_sys ) ; then
1813	echo "IGCM_sys_ListJobInQueue"
1814	fi
1815
1816	# With -f option, the full job name is given in the last column
1817	set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1818	'( $2 == User && \
1819	$NF != /TS/ && \
1820	$NF !~ /PACK/ && \
1821	$NF !~ /REBUILD/ && \
1822	$NF !~ /pack/ ) \
1823	{ print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1824
1825	eval set -A ${2} ${JobList[*]}
1826
1827	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1828	}
1829
1830	#D-#==================================================
1831	#D-function IGCM_sys_atlas
1832	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1833	#D-* Examples:
1834	#D-
1835	function IGCM_sys_atlas {
1836	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1837	if ( $DEBUG_sys ) ; then
1838	echo "IGCM_sys_atlas :" $@
1839	fi
1840
1841	typeset status
1842
1843	\ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1844	status=$?
1845	if [ ${status} -gt 0 ] ; then
1846	echo "IGCM_sys_atlas : error code ${status}"
1847	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1848	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1849	IGCM_debug_PopStack "IGCM_sys_atlas"
1850	return 1
1851	else
1852	IGCM_debug_PopStack "IGCM_sys_atlas"
1853	return 0
1854	fi
1855
1856	IGCM_debug_PopStack "IGCM_sys_atlas"
1857	}
1858
1859	#D-#==================================================
1860	#D-function IGCM_sys_rebuild_nemo
1861	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1862	#D-* Examples:
1863	#D-
1864
1865	function IGCM_sys_rebuild_nemo {
1866	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1867	if ( $DEBUG_sys ) ; then
1868	echo "IGCM_sys_rebuild_nemo :" $@
1869	fi
1870
1871	$( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1872
1873	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1874	}
1875

Note: See TracBrowser for help on using the repository browser.

Download in other formats: