Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh @ 1522

Last change on this file since 1522 was 1517, checked in by jgipsl, 4 years ago
Revert commit [1515], see ticket #348
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 58.6 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Jean-Zay
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe/param/files \| \| \| \|
46	# \| \| Chmod Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host user names project maxCpuTime
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=jeanzay
71	# add default project set by ins_job
72	typeset PROJECT=$( echo ${SLURM_JOB_ACCOUNT:=::default_project::} \| cut -c "1-3" )
73	### jobWarningDelay in seconds
74	##typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76	#D-
77	#D-#==================================================
78	#D-Program used in libIGCM
79	#D-#==================================================
80
81	# Submit command
82	typeset SUBMIT=${SUBMIT:=sbatch}
83	# rsync with path
84	typeset -r RSYNC=/usr/bin/rsync
85	# RSYNC_opt args to rsync
86	typeset -r RSYNC_opt="-va"
87	# ie storage filesystem
88	typeset -r STOREHOST=${MASTER}
89	typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91	#====================================================
92	# Access to module command
93	#====================================================
94	# See ticket #348
95	# Following path should be used according to assist IDRIS but it makes errors furter below
96	#. /gpfslocalsup/spack_soft/environment-modules/current/init/ksh
97	# Following old path gives error messages while loading modules but it still works
98	. /usr/share/Modules/init/ksh
99
100	#====================================================
101	# Set environment tools (ferret, nco, cdo, rebuild, ...)
102	#====================================================
103	if [ X${TaskType} = Xcomputing ] ; then
104	IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
105	else
106	module purge > /dev/null 2>&1
107	. /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1
108
109	. /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1
110
111	# export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
112	export UVCDAT_ANONYMOUS_LOG=FALSE
113	fi
114
115	# FYI
116	[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
117	[ ! X${TaskType} = Xchecking ] && module list
118
119	# For AddNoise (not yet AddPertu3DOCE)
120	export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/AddNoise/bin
121
122	# For rebuild
123	export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
124
125	#====================================================
126	# Host specific DIRECTORIES
127	#====================================================
128
129	#====================================================
130	#- MirrorlibIGCM for frontend
131	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132
133	#====================================================
134	#- libIGCM_POST for frontend
135	typeset -r libIGCM_POST=${libIGCM}
136
137	#====================================================
138	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
139	typeset -r R_EXE="${MODIPSL}/bin"
140
141	#====================================================
142	#- SUBMIT_DIR : submission dir
143	typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
144
145	#====================================================
146	#- IN
147	typeset -r R_IN=${R_IN:=/gpfswork/rech/psl/commun/IGCM}
148
149	#====================================================
150	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
151	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}}
152
153	#====================================================
154	#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
155	typeset -r OUTCOMMAND_PATH=/tmp
156
157	#====================================================
158	#- HOST_MPIRUN_COMMAND
159	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time srun "}
160
161	#====================================================
162	#- Max number of arguments passed to nco operator or demigration command
163	UNIX_MAX_LIMIT=360
164
165	#====================================================
166	#- set PackDefault to true
167	PackDefault=true
168
169	#====================================================
170	#- Default number of MPI task for IPSL coupled model
171	#- required for backward compatibility
172	#-
173	DEFAULT_NUM_PROC_OCE=5
174	DEFAULT_NUM_PROC_CPL=1
175	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
176
177	#====================================================
178	#- Number of computing cores per node
179	#-
180	NB_CORE_PER_NODE=40
181
182	#D-#==================================================
183	#D-function IGCM_sys_defineArchives
184	#D-* Purpose:
185	#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
186	#D-* Define ARCHIVE : Dedicated to large files
187	#D-* Define STORAGE : Dedicated to small/medium files
188	#D-* Define R_OUT : Output tree located on ARCHIVE
189	#D-* Define R_FIG : Output tree located on STORAGE hosting figures (monsitoring and atlas, and/or small files)
190	#D-* Define R_BUF : Output tree located on SCRATCH hosting files waiting for rebuild or pack processes
191	#D-* if SpaceName=TEST everything is stored on SCRATCH
192	#D-* Examples:
193	#D-
194	function IGCM_sys_defineArchives {
195	IGCM_debug_PushStack "IGCM_sys_defineArchives"
196
197	# Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
198	if [ X${config_UserChoices_DataProject} = X ] \|\| [ X${config_UserChoices_DataProject} = DEFAULT ]; then
199	# The variable DataProject is not set in in section UserChoices in config.card or it is set to DEFAULT
200	# Default option: Set DataProject according to the project used for submission
201	DataProject=${PROJECT}
202	else
203	# Use the project set by the variable DataProject in section UserChoices in config.card
204	IGCM_debug_Print 1 "Use DataProject as set in config.card : ${config_UserChoices_DataProject}"
205	DataProject=${config_UserChoices_DataProject}
206	fi
207	IGCM_debug_Print 1 "Data project for output to be used: ${DataProject}"
208
209	#====================================================
210	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
211	#- DataProject = for is the dataproject for Training account at Idris
212	#- they are not store in rech but in another partition
213	if [ X${DataProject} = Xfor ]; then
214	typeset -r RUN_DIR_PATH=$SCRATCH/RUN_DIR/${SLURM_JOBID}_${$}
215	else
216	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/gpfsscratch/rech/${DataProject}/${LOGIN}/RUN_DIR/${SLURM_JOBID}_${$}}
217	fi
218
219	if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
220	#====================================================
221	#- ARCHIVE (dedicated to large files)
222	ARCHIVE=${config_UserChoices_ARCHIVE}
223	else
224	if [ X${DataProject} = Xfor ]; then
225	ARCHIVE=$STORE
226	else
227	#====================================================
228	#- ARCHIVE (dedicated to large files)
229	ARCHIVE=/gpfsstore/rech/${DataProject}/${LOGIN}
230	fi
231	fi
232
233	if [ ! X${config_UserChoices_STORAGE} = X ]; then
234	#====================================================
235	#- STORAGE (dedicated to small/medium files)
236	STORAGE=${config_UserChoices_STORAGE}
237	else
238	#====================================================
239	#- STORAGE (dedicated to small/medium files)
240	if [ X${DataProject} = Xfor ]; then
241	STORAGE=$WORK
242	else
243	STORAGE=/gpfswork/rech/${DataProject}/${LOGIN}
244	fi
245	fi
246
247	if [ X${config_UserChoices_SpaceName} = XTEST ]; then
248	#====================================================
249	#- R_OUT
250	if [ X${DataProject} = Xfor ]; then
251	R_OUT=$SCRATCH/IGCM_OUT
252	else
253	R_OUT=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
254	fi
255	#====================================================
256	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
257	R_FIG=${R_OUT}
258
259	IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
260
261	else
262	#====================================================
263	#- R_OUT
264	R_OUT=${ARCHIVE}/IGCM_OUT
265
266	#====================================================
267	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
268	R_FIG=${STORAGE}/IGCM_OUT
269	fi
270
271	#====================================================
272	#- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
273	CMIP6_BUF=${STORAGE}/IGCM_OUT
274
275	#====================================================
276	#- R_BUF (ONLY FOR double copy an scratch)
277
278	if [ X${DataProject} = Xfor ]; then
279	R_BUF=$SCRATCH/IGCM_OUT
280	else
281	R_BUF=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
282	fi
283
284	IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
285	IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
286	IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
287
288	IGCM_debug_PopStack "IGCM_sys_defineArchives"
289	}
290
291	#D-#==================================================
292	#D-function IGCM_sys_RshArchive
293	#D-* Purpose: Archive rsh command
294	#D-* Examples:
295	#D-
296	function IGCM_sys_RshArchive {
297	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
298	/bin/ksh <<-EOF
299	${@}
300	EOF
301	status=$?
302	if [ ${status} -gt 0 ] ; then
303	IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
304	IGCM_debug_Exit "IGCM_sys_RshArchive"
305	fi
306	IGCM_debug_PopStack "IGCM_sys_RshArchive"
307	}
308
309	#D-#==================================================
310	#D-function IGCM_sys_RshArchive_NoError
311	#D-* Purpose: Archive rsh command, without error
312	#D-* used only in monitoring.job
313	#D-* Examples:
314	#D-
315	function IGCM_sys_RshArchive_NoError {
316	IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
317	/bin/ksh <<-EOF
318	${@} 2> /dev/null
319	EOF
320	IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
321	}
322
323	#D-#==================================================
324	#D-function IGCM_sys_MkdirArchive
325	#D-* Purpose: Mkdir on Archive
326	#D-* Examples:
327	#D-
328	function IGCM_sys_MkdirArchive {
329	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
330	if ( $DEBUG_sys ) ; then
331	echo "IGCM_sys_MkdirArchive :" $@
332	fi
333	#- creation de repertoire sur le serveur fichier
334	if [ ! -d ${1} ]; then
335	\mkdir -p $1
336	status=$?
337
338	if [ ${status} -gt 0 ] ; then
339	IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
340	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
341	fi
342	fi
343	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
344	}
345
346	#D-#==================================================
347	#D-function IGCM_sys_TestDirArchive
348	#D-* Purpose: Test Directory that must exists on Archive
349	#D-* Examples:
350	#D-
351	function IGCM_sys_TestDirArchive {
352	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
353	if ( $DEBUG_sys ) ; then
354	echo "IGCM_sys_TestDirArchive :" $@
355	fi
356	typeset ExistFlag
357	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
358	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
359	return ${ExistFlag}
360	}
361
362	#D-#==================================================
363	#D-function IGCM_sys_IsFileArchived
364	#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
365	#D-* Examples:
366	#D-
367	function IGCM_sys_IsFileArchived {
368	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
369	if ( $DEBUG_sys ) ; then
370	echo "IGCM_sys_IsFileArchived :" $@
371	fi
372	typeset IsArchivedFlag
373	# To be implemented on Jean Zay
374	# IsArchivedFlag=$( [ X$( echo $@ \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
375	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
376
377	return ${IsArchivedFlag}
378	}
379
380	#D-#==================================================
381	#D-function IGCM_sys_TestFileArchive
382	#D-* Purpose: Test file that must NOT EXISTS on Archive
383	#D-* Examples:
384	#D-
385	function IGCM_sys_TestFileArchive {
386	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
387	typeset ExistFlag
388	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
389	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
390
391	return ${ExistFlag}
392	}
393
394	#D-#==================================================
395	#D-function IGCM_sys_CountFileArchive
396	#D-* Purpose: Count files on Archive filesystem
397	#D-* Examples:
398	#D-
399	function IGCM_sys_CountFileArchive {
400	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
401	ls ${@} 2>/dev/null \| wc -l
402	if [ $? -gt 0 ] ; then
403	echo "IGCM_sys_CountFileArchive : erreur."
404	fi
405	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
406	}
407
408	#D-#==================================================
409	#D-function IGCM_sys_Tree
410	#D-* Purpose: Tree directories with files on ${ARCHIVE}
411	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
412	#D-
413	function IGCM_sys_Tree {
414	IGCM_debug_PushStack "IGCM_sys_Tree" $@
415	if ( $DEBUG_sys ) ; then
416	echo "IGCM_sys_Tree :" $@
417	fi
418
419	\ls -lR ${@}
420
421	IGCM_debug_PopStack "IGCM_sys_Tree"
422	}
423
424	#D-#==================================================
425	#D-function IGCM_sys_Qsub
426	#D-* Purpose: Qsub new job
427	#D-* Examples:
428	#D-
429	function IGCM_sys_Qsub {
430	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
431	if ( $DEBUG_sys ) ; then
432	echo "IGCM_sys_Qsub :" $@
433	fi
434	typeset options status
435	options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
436
437	/usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
438	status=$?
439
440	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
441	if [ ${status} -gt 0 ] ; then
442	IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
443	IGCM_debug_Exit "IGCM_sys_Qsub"
444	else
445	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
446	fi
447	IGCM_debug_PopStack "IGCM_sys_Qsub"
448	}
449
450	#D-#==================================================
451	#D-function IGCM_sys_QsubPost
452	#D-* Purpose: Qsub new job on scalaire
453	#D-* Examples:
454	#D-
455	function IGCM_sys_QsubPost {
456	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
457	if ( $DEBUG_sys ) ; then
458	echo "IGCM_sys_QsubPost :" $@
459	fi
460	typeset options status
461	options="-o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
462
463	sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
464	status=$?
465
466	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
467	if [ ${status} -gt 0 ] ; then
468	IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
469	IGCM_debug_Exit "IGCM_sys_QsubPost"
470	else
471	JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
472	fi
473	IGCM_debug_PopStack "IGCM_sys_QsubPost"
474	}
475
476	#D-*************************
477	#D- File transfer functions
478	#D-*************************
479	#D-
480
481	#D-#==================================================
482	#D-function IGCM_sys_RmRunDir
483	#D-* Purpose: rm tmpdir (dummy function most of the time batch
484	#D- scheduler will do the job)
485	#D-* Examples:
486	#D-
487	function IGCM_sys_RmRunDir {
488	IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
489	if ( $DEBUG_sys ) ; then
490	echo "IGCM_sys_RmRunDir :" $@
491	fi
492
493	typeset status
494
495	echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
496	\rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
497	status=$?
498
499	if [ ${status} -gt 0 ] ; then
500	IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
501	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
502	IGCM_debug_Exit "IGCM_sys_RmRunDir"
503	else
504	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
505	fi
506	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
507	}
508
509	#D-#==================================================
510	#D-function IGCM_sys_Put_Dir
511	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
512	#D-* Examples:
513	#D-
514	function IGCM_sys_Put_Dir {
515	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
516	if ( $DEBUG_sys ) ; then
517	echo "IGCM_sys_Put_Dir :" $@
518	fi
519	if [ $DRYRUN = 0 ]; then
520	if [ ! -d ${1} ] ; then
521	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
522	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
523	return
524	fi
525
526	typeset status
527
528	# Only if we use rsync
529	#IGCM_sys_TestDirArchive $( dirname $2 )
530	#
531	#USUAL WAY
532	\cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
533	status=$?
534
535	if [ ${status} -gt 0 ] ; then
536	IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
537	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
538	IGCM_debug_Exit "IGCM_sys_Put_Dir"
539	else
540	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
541	fi
542	fi
543	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
544	}
545
546	#D-#==================================================
547	#D-function IGCM_sys_Get_Dir
548	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
549	#D-* Examples:
550	#D-
551	function IGCM_sys_Get_Dir {
552	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
553	if ( $DEBUG_sys ) ; then
554	echo "IGCM_sys_Get_Dir :" $@
555	fi
556	if [ $DRYRUN = 0 ]; then
557	typeset NB_ESSAI DELAI status i
558	# number of tentative
559	NB_ESSAI=3
560	# time delay between tentative
561	DELAI=2
562
563	#
564	# USUAL WAY
565	# add here command to demigrate all offline files to reduce time of this command
566	# actually on Jean Zay files are not archive on bands
567
568	i=0
569	while [ $i -lt $NB_ESSAI ] ; do
570	\cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
571	status=$?
572	if [ ${status} -gt 0 ] ; then
573	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
574	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
575	sleep $DELAI
576	else
577	break
578	fi
579	(( i = i + 1 ))
580	done
581
582	if [ ${status} -gt 0 ] ; then
583	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
584	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
585	IGCM_debug_Exit "IGCM_sys_Get_Dir"
586	else
587	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
588	fi
589	fi
590	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
591	}
592
593	#D-#==================================================
594	#D-function IGCM_sys_Put_Rest
595	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
596	#D- File and target directory must exist.
597	#D-* Examples:
598	#D-
599	function IGCM_sys_Put_Rest {
600	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
601	if ( $DEBUG_sys ) ; then
602	echo "IGCM_sys_Put_Rest :" $@
603	fi
604	if [ $DRYRUN = 0 ]; then
605	if [ ! -f ${1} ] ; then
606	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
607	IGCM_debug_Exit "IGCM_sys_Put_Rest"
608	fi
609
610	typeset status
611	#
612	# USUAL WAY
613	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
614	status=$?
615
616	# #RSYNC WITH NETWORK SSH CALL
617	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
618	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
619
620	# #RSYNC WITH NFS USE
621	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
622	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
623
624	# status=$?
625	# IGCM_sys_Rsync_out $status
626
627	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
628	# (( status=status+$? ))
629
630	if [ ${status} -gt 0 ] ; then
631	IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
632	[ -f ${1} ] && ls -l ${1}
633	[ -f ${2} ] && ls -l ${2}
634	[ -f ${2}/${1} ] && ls -l ${2}/${1}
635	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636	IGCM_debug_Exit "IGCM_sys_Put_Rest"
637	else
638
639	if [ X${JobType} = XRUN ] ; then
640	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
641	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
642	fi
643
644	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645	fi
646	fi
647	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
648	}
649
650	#D-#==================================================
651	#D-function IGCM_sys_Put_Out
652	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
653	#D-* Examples:
654	#D-
655	function IGCM_sys_Put_Out {
656	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
657	if ( $DEBUG_sys ) ; then
658	echo "IGCM_sys_Put_Out :" $@
659	fi
660
661	typeset NB_ESSAI DELAI status i exist skip
662	typeset fileDeviceNumberInHex directoryDeviceNumberInHex
663
664	# number of tentative
665	NB_ESSAI=3
666	# time delay between tentative
667	DELAI=2
668
669	if [ $DRYRUN = 0 ]; then
670	if [ ! -f ${1} ] ; then
671	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
672	IGCM_debug_PopStack "IGCM_sys_Put_Out"
673	return 1
674	fi
675	#
676	IGCM_sys_MkdirArchive $( dirname $2 )
677	#
678	exist=false
679	skip=false
680	if [ -f $2 ] ; then
681	IGCM_debug_Print 1 "$2 already exist"
682	# add here command to demigrate all offline files to reduce time of this command
683	# actually on Jean Zay files are not archive on bands
684
685	exist=true
686	if [ "X$( diff $1 $2 )" = X ] ; then
687	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
688	skip=true
689	else
690	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
691	skip=false
692	fi
693	fi
694	#
695	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
696	IGCM_sys_Chmod u+w $2
697	fi
698
699	if [ X${skip} = Xfalse ] ; then
700	i=0
701	while [ $i -lt $NB_ESSAI ] ; do
702	# Identify file system
703	fileDeviceNumberInHex=$( stat -c %d $1 )
704	status=$?
705	if [ ${status} -gt 0 ] ; then
706	IGCM_debug_Exit "IGCM_sys_Put_Out"
707	fi
708	# Identify file system
709	directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
710	status=$?
711	if [ ${status} -gt 0 ] ; then
712	IGCM_debug_Exit "IGCM_sys_Put_Out"
713	fi
714
715	if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] \|\| [ X$3 = XNOMOVE ] ) ; then
716	# They are not on the same device. USUAL WAY
717	\cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718	status=$?
719	else
720	# They are on the same device. NOT SO USUAL WAY
721	\mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
722	status=$?
723	fi
724	if [ ${status} -gt 0 ]; then
725	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
726	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
727	[ -f ${1} ] && ls -l ${1}
728	[ -f ${2} ] && ls -l ${2}
729	[ -f ${2}/${1} ] && ls -l ${2}/${1}
730	sleep $DELAI
731	else
732	break
733	fi
734	(( i = i + 1 ))
735	done
736	fi
737
738	# #RSYNC WITH NETWORK SSH CALL
739	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
740	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741
742	# #RSYNC WITH NFS USE
743	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
744	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
745
746	# status=$?
747	# IGCM_sys_Rsync_out $status
748
749	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
750	# (( status=status+$? ))
751
752	if [ ${status} -gt 0 ] ; then
753	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
754	[ -f ${1} ] && ls -l ${1}
755	[ -f ${2} ] && ls -l ${2}
756	[ -f ${2}/${1} ] && ls -l ${2}/${1}
757	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
758	IGCM_debug_Exit "IGCM_sys_Put_Out"
759	else
760
761	if [ X${JobType} = XRUN ] ; then
762	if [ X${3} = X ] ; then
763	[ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
764	[ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
765	fi
766	fi
767
768	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
769	fi
770	fi
771	IGCM_debug_PopStack "IGCM_sys_Put_Out"
772	return 0
773	}
774
775	#D-#==================================================
776	#D-function IGCM_sys_Get
777	#D-* Purpose: Get a file from ${ARCHIVE}
778	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
779	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
780	function IGCM_sys_Get {
781	IGCM_debug_PushStack "IGCM_sys_Get" $@
782
783	typeset DEST dm_liste target file_work
784	typeset NB_ESSAI DELAI status i
785
786	if ( $DEBUG_sys ) ; then
787	echo "IGCM_sys_Get :" $@
788	fi
789
790	# number of tentative
791	NB_ESSAI=3
792	# time delay between tentative
793	DELAI=2
794
795	if [ $DRYRUN -le 2 ]; then
796	if [ X${1} = X'/l' ] ; then
797	eval set +A dm_liste \${${2}}
798	else
799	eval set +A dm_liste ${1}
800	fi
801	eval DEST=\${${#}}
802	# add here command to demigrate all offline files to reduce time of this command
803	# actually on Jean Zay files are not archive on bands
804
805	status=$?
806	if [ ${status} -gt 0 ] ; then
807	echo "WARNING IGCM_sys_Get : error code ${status}"
808	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
810	fi
811
812	#if [ ${status} -gt 0 ] ; then
813	# if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
814	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
815	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
816	# sleep 30
817	# echo "We try another time"
818	## dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819	# ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
820	# status=$?
821	# if [ ${status} -gt 0 ] ; then
822	# echo "ERROR IGCM_sys_Get : again demigration error :"
823	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
824	# IGCM_debug_Exit "IGCM_sys_Get"
825	# fi
826	# else
827	# echo "ERROR IGCM_sys_Get : demigration error :"
828	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829	# IGCM_debug_Exit "IGCM_sys_Get"
830	# fi
831	#fi
832
833	# #RSYNC WITH NETWORK SSH CALL
834	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
835	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
836
837	# #RSYNC WITH NFS USE
838	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840
841	# status=$?
842	# IGCM_sys_Rsync_out $status
843
844	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
845	# (( status=status+$? ))
846
847	#USUAL WAY
848	if [ X${1} = X'/l' ] ; then
849	for target in ${dm_liste[*]} ; do
850	local_file=$( basename ${target} )
851	# test if the target file is present before the loop
852	IGCM_sys_TestFileArchive ${target}
853	status=$?
854	if [ ${status} -gt 0 ] ; then
855	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
856	IGCM_debug_Exit "IGCM_sys_Get"
857	else
858	i=0
859	while [ $i -lt $NB_ESSAI ] ; do
860	#if [ X${DoLink} = Xtrue ] ; then
861	# \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862	# status=$?
863	# else
864	# \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
865	# status=$?
866	#fi
867	\ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
868	status=$?
869	if [ ${status} -gt 0 ]; then
870	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
871	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
872	sleep $DELAI
873	else
874	break
875	fi
876	(( i = i + 1 ))
877	done
878	if [ ${status} -gt 0 ] ; then
879	echo "IGCM_sys_Get : error"
880	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
881	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
882	IGCM_debug_Exit "IGCM_sys_Get"
883	else
884	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
885	fi
886	fi
887	done
888	else
889	i=0
890	while [ $i -lt $NB_ESSAI ] ; do
891	\cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
892	status=$?
893	if [ ${status} -gt 0 ]; then
894	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
895	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
896	sleep $DELAI
897	else
898	break
899	fi
900	(( i = i + 1 ))
901	done
902	if [ ${status} -gt 0 ] ; then
903	echo "IGCM_sys_Get : error"
904	cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
905	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
906	IGCM_debug_Exit "IGCM_sys_Get"
907	else
908	\rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
909	fi
910	fi
911	fi
912	IGCM_debug_PopStack "IGCM_sys_Get"
913	}
914
915	#D-#==================================================
916	#D-function IGCM_sys_GetDate_Monitoring
917	#D-* Purpose: get the last year for which the monitoring has been computed
918	#D-* Examples:
919	#D-
920	function IGCM_sys_GetDate_Monitoring {
921	IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
922	if ( $DEBUG_sys ) ; then
923	echo "IGCM_sys_GetDate_Monitoring :" $@
924	fi
925
926	eval ${2}=$( cdo showyear ${1} 2> /dev/null \| gawk '{ print $NF }' )
927
928	IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
929	}
930
931	#D-#==================================================
932	#D-function IGCM_sys_Dods_Rm
933	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
934	#D-* Examples:
935	#D-
936	function IGCM_sys_Dods_Rm {
937	if ( $DEBUG_sys ) ; then
938	echo "IGCM_sys_Dods_Rm :" $@
939	fi
940	typeset status
941	if [ $DRYRUN = 0 ]; then
942
943	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
944	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
945	# echo "Nothing has been done."
946	# return
947	# fi
948
949	if [ "$#" -eq 1 ]; then
950	# $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
951	status=$?
952	else
953	# $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
954	status=$?
955	fi
956
957	# if [ ${status} -gt 0 ] ; then
958	# echo "IGCM_sys_Dods_Rm : error."
959	# cat out_dods_rm
960	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
961	# else
962	# rm out_dods_rm
963	# fi
964
965	fi
966	return $status
967	}
968
969	#D-#==================================================
970	#D-function IGCM_sys_Dods_Cp
971	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
972	#D-* Examples:
973	#D-
974	function IGCM_sys_Dods_Cp {
975	if ( $DEBUG_sys ) ; then
976	echo "IGCM_sys_Dods_Cp :" $@
977	fi
978	typeset status
979	if [ $DRYRUN = 0 ]; then
980
981	# if [ ! -d ${R_SAVE}/${1} ] ; then
982	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
983	# echo "Nothing has been done."
984	# return
985	# fi
986
987	# $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
988
989	status=$?
990
991	# if [ ${status} -gt 0 ] ; then
992	# echo "IGCM_sys_Dods_Cp : error."
993	# cat out_dods_cp
994	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
995	# else
996	# rm out_dods_cp
997	# fi
998
999	fi
1000	return $status
1001	}
1002
1003	#D-#==================================================
1004	#D-function IGCM_sys_Put_Dods
1005	#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1006	#D-* Examples:
1007	#D-
1008	function IGCM_sys_Put_Dods {
1009	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1010	if ( $DEBUG_sys ) ; then
1011	echo "IGCM_sys_Put_Dods :" $@
1012	fi
1013	typeset status
1014	if [ $DRYRUN = 0 ]; then
1015	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1016	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1017	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1018	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1019	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1020	return
1021	fi
1022
1023	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1024	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1025	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1026	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1027	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1028	return
1029	fi
1030	#
1031	if [ -d ${R_SAVE}/${1} ] ; then
1032	cd ${R_SAVE}
1033	elif [ -d ${R_FIGR}/${1} ] ; then
1034	cd ${R_FIGR}
1035	fi
1036
1037	IGCM_sys_Dods_Rm ${1}
1038	IGCM_sys_Dods_Cp ${1}
1039	status=0
1040
1041	if [ ${status} -gt 0 ] ; then
1042	echo "IGCM_sys_Put_Dods : error."
1043	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1044	fi
1045	fi
1046	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1047	}
1048
1049	##############################################################
1050	# REBUILD OPERATOR
1051
1052	#D-#==================================================
1053	#D-function IGCM_sys_sync
1054	#D-* Purpose: flush buffer on disk
1055	#D-* Examples:
1056	#D-
1057	function IGCM_sys_sync {
1058	IGCM_debug_PushStack "IGCM_sys_sync" $@
1059	if ( $DEBUG_sys ) ; then
1060	echo "IGCM_sys_sync :" $@
1061	fi
1062
1063	/bin/sync
1064
1065	IGCM_debug_PopStack "IGCM_sys_sync"
1066	}
1067
1068	############################################################
1069	# Activate Running Environnment Variables
1070
1071	#D-#==================================================
1072	#D-function IGCM_sys_activ_variables
1073	#D-* Purpose: set environement variables prior to execution
1074	#D-* Examples:
1075	#D-
1076	function IGCM_sys_activ_variables {
1077	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1078	if ( $DEBUG_sys ) ; then
1079	echo "IGCM_sys_activ_variables"
1080	fi
1081
1082	# --------------------------------------------------------------------
1083	#D- MODULE specifications
1084	# --------------------------------------------------------------------
1085	# Source the file EnvFile containing all module load needed to run the model.
1086	# EnvFile can be specified in config.card. If this is the case and if the file exists,
1087	# it will be used. Otherwise a default file will be used.
1088	if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1089	# EnvFile is set in config.card and the file exists
1090	IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1091	EnvFile=${config_UserChoices_EnvFile}
1092	else
1093	IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1094	module purge > /dev/null 2>&1
1095	EnvFile=/gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay
1096	fi
1097	IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :"
1098	IGCM_debug_Print 1 ${EnvFile}
1099	. ${EnvFile}
1100
1101	IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Jean-Zay. "
1102	module list
1103
1104	# --------------------------------------------------------------------
1105	#D- MPI specifications
1106	# --------------------------------------------------------------------
1107
1108	# --------------------------------------------------------------------
1109	#D- Other specifications
1110	# --------------------------------------------------------------------
1111
1112	ulimit -s unlimited
1113	if [ ${executionType} -eq 2 ] ; then
1114	export SLURM_HOSTFILE=./hostlist
1115	fi
1116
1117
1118	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1119	}
1120
1121	############################################################
1122	# Desactivate Running Environnment Variables
1123
1124	#D-#==================================================
1125	#D-function IGCM_sys_desactiv_variables
1126	#D-* Purpose: unset environement variables after execution
1127	#D-* Examples:
1128	#D-
1129	function IGCM_sys_desactiv_variables {
1130	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1131	if ( $DEBUG_sys ) ; then
1132	echo "IGCM_sys_desactiv_variables"
1133	fi
1134	# --------------------------------------------------------------------
1135	#D- MPI specifications
1136	# --------------------------------------------------------------------
1137	if [ ${executionType} -eq 2 ] ; then
1138	unset SLURM_HOSTFILE
1139	fi
1140	# --------------------------------------------------------------------
1141	#D- Other specifications
1142	# --------------------------------------------------------------------
1143
1144	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1145	}
1146
1147	############################################################
1148	# Update job headers to be used by the scheduler
1149
1150	#D-#==================================================
1151	#D-function IGCM_sys_updateHeaders
1152	#D-* Purpose: Update job headers to be used by the scheduler
1153	#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1154	#D-
1155	function IGCM_sys_updateHeaders {
1156	IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1157	if ( $DEBUG_sys ) ; then
1158	echo "IGCM_sys_updateHeaders"
1159	fi
1160	typeset file
1161	file=$1
1162
1163	if [ ${executionType} -eq 1 ] ; then
1164	# MPMD + MPI
1165	sed -e "/::openMPthreads::/d" \
1166	-e "s/::JobNumProcTot::/${coreNumber}/" \
1167	${file} > ${file}.tmp
1168
1169	elif [ ${executionType} -eq 2 ] ; then
1170	# MPMD + MPI + OMP
1171	(( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1172	[ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1173	sed -e "/::openMPthreads::/d" \
1174	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1175	-e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
1176	-e "/ntasks/i\#SBATCH --exclusive" \
1177	${file} > ${file}.tmp
1178	elif [ ${executionType} -eq 3 ] ; then
1179	# SPMD + MPI/OMP
1180	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1181	-e "s/::JobNumProcTot::/${mpiTasks}/" \
1182	${file} > ${file}.tmp
1183
1184	elif [ ${executionType} -eq 4 ] ; then
1185	# SPMD + MPI only
1186	sed -e "s/::JobNumProcTot::/${mpiTasks}/" \
1187	-e "/::openMPthreads::/d" \
1188	${file} > ${file}.tmp
1189
1190	elif [ ${executionType} -eq 5 ] ; then
1191	# SPMD + OMP only
1192	sed -e "s/::openMPthreads::/${openMPthreads}/" \
1193	-e "/::JobNumProcTot::/d" \
1194	${file} > ${file}.tmp
1195
1196	elif [ ${executionType} -eq 6 ] ; then
1197	# SEQUENTIAL THEN
1198	sed -e "s/::JobNumProcTot::/1/" \
1199	-e "/::openMPthreads::/d" \
1200	${file} > ${file}.tmp
1201
1202	fi
1203
1204	IGCM_sys_Mv ${file}.tmp ${file}
1205
1206	IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1207	}
1208
1209	############################################################
1210	# Build MPI/OMP scripts run file (dummy function)
1211
1212	#D-#==================================================
1213	#D-function IGCM_sys_build_run_file
1214	#D-* Purpose: build run file (deprecated)
1215	#D-* Examples:
1216	#D-
1217	function IGCM_sys_build_run_file {
1218	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1219	}
1220
1221	############################################################
1222	# Build MPI/OMP scripts
1223
1224	#D-#==================================================
1225	#D-function IGCM_sys_build_execution_scripts
1226	#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1227	#D-* Examples:
1228	#D-
1229	function IGCM_sys_build_execution_scripts
1230	{
1231	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1232	if ( $DEBUG_sys ) ; then
1233	echo "IGCM_sys_build_execution_scripts " $@
1234	fi
1235	IGCM_debug_Print 1 "executionType= ${executionType}"
1236	EXECUTION=${HOST_MPIRUN_COMMAND}
1237
1238	if ( ${OK_PARA_MPMD} ) ; then
1239
1240	# MPMD mode
1241	# 1 MPI only : executionType=1
1242	# 2 MPI/OpenMP : executionType=2
1243
1244	if [ -f run_file ] ; then
1245	IGCM_sys_Rm -f run_file
1246	fi
1247	touch run_file
1248
1249	# case 1 : Only MPI (MPMD)
1250	if ( ! ${OK_PARA_OMP} ) ; then
1251	# Build run_file
1252	current_core=0
1253	# First loop on the components for the coupler ie oasis (only if oasis3)
1254	# the coupler ie oasis3 must be the first one
1255	for comp in ${config_ListOfComponents[*]} ; do
1256
1257	eval ExeNameIn=\${config_Executable_${comp}[0]}
1258	eval ExeNameOut=\${config_Executable_${comp}[1]}
1259
1260	# Only if the component has an executable
1261	if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1262
1263	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1264	(( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
1265	echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
1266	(( current_core = ${end_core} + 1 ))
1267	fi
1268	done
1269
1270	EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file"
1271
1272	IGCM_sys_Chmod u+x run_file
1273	if ( $DEBUG_sys ) ; then
1274	echo "run_file contains : "
1275	cat run_file
1276	fi
1277
1278	else
1279
1280	# 2 MPI/OpenMP : executionType=2
1281
1282	# MPI-OpenMP (MPMD)
1283	# export SLURM_HOSTFILE=./hostlist
1284	# srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1285	# example of hostlist file :
1286	# r3i3n33
1287	# r3i3n33
1288	# ...
1289	# example of run_file :
1290	# 0 taskset -c 0-4 ./script_lmdz.x.ksh
1291	# 1 taskset -c 5-9 ./script_lmdz.x.ksh
1292	#...
1293	# 32 taskset -c 0 ./script_opa.xx.ksh
1294	# 33 taskset -c 1 ./script_opa.xx.ksh
1295	#...
1296	# 62 taskset -c 30 ./script_xios.x.ksh
1297
1298	# Hosts treatment
1299	IFS=$'\n'; set -f
1300	listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1301	unset IFS; set +f
1302	rm -f hostlist
1303
1304	# Loop on the components to build run_file and script_exec files
1305	rank=0
1306	current_core=0
1307
1308	for comp in ${config_ListOfComponents[*]} ; do
1309
1310	eval ExeNameIn=\${config_Executable_${comp}[0]}
1311	eval ExeNameOut=\${config_Executable_${comp}[1]}
1312
1313	# Not possible if oasis has an executable (i.e old version of oasis3)
1314	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then
1315	IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1316	IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1317	IGCM_debug_Verif_Exit
1318	fi
1319
1320	# Only if we really have an executable for the component :
1321	if [ "X${ExeNameOut}" != X\"\" ] ; then
1322
1323	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1324	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1325	eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1326
1327
1328	# Build script files
1329
1330	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1331	echo "" >> script_${ExeNameOut}.ksh
1332	if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1333	# Check if the number of threads is correct
1334	case ${comp_proc_omp_loc} in
1335	2\|4\|5\|10\|20)
1336	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1337	;;
1338	*)
1339	IGCM_debug_Exit "ERROR with OMP parameters !"
1340	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1341	IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1342	IGCM_debug_Verif_Exit
1343	;;
1344	esac
1345	echo "export OMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1346	echo "export OMP_PLACES=cores" >> script_${ExeNameOut}.ksh
1347	echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1348
1349	fi
1350
1351	# to have out/err per process on different files
1352	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1353
1354	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1355
1356	# Complete run_file
1357	if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1358	(( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1359	else
1360	(( offset_comp_proc_loc = comp_proc_omp_loc ))
1361	fi
1362
1363	for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1364	(( index_host = current_core / NB_CORE_PER_NODE ))
1365	host_value=${listnodes[${index_host}]}
1366	(( slot = current_core % NB_CORE_PER_NODE ))
1367	echo "$host_value" >> hostlist
1368	echo "$rank taskset -c $slot-$((slot+comp_proc_omp_loc-1)) ./script_${ExeNameOut}.ksh" >> run_file
1369	(( rank = rank + 1 ))
1370	(( current_core = current_core + offset_comp_proc_loc ))
1371	done
1372	fi
1373	done
1374
1375	## variable added to stop after 60s instead of 600s by default.
1376	## This is used when no error comes from executables and when something stopped an executable without notice.
1377	export SLURM_WAIT=60
1378
1379	EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1380
1381	IGCM_sys_Chmod u+x run_file
1382	if ( $DEBUG_sys ) ; then
1383	echo "run_file contains : "
1384	cat run_file
1385	fi
1386
1387	fi # if ${OK_PARA_MPMD}
1388
1389	else
1390	# Only one executable (SPMD mode): executionType=3, 4, 5 and 6
1391
1392	for comp in ${config_ListOfComponents[*]} ; do
1393
1394	# Only if we really have an executable for the component :
1395	eval ExeNameOut=\${config_Executable_${comp}[1]}
1396	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1397
1398	# Build script files
1399
1400	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1401	echo "" >> script_${ExeNameOut}.ksh
1402	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1403
1404	if ( ${OK_PARA_OMP} ) ; then
1405	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1406	# Check if the number of threads is correct
1407	case ${comp_proc_omp_loc} in
1408	2\|4\|5\|10\|20)
1409	IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1410	;;
1411	*)
1412	IGCM_debug_Exit "ERROR with OMP parameters !"
1413	IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1414	IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1415	IGCM_debug_Verif_Exit
1416	;;
1417	esac
1418	echo "" >> script_${ExeNameOut}.ksh
1419	echo "export OMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1420	echo "export OMP_PLACES=cores" >> script_${ExeNameOut}.ksh
1421	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1422	fi
1423
1424	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1425
1426	# To have out/err per process on different files
1427	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1428	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1429
1430	IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1431	cat script_${ExeNameOut}.ksh
1432
1433	fi
1434
1435	done
1436
1437	fi # ${OK_PARA_MPMD}
1438
1439	IGCM_debug_Print 1 "sys Irene : execution command is "
1440	IGCM_debug_Print 1 "$EXECUTION"
1441
1442	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1443	}
1444
1445	#D-#==================================================
1446	#D-function IGCM_sys_check_path
1447	#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1448	#D-* do not point to an important use directory. Stop immediately in that case.
1449	#D-* Examples:
1450	#D-
1451	function IGCM_sys_check_path {
1452	IGCM_debug_PushStack "IGCM_sys_check_path"
1453	if ( $DEBUG_sys ) ; then
1454	echo "IGCM_sys_check_path"
1455	fi
1456
1457	if ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X${WORK} ] \|\| [ X${RUN_DIR_PATH} = X${SCRATCH} ] \|\| [ X${RUN_DIR_PATH} = X${STORE} ] ) ; then
1458	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1459	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1460	IGCM_debug_Exit "This will stop the job"
1461	elif ( [ X${RUN_DIR_PATH} = X${HOME} ] \|\| [ X${RUN_DIR_PATH} = X/gpfswork/rech/${DataProject}/${LOGIN} ] \|\| [ X${RUN_DIR_PATH} = X/gpfsscratch/rech/${DataProject}/${LOGIN} ] \|\| [ X${RUN_DIR_PATH} = X/gpfsstore/rech/${DataProject}/${LOGIN} ] ) ; then
1462	IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1463	IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1464	IGCM_debug_Exit "This will stop the job"
1465	fi
1466	IGCM_debug_PopStack "IGCM_sys_check_path"
1467	}
1468
1469	#D-#==================================================
1470	#D-function IGCM_sys_check_quota
1471	#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1472	#D-* Examples:
1473	#D-
1474	function IGCM_sys_check_quota {
1475	IGCM_debug_PushStack "IGCM_sys_check_quota"
1476	echo "IGCM_sys_check_quota is desactivated on JeanZay because there is actually no way to check scratch quota on this computing center (2019.11.19)"
1477	# if ( $DEBUG_sys ) ; then
1478	# echo "IGCM_sys_check_quota"
1479	# fi
1480	# # Limit of quota (in %)
1481	# limit_quota=90
1482	#
1483	# # Check of the volume
1484	# volume_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $2}')
1485	# volume_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $3}')
1486	#
1487	# if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1488	#
1489	# unit_avail=${volume_avail: -1}
1490	# unit_quota=${volume_quota: -1}
1491	#
1492	# if [ "${unit_quota}" = "*" ] ; then
1493	# IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1494	# IGCM_debug_Print 1 "More than 100% of your quota is used"
1495	# IGCM_debug_Print 1 "Use the ccc_quota command to check"
1496	# IGCM_debug_Print 1 "You must have more than 10% available to run"
1497	# IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1498	# IGCM_debug_Verif_Exit
1499	# fi
1500	#
1501	# temp_avail=${volume_avail%%${volume_avail: -1}*}
1502	# temp_quota=${volume_quota%%${volume_quota: -1}*}
1503	#
1504	# if [ ! ${unit_avail} = ${unit_quota} ] ; then
1505	#
1506	# # Convertion
1507	# if [ ${volume_avail: -1} = "T" ] ; then
1508	# (( temp_avail = temp_avail * 1000000000000 ))
1509	# elif [ ${volume_avail: -1} = "G" ] ; then
1510	# (( temp_avail = temp_avail * 1000000000 ))
1511	# elif [ ${volume_avail: -1} = "M" ] ; then
1512	# (( temp_avail = temp_avail * 1000000 ))
1513	# elif [ ${volume_avail: -1} = "k" ] ; then
1514	# (( temp_avail = temp_avail * 1000 ))
1515	# else
1516	# (( temp_avail = volume_avail ))
1517	# fi
1518	# if [ ${volume_quota: -1} = "T" ] ; then
1519	# (( temp_quota = temp_quota * 1000000000000 ))
1520	# elif [ ${volume_quota: -1} = "G" ] ; then
1521	# (( temp_quota = temp_quota * 1000000000 ))
1522	# elif [ ${volume_quota: -1} = "M" ] ; then
1523	# (( temp_quota = temp_quota * 1000000 ))
1524	# elif [ ${volume_quota: -1} = "k" ] ; then
1525	# (( temp_quota = temp_quota * 1000 ))
1526	# else
1527	# (( temp_quota = volume_quota ))
1528	# fi
1529	# fi
1530	#
1531	# quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1532	## echo "volume ratio is " $quota_volume
1533	#
1534	# if [ ${quota_volume} -ge ${limit_quota} ] ; then
1535	# IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1536	# IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1537	# IGCM_debug_Print 1 "Use the ccc_quota command to check"
1538	# IGCM_debug_Print 1 "You must have more than 10% available to run"
1539	# IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1540	# IGCM_debug_Verif_Exit
1541	# fi
1542	#
1543	# fi
1544	#
1545	## Check of the number of inodes
1546	#
1547	# inode_quota=$(ccc_quota \| grep ' scratch' \| gawk '{print $6}')
1548	# inode_avail=$(ccc_quota \| grep ' scratch' \| gawk '{print $7}')
1549	#
1550	# if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1551	#
1552	# unit_avail=${inode_avail: -1}
1553	# unit_quota=${inode_quota: -1}
1554	#
1555	# if [ "${unit_quota}" = "*" ] ; then
1556	# IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1557	# IGCM_debug_Print 1 "More than 100% of your quota is used"
1558	# IGCM_debug_Print 1 "Use the ccc_quota command to check"
1559	# IGCM_debug_Print 1 "You must have more than 10% available to run"
1560	# IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1561	# IGCM_debug_Verif_Exit
1562	# fi
1563	#
1564	# temp_avail=${inode_avail%%${inode_avail: -1}*}
1565	# temp_quota=${inode_quota%%${inode_quota: -1}*}
1566	#
1567	# if [ ! ${unit_avail} = ${unit_quota} ] ; then
1568	#
1569	# # Convertion
1570	# if [ ${inode_avail: -1} = "T" ] ; then
1571	# (( temp_avail = temp_avail * 1000000000000 ))
1572	# elif [ ${inode_avail: -1} = "G" ] ; then
1573	# (( temp_avail = temp_avail * 1000000000 ))
1574	# elif [ ${inode_avail: -1} = "M" ] ; then
1575	# (( temp_avail = temp_avail * 1000000 ))
1576	# elif [ ${inode_avail: -1} = "k" ] ; then
1577	# (( temp_avail = temp_avail * 1000 ))
1578	# else
1579	# (( temp_avail = inode_avail ))
1580	# fi
1581	#
1582	# if [ ${inode_quota: -1} = "T" ] ; then
1583	# (( temp_quota = temp_quota * 1000000000000 ))
1584	# elif [ ${inode_quota: -1} = "G" ] ; then
1585	# (( temp_quota = temp_quota * 1000000000 ))
1586	# elif [ ${inode_quota: -1} = "M" ] ; then
1587	# (( temp_quota = temp_quota * 1000000 ))
1588	# elif [ ${inode_quota: -1} = "k" ] ; then
1589	# (( temp_quota = temp_quota * 1000 ))
1590	# else
1591	# (( temp_quota = inode_quota ))
1592	# fi
1593	# fi
1594	# quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
1595	## echo "inode ratio is " $quota_inode
1596	#
1597	# if [ ${quota_inode} -ge ${limit_quota} ] ; then
1598	# IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1599	# IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1600	# IGCM_debug_Print 1 "Use the ccc_quota command to check"
1601	# IGCM_debug_Print 1 "You must have more than 10% available to run"
1602	# IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1603	# IGCM_debug_Verif_Exit
1604	# fi
1605	# fi
1606	IGCM_debug_PopStack "IGCM_sys_check_quota"
1607	}
1608
1609	#D-#==================================================
1610	#D-function IGCM_sys_projectAccounting
1611	#D-* Purpose: store project accounting information in a file
1612	#D-* Examples:
1613	#D-
1614	function IGCM_sys_projectAccounting {
1615	IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1616	if ( $DEBUG_sys ) ; then
1617	echo "IGCM_sys_check_quota"
1618	fi
1619	#need to implemented on Jean Zay
1620	touch $1
1621	# ssh irene191 /usr/bin/ccc_myproject > $1
1622
1623	IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1624	}
1625
1626	#D-#==================================================
1627	#D-function IGCM_sys_getJobSchedulerID
1628	#D-* Purpose: Get the job ID during execution
1629	#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1630	#D-
1631	function IGCM_sys_getJobSchedulerID {
1632	IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1633	if ( $DEBUG_sys ) ; then
1634	echo "IGCM_sys_getJobSchedulerID"
1635	fi
1636
1637	eval ${1}=${SLURM_JOBID}
1638
1639	IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1640	}
1641
1642	#D-#==================================================
1643	#D-function IGCM_sys_GetJobID
1644	#D-* Purpose: Get the job ID from the JobName
1645	#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1646	#D-
1647	function IGCM_sys_GetJobID {
1648	IGCM_debug_PushStack "IGCM_sys_GetJobID"
1649	if ( $DEBUG_sys ) ; then
1650	echo "IGCM_sys_GetJobID"
1651	fi
1652
1653	ID=$( squeue -u $user \| grep Job_${config_UserChoices_JobName} \| gawk ' { print $1}' )
1654
1655
1656	eval ${3}=${ID}
1657	IGCM_debug_PopStack "IGCM_sys_GetJobID"
1658	}
1659
1660	#D-#==================================================
1661	#D-function IGCM_sys_CountJobInQueue
1662	#D-* Purpose: Count number of users job
1663	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1664	#D-
1665	function IGCM_sys_CountJobInQueue {
1666	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1667	if ( $DEBUG_sys ) ; then
1668	echo "IGCM_sys_CountJobInQueue"
1669	fi
1670
1671	# With -f option, the full job name is given in the last column
1672	NbRun=$(squeue -u $user \| \
1673	grep -v JOBID \| gawk 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1674
1675	eval ${2}=${NbRun}
1676
1677	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1678	}
1679
1680	#D-#==================================================
1681	#D-function IGCM_sys_ListJobInQueue
1682	#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1683	#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1684	#D-
1685	function IGCM_sys_ListJobInQueue {
1686	IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1687	if ( $DEBUG_sys ) ; then
1688	echo "IGCM_sys_ListJobInQueue"
1689	fi
1690
1691	# to be implemented on Jean Zay
1692	set -A JobList $( squeue -u $user \| gawk '{print $3}' \| grep -v NAME)
1693	# # With -f option, the full job name is given in the last column
1694	# set -A JobList $( ccc_mstat -f \| gawk -v User=$1 \
1695	# '( $2 == User && \
1696	# $NF != /TS/ && \
1697	# $NF !~ /PACK/ && \
1698	# $NF !~ /REBUILD/ && \
1699	# $NF !~ /pack/ ) \
1700	# { print $NF }' \| sed -e "s/\(.\)\.[0-9]/\1/" )
1701	#
1702	eval set -A ${2} ${JobList[*]}
1703
1704	IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1705	}
1706
1707	#D-#==================================================
1708	#D-function IGCM_sys_atlas
1709	#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1710	#D-* Examples:
1711	#D-
1712	function IGCM_sys_atlas {
1713	IGCM_debug_PushStack "IGCM_sys_atlas" $@
1714	if ( $DEBUG_sys ) ; then
1715	echo "IGCM_sys_atlas :" $@
1716	fi
1717
1718	typeset status
1719	# To be implemented on Jean Zay
1720	#
1721	# \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1722	# status=$?
1723	# if [ ${status} -gt 0 ] ; then
1724	# echo "IGCM_sys_atlas : error code ${status}"
1725	# cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1726	# \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1727	# IGCM_debug_PopStack "IGCM_sys_atlas"
1728	# return 1
1729	# else
1730	# IGCM_debug_PopStack "IGCM_sys_atlas"
1731	# return 0
1732	# fi
1733	#
1734	IGCM_debug_PopStack "IGCM_sys_atlas"
1735	}
1736
1737	#D-#==================================================
1738	#D-function IGCM_sys_rebuild_nemo
1739	#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1740	#D-* Examples:
1741	#D-
1742
1743	function IGCM_sys_rebuild_nemo {
1744	IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1745	if ( $DEBUG_sys ) ; then
1746	echo "IGCM_sys_rebuild_nemo :" $@
1747	fi
1748
1749	/gpfswork/rech/psl/commun/Tools/rebuild_nemo/bin/rebuild_nemo ${1} ${2}
1750
1751	IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1752	}
1753

Note: See TracBrowser for help on using the repository browser.

Download in other formats: