Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh @ 782

Last change on this file since 782 was 782, checked in by labetoulle, 11 years ago
Move REBUILD_DIR initialization into IGCM_config_CommonConfiguration function ; Delete not used variable R_OUT_POST.
Property svn:keywords set to `Revision Author Date`
File size: 84.6 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Titane
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe param files \| \| Chmod \| \|
46	# \| \| Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=titane
71
72	#D-
73	#D-#==================================================
74	#D-Program used in libIGCM
75	#D-#==================================================
76
77	# Submit command
78	typeset SUBMIT=${SUBMIT:=ccc_msub}
79	# rsync with path
80	typeset -r RSYNC=/usr/bin/rsync
81	# RSYNC_opt args to rsync
82	typeset -r RSYNC_opt="-va"
83	# ie storage filesystem
84	typeset -r STOREHOST=${MASTER}
85
86	#====================================================
87	# Source default environment
88	#====================================================
89	. /etc/profile
90
91	#====================================================
92	# Set environment tools (ferret, nco, cdo)
93	#====================================================
94	. /home/cont003/p86ipsl/.atlas_env_titane_ksh
95	if ( [ X${LSB_QUEUE} = Xmono ] \|\| [ X${LSB_QUEUE} = Xmonoext ] ) ; then
96	module switch nco/3.9.4 nco/3.9.4_netcdf4
97	fi
98
99	#====================================================
100	# Specific for ocean additionnal diagnostic
101	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
102	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
103
104	#====================================================
105	# Host specific DIRECTORIES
106	#====================================================
107
108	# ============ CESIUM START ============ #
109
110	#====================================================
111	#- Mirror libIGCM from titane to cesium if needed
112	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
113	#if [ ! ${ROOTSYS} = "home" ] ; then
114	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
115	#else
116	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
117	#fi
118
119	#====================================================
120	#- libIGCM_POST
121	#if ( ${MirrorlibIGCM} ) ; then
122	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
123	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
124	#else
125	# typeset -r libIGCM_POST=${libIGCM}
126	#fi
127
128	# ============ CESIUM END ============ #
129
130	#====================================================
131	#- MirrorlibIGCM for frontend
132	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
133
134	#====================================================
135	#- libIGCM_POST for frontend
136	typeset -r libIGCM_POST=${libIGCM}
137
138	#====================================================
139	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
140	typeset -r R_EXE="${MODIPSL}/bin"
141
142	#====================================================
143	#- SUBMIT_DIR : submission dir
144	typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
145
146	#====================================================
147	#- IN
148	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
149	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
150
151	#====================================================
152	#- ARCHIVE (dedicated to large files)
153	typeset ARCHIVE=${CCCSTOREDIR}
154
155	#====================================================
156	#- STORAGE (dedicated to small/medium files)
157	typeset STORAGE=${CCCWORKDIR}
158
159	#====================================================
160	#- R_OUT
161	typeset R_OUT=${ARCHIVE}/IGCM_OUT
162
163	#====================================================
164	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
165	typeset R_FIG=${STORAGE}/IGCM_OUT
166
167	#====================================================
168	#- R_BUF (ONLY FOR double copy an scratch)
169	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
170
171	#====================================================
172	#- BIG_DIR : BIG_DIR to store files waiting for rebuild
173	typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
174
175	#====================================================
176	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
177	if ( [ ! X${LSB_QUEUE} = Xmono ] && [ ! X${LSB_QUEUE} = Xmonoext ] ) ; then
178	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
179	else
180	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
181	fi
182
183	#====================================================
184	#- HOST_MPIRUN_COMMAND
185	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
186
187	#====================================================
188	#- Max number of arguments passed to nco operator or demigration command
189	UNIX_MAX_LIMIT=360
190
191	#====================================================
192	#- set PackDefault true on titane
193	PackDefault=true
194
195	#====================================================
196	#- Number of core per node (max number of OpenMP task)
197	NUM_COREPERNODE=8
198
199	#====================================================
200	#- Default number of MPI task for IPSL coupled model
201	#- required for backward compatibility
202	#-
203	DEFAULT_NUM_PROC_OCE=5
204	DEFAULT_NUM_PROC_CPL=1
205	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
206	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
207
208	#D-#==================================================
209	#D-function IGCM_sys_ChangeArchive
210	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
211	#D-* Examples:
212	#D-
213	function IGCM_sys_ChangeArchive {
214	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
215
216	R_OUT=${SCRATCHDIR}/IGCM_OUT
217	R_FIG=${SCRATCHDIR}/IGCM_OUT
218
219	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
220	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
221
222	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
223	}
224
225	#D-#==================================================
226	#D-function IGCM_sys_RshMaster
227	#D-* Purpose: Connection to frontend machine.
228	#D-* Examples:
229	#D-
230	function IGCM_sys_RshMaster {
231	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
232	/bin/ksh <<-EOF
233	export libIGCM=${libIGCM}
234	export DEBUG_debug=${DEBUG_debug}
235	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
236	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
237	${@}
238	EOF
239	if [ $? -gt 0 ] ; then
240	echo "IGCM_sys_RshMaster : erreur."
241	IGCM_debug_Exit "IGCM_sys_RshMaster"
242	fi
243	IGCM_debug_PopStack "IGCM_sys_RshMaster"
244	}
245
246	#D-#==================================================
247	#D-function IGCM_sys_RshArchive
248	#D-* Purpose: Archive rsh command
249	#D-* Examples:
250	#D-
251	function IGCM_sys_RshArchive {
252	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
253	/bin/ksh <<-EOF
254	${@}
255	EOF
256	if [ $? -gt 0 ] ; then
257	echo "IGCM_sys_RshArchive : erreur."
258	IGCM_debug_Exit "IGCM_sys_RshArchive"
259	fi
260	IGCM_debug_PopStack "IGCM_sys_RshArchive"
261	}
262
263	#D-#==================================================
264	#D-function IGCM_sys_RshPost
265	#D-* Purpose: Post-process rsh command
266	#D-* Examples:
267	#D-
268	function IGCM_sys_RshPost {
269	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
270	if ( $DEBUG_sys ) ; then
271	echo "IGCM_sys_RshPost :" $@
272	fi
273
274	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
275	#cat tmp_IGCM_sys_RshPost_$$
276	# keep standard input (stdin) for the loop onto temporary file
277	cat >tmp_IGCM_sys_RshPost_$$
278
279	# ============ FRONTEND START ============ #
280
281	/bin/ksh <tmp_IGCM_sys_RshPost_$$
282	if [ $? -gt 0 ] ; then
283	echo "IGCM_sys_RshPost : erreur."
284	IGCM_debug_Exit "IGCM_sys_RshPost"
285	fi
286	# delete temporary file
287	\rm tmp_IGCM_sys_RshPost_$$
288
289	# ============ FRONTEND END ============ #
290
291	# ============ CESIUM START ============ #
292	# typeset NB_ESSAI DELAI status i
293	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
294	# #little hack so that rebuild submission is done on titane not an cesium
295	#
296	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
297	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
298	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
299	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
300	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
301	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
302	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
303	#
304	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
305	# cat tmp_IGCM_sys_RshPost_$$
306	#
307	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
308	# if [ $? -gt 0 ] ; then
309	# echo "IGCM_sys_RshPost : erreur."
310	# IGCM_debug_Exit "IGCM_sys_RshPost"
311	# fi
312	# # delete temporary file
313	# \rm tmp_IGCM_sys_RshPost_$$
314	#
315	# else
316	# # number of tentative
317	# NB_ESSAI=10
318	# # time delay between tentative
319	# DELAI=10
320	# i=0
321	# while [ $i -ne $NB_ESSAI ] ; do
322	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
323	# status=$?
324	# if [ ${status} -ne 0 ];
325	# then
326	# sleep $DELAI
327	# else
328	# break
329	# fi
330	# let i=$i+1
331	# done
332	# # delete temporary file
333	# \rm tmp_IGCM_sys_RshPost_$$
334	#
335	# if [ ${status} -gt 0 ] ; then
336	# echo "IGCM_sys_RshPost : erreur."
337	# IGCM_debug_Exit "IGCM_sys_RshPost"
338	# fi
339	# fi
340
341	# ============ CESIUM END ============ #
342
343	IGCM_debug_PopStack "IGCM_sys_RshPost"
344	}
345
346	#D-#==================================================
347	#D-function IGCM_sys_SendMail
348	#D-* Purpose: Send mail when simulation is over
349	#D-* Examples:
350	#D-
351	function IGCM_sys_SendMail {
352	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
353	if ( $DEBUG_sys ) ; then
354	echo "IGCM_sys_SendMail :" $@
355	fi
356
357	if ( ${ExitFlag} ) ; then
358	status=failed
359	else
360	status=completed
361	fi
362
363	cat << END_MAIL > job_end.mail
364	Dear ${LOGIN},
365
366	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
367	Job started : ${DateBegin}
368	Job ended : ${DateEnd}
369	Output files are available in ${R_SAVE}
370	Files to be rebuild are temporarily available in ${REBUILD_DIR}
371	Pre-packed files are temporarily available in ${R_BUFR}
372	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
373	END_MAIL
374
375	if [ ! -z ${config_UserChoices_MailName} ] ; then
376	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
377	elif [ -f ~/.forward ] ; then
378	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
379	fi
380	sleep 10
381	rm -f job_end.mail
382
383	if [ $? -gt 0 ] ; then
384	echo "IGCM_sys_SendMail : erreur."
385	IGCM_debug_Exit "IGCM_sys_SendMail"
386	fi
387	IGCM_debug_PopStack "IGCM_sys_SendMail"
388	}
389
390	#D-#==================================================
391	#D-function IGCM_sys_Mkdir
392	#D-* Purpose: Master locale mkdir command
393	#D-* Examples:
394	#D-
395	function IGCM_sys_Mkdir {
396	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
397	if ( $DEBUG_sys ) ; then
398	echo "IGCM_sys_Mkdir :" $@
399	fi
400	if [ ! -d ${1} ]; then
401	\mkdir -p $1
402	if [ $? -gt 0 ] ; then
403	echo "IGCM_sys_Mkdir : erreur."
404	IGCM_debug_Exit "IGCM_sys_Mkdir"
405	fi
406	fi
407	# vérification :
408	if [ ! -d ${1} ] ; then
409	echo "IGCM_sys_Mkdir : erreur."
410	IGCM_debug_Exit "IGCM_sys_Mkdir"
411	fi
412	IGCM_debug_PopStack "IGCM_sys_Mkdir"
413	}
414
415	#D-#==================================================
416	#D-function IGCM_sys_MkdirArchive
417	#D-* Purpose: Mkdir on Archive
418	#D-* Examples:
419	#D-
420	function IGCM_sys_MkdirArchive {
421	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
422	if ( $DEBUG_sys ) ; then
423	echo "IGCM_sys_MkdirArchive :" $@
424	fi
425	#- creation de repertoire sur le serveur fichier
426	if [ ! -d ${1} ]; then
427	\mkdir -p $1
428	if [ $? -gt 0 ] ; then
429	echo "IGCM_sys_MkdirArchive : erreur."
430	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
431	fi
432	fi
433	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
434	}
435
436	#D-#==================================================
437	#D-function IGCM_sys_MkdirWork
438	#D-* Purpose: Mkdir on Work
439	#D-* Examples:
440	#D-
441	function IGCM_sys_MkdirWork {
442	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
443	if ( $DEBUG_sys ) ; then
444	echo "IGCM_sys_MkdirWork :" $@
445	fi
446	#- creation de repertoire sur le serveur fichier
447	if [ ! -d ${1} ]; then
448	\mkdir -p $1
449	if [ $? -gt 0 ] ; then
450	echo "IGCM_sys_MkdirWork : erreur."
451	IGCM_debug_Exit "IGCM_sys_MkdirWork"
452	fi
453	fi
454	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
455	}
456
457	#D-#==================================================
458	#D-function IGCM_sys_Cd
459	#D-* Purpose: master cd command
460	#D-* Examples:
461	#D-
462	function IGCM_sys_Cd {
463	IGCM_debug_PushStack "IGCM_sys_Cd" $@
464	if ( $DEBUG_sys ) ; then
465	echo "IGCM_sys_Cd :" $@
466	fi
467	\cd $1
468	if [ $? -gt 0 ] ; then
469	echo "IGCM_sys_Cd : erreur."
470	IGCM_debug_Exit "IGCM_sys_Cd"
471	fi
472	IGCM_debug_PopStack "IGCM_sys_Cd"
473	}
474
475	#D-#==================================================
476	#D-function IGCM_sys_Chmod
477	#D-* Purpose: Chmod
478	#D-* Examples:
479	#D-
480	function IGCM_sys_Chmod {
481	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
482	if ( $DEBUG_sys ) ; then
483	echo "IGCM_sys_Chmod :" $@
484	fi
485	if [ $DRYRUN -le 1 ]; then
486	\chmod $@
487	if [ $? -gt 0 ] ; then
488	echo "IGCM_sys_Chmod : erreur."
489	IGCM_debug_Exit "IGCM_sys_Chmod"
490	fi
491	else
492	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
493	fi
494	IGCM_debug_PopStack "IGCM_sys_Chmod"
495	}
496
497	#D-#==================================================
498	#D-function IGCM_sys_FileSize
499	#D-* Purpose: Filesize
500	#D-* Examples:
501	#D-
502	function IGCM_sys_FileSize {
503	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
504
505	typeset sizeF
506	set +A sizeF -- $( ls -la ${1} )
507	if [ $? -gt 0 ] ; then
508	IGCM_debug_Exit "IGCM_sys_FileSize"
509	fi
510	eval ${2}=${sizeF[4]}
511
512	IGCM_debug_PopStack "IGCM_sys_FileSize"
513	}
514
515	#D-#==================================================
516	#D-function IGCM_sys_TestDir
517	#D-* Purpose: Test Directory that must exists
518	#D-* Examples:
519	#D-
520	function IGCM_sys_TestDir {
521	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
522	if ( $DEBUG_sys ) ; then
523	echo "IGCM_sys_TestDir :" $@
524	fi
525	typeset ExistFlag
526	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
527	IGCM_debug_PopStack "IGCM_sys_TestDir"
528
529	return ${ExistFlag}
530	}
531
532	#D-#==================================================
533	#D-function IGCM_sys_TestDirArchive
534	#D-* Purpose: Test Directory that must exists on Archive
535	#D-* Examples:
536	#D-
537	function IGCM_sys_TestDirArchive {
538	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
539	if ( $DEBUG_sys ) ; then
540	echo "IGCM_sys_TestDirArchive :" $@
541	fi
542	typeset ExistFlag
543	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
544	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
545
546	return ${ExistFlag}
547	}
548
549	#D-#==================================================
550	#D-function IGCM_sys_IsFileArchived
551	#D-* Purpose: Test file that must NOT EXISTS on Archive
552	#D-* Examples:
553	#D-
554	function IGCM_sys_IsFileArchived {
555	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
556	if ( $DEBUG_sys ) ; then
557	echo "IGCM_sys_IsFileArchived :" $@
558	fi
559	typeset IsArchivedFlag
560	IsArchivedFlag=$( [ X$( echo $1 \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
561	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
562
563	return ${IsArchivedFlag}
564	}
565
566	#D-#==================================================
567	#D-function IGCM_sys_TestFileArchive
568	#D-* Purpose: Test file that must NOT EXISTS on Archive
569	#D-* Examples:
570	#D-
571	function IGCM_sys_TestFileArchive {
572	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
573	typeset ExistFlag
574	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
575	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
576
577	return ${ExistFlag}
578	}
579
580	#D-#==================================================
581	#D-function IGCM_sys_TestFileBuffer
582	#D-* Purpose: Test file that must NOT EXISTS on Buffer
583	#D-* Examples:
584	#D-
585	function IGCM_sys_TestFileBuffer {
586	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
587	typeset ExistFlag
588	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
589	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
590
591	return ${ExistFlag}
592	}
593
594	#D-#==================================================
595	#D-function IGCM_sys_CountFileArchive
596	#D-* Purpose: Count files on Archive filesystem
597	#D-* Examples:
598	#D-
599	function IGCM_sys_CountFileArchive {
600	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
601	ls ${@} 2>/dev/null \| wc -l
602	if [ $? -gt 0 ] ; then
603	echo "IGCM_sys_CountFileArchive : erreur."
604	fi
605	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
606	}
607
608	#D-#==================================================
609	#D-function IGCM_sys_CountFileBuffer
610	#D-* Purpose: Count files on Scratch filesystem
611	#D-* Examples:
612	#D-
613	function IGCM_sys_CountFileBuffer {
614	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
615	ls ${@} 2>/dev/null \| wc -l
616	if [ $? -gt 0 ] ; then
617	echo "IGCM_sys_CountFileBuffer : erreur."
618	fi
619	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
620	}
621
622	#D-#==================================================
623	#D-function IGCM_sys_Tree
624	#D-* Purpose: Tree directories with files on ${ARCHIVE}
625	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
626	#D-
627	function IGCM_sys_Tree {
628	IGCM_debug_PushStack "IGCM_sys_Tree" $@
629	if ( $DEBUG_sys ) ; then
630	echo "IGCM_sys_Tree :" $@
631	fi
632
633	\ls -lR ${@}
634
635	IGCM_debug_PopStack "IGCM_sys_Tree"
636	}
637
638	#D-#==================================================
639	#D-function IGCM_sys_Tar
640	#D-* Purpose: master tar command
641	#D-* Examples:
642	#D-
643	function IGCM_sys_Tar {
644	IGCM_debug_PushStack "IGCM_sys_Tar" $@
645	if ( $DEBUG_sys ) ; then
646	echo "IGCM_sys_Tar :" $@
647	fi
648	\tar cf $@
649	if [ $? -gt 0 ] ; then
650	echo "IGCM_sys_Tar : erreur."
651	IGCM_debug_Exit "IGCM_sys_Tar"
652	fi
653	IGCM_debug_PopStack "IGCM_sys_Tar"
654	}
655
656	#D-#==================================================
657	#D-function IGCM_sys_UnTar
658	#D-* Purpose: master un-tar command
659	#D-* Examples:
660	#D-
661	function IGCM_sys_UnTar {
662	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
663	if ( $DEBUG_sys ) ; then
664	echo "IGCM_sys_UnTar :" $@
665	fi
666	\tar xvf $1
667	if [ $? -gt 0 ] ; then
668	echo "IGCM_sys_UnTar : erreur."
669	IGCM_debug_Exit "IGCM_sys_UnTar"
670	fi
671	IGCM_debug_PopStack "IGCM_sys_UnTar"
672	}
673
674	#D-#==================================================
675	#D-function IGCM_sys_Qsub
676	#D-* Purpose: Qsub new job
677	#D-* Examples:
678	#D-
679	function IGCM_sys_Qsub {
680	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
681	if ( $DEBUG_sys ) ; then
682	echo "IGCM_sys_Qsub :" $@
683	fi
684	typeset options
685	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
686	options=" -o ${Script_Output} -e ${Script_Output}"
687	else
688	options=" -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
689	fi
690
691	/usr/local/bin/ccc_msub ${options} < $1
692	if [ $? -gt 0 ] ; then
693	echo "IGCM_sys_Qsub : erreur ${options} $1"
694	IGCM_debug_Exit "IGCM_sys_Qsub"
695	fi
696	IGCM_debug_PopStack "IGCM_sys_Qsub"
697	}
698
699	#D-#==================================================
700	#D-function IGCM_sys_QsubPost
701	#D-* Purpose: Qsub new job on scalaire
702	#D-* Examples:
703	#D-
704	function IGCM_sys_QsubPost {
705	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
706	if ( $DEBUG_sys ) ; then
707	echo "IGCM_sys_QsubPost :" $@
708	fi
709
710	# Need to export listVarEnv for cesium
711	# NO Need to export listVarEnv for titane
712	# ============ FRONTEND START ============ #
713
714	/usr/local/bin/ccc_msub -p ${BRIDGE_MSUB_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
715
716	# ============ FRONTEND END ============ #
717
718	# ============ CESIUM START ============ #
719	# typeset NB_ESSAI DELAI status i
720	# # number of tentative
721	# NB_ESSAI=10
722	# # time delay between tentative
723	# DELAI=10
724	# (( i = 0 ))
725	# while [ $i -lt $NB_ESSAI ] ; do
726	# /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
727	# status=$?
728	# if [ ${status} -gt 0 ]; then
729	# sleep $DELAI
730	# else
731	# break
732	# fi
733	# (( i = i + 1 ))
734	# done
735	# ============ CESIUM END ============ #
736
737	if [ $? -gt 0 ] ; then
738	echo "IGCM_sys_QsubPost : erreur " $@
739	IGCM_debug_Exit "IGCM_sys_QsubPost"
740	fi
741	IGCM_debug_PopStack "IGCM_sys_QsubPost"
742	}
743
744	#D-*************************
745	#D- File transfer functions
746	#D-*************************
747	#D-
748
749	#D-#==================================================
750	#D-function IGCM_sys_Rsync_out
751	#D-* Purpose: treat return val of rsync
752	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
753	#D- Error values and explanations can depend on your system version.
754	function IGCM_sys_Rsync_out {
755	status=$1
756	if [ ! $status ] ; then
757	echo "rsync error !"
758	fi
759
760	if [ $MYLANG = "fr" ]; then
761	case $status in
762	0) return ;;
763	1) echo "Erreur de rsync ; RERR_SYNTAX : "
764	echo "Erreur de syntaxe ou d'utilisation."
765	return;;
766	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
767	echo "Incompatibilité de protocole."
768	return;;
769	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
770	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
771	echo "répertoires"
772	return;;
773	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
774	echo "Action demandée non supportée : une tentative de manipulation de"
775	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
776	echo "été faite ; ou une option qui est supportée par le client mais"
777	echo "pas par le serveur a été spécifiée."
778	return;;
779	10) echo "Erreur de rsync ; RERR_SOCKETIO"
780	echo "Erreur dans le socket d'entrée sortie"
781	return;;
782	11) echo "Erreur de rsync ; RERR_FILEIO"
783	echo "Erreur d'entrée sortie fichier"
784	return;;
785	12) echo "Erreur de rsync ; RERR_STREAMIO"
786	echo "Erreur dans flux de donnée du protocole rsync"
787	return;;
788	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
789	echo "Erreur avec les diagnostics du programme"
790	return;;
791	14) echo "Erreur de rsync ; RERR_IPC"
792	echo "Erreur dans le code IPC"
793	return;;
794	20) echo "Erreur de rsync ; RERR_SIGNAL"
795	echo "SIGUSR1 ou SIGINT reçu"
796	return;;
797	21) echo "Erreur de rsync ; RERR_WAITCHILD"
798	echo "Une erreur retournée par waitpid()"
799	return;;
800	22) echo "Erreur de rsync ; RERR_MALLOC"
801	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
802	return;;
803	23) echo ""
804	echo "Erreur fichier inexistant"
805	return;;
806	30) echo "Erreur de rsync ; RERR_TIMEOUT"
807	echo "Temps d'attente écoulé dans l'envoi/réception de données"
808	return;;
809	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
810	return;;
811	esac
812	elif [ $MYLANG = "en" ] ; then
813	case $status in
814	0) return;;
815	1) echo "rsync error : Syntax or usage error "
816	return;;
817	2) echo "rsync error : Protocol incompatibility "
818	return;;
819	3) echo "rsync error : Errors selecting input/output files, dirs"
820	return;;
821	4) echo "rsync error : Requested action not supported: an attempt"
822	echo "was made to manipulate 64-bit files on a platform that cannot support"
823	echo "them; or an option was specified that is supported by the client and"
824	echo "not by the server."
825	return;;
826	5) echo "rsync error : Error starting client-server protocol"
827	return;;
828	10) echo "rsync error : Error in socket I/O "
829	return;;
830	11) echo "rsync error : Error in file I/O "
831	return;;
832	12) echo "rsync error : Error in rsync protocol data stream "
833	return;;
834	13) echo "rsync error : Errors with program diagnostics "
835	return;;
836	14) echo "rsync error : Error in IPC code "
837	return;;
838	20) echo "rsync error : Received SIGUSR1 or SIGINT "
839	return;;
840	21) echo "rsync error : Some error returned by waitpid() "
841	return;;
842	22) echo "rsync error : Error allocating core memory buffers "
843	return;;
844	23) echo "rsync error : Partial transfer due to error"
845	return;;
846	24) echo "rsync error : Partial transfer due to vanished source files"
847	return;;
848	30) echo "rsync error : Timeout in data send/receive "
849	return;;
850	*) echo "rsync error : return code of rsync unknown :" $status
851	return;;
852	esac
853	else
854	echo "unknown language $MYLANG."
855	return
856	fi
857	}
858
859	#D-#==================================================
860	#D-function IGCM_sys_Miror_libIGCM
861	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
862	#D-* Examples:
863	#D-
864	function IGCM_sys_Mirror_libIGCM {
865	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
866	if ( $DEBUG_sys ) ; then
867	echo "IGCM_sys_Mirror_libIGCM"
868	fi
869
870	typeset status DEST
871
872	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
873
874	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
875	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
876	status=$?
877
878	if [ ${status} -gt 0 ] ; then
879	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
880	cat out_rsync
881	fi
882	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
883	}
884
885	#D-#==================================================
886	#D-function IGCM_sys_Cp
887	#D-* Purpose: generic cp
888	#D-* Examples:
889	#D-
890	function IGCM_sys_Cp {
891	IGCM_debug_PushStack "IGCM_sys_Cp" $@
892	if ( $DEBUG_sys ) ; then
893	echo "IGCM_sys_Cp :" $@
894	fi
895
896	typeset status
897
898	echo cp $@ > out_rsync 2>&1
899	\cp $@ >> out_rsync 2>&1
900	status=$?
901
902	if [ ${status} -gt 0 ] ; then
903	echo "IGCM_sys_Cp : error code ${status}"
904	cat out_rsync
905	IGCM_debug_Exit "IGCM_sys_Cp"
906	else
907	\rm out_rsync
908	fi
909	IGCM_debug_PopStack "IGCM_sys_Cp"
910	}
911
912	#D-#==================================================
913	#D-function IGCM_sys_Rm
914	#D-* Purpose: generic rm
915	#D-* Examples:
916	#D-
917	function IGCM_sys_Rm {
918	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
919	if ( $DEBUG_sys ) ; then
920	echo "IGCM_sys_Rm :" $@
921	fi
922
923	typeset status
924
925	echo rm $@ > out_rsync 2>&1
926	\rm $@ >> out_rsync 2>&1
927	status=$?
928
929	if [ ${status} -gt 0 ] ; then
930	echo "IGCM_sys_Rm : error code ${status}"
931	cat out_rsync
932	IGCM_debug_Exit "IGCM_sys_Rm"
933	else
934	\rm out_rsync
935	fi
936	IGCM_debug_PopStack "IGCM_sys_Rm"
937	}
938
939	#D-#==================================================
940	#D-function IGCM_sys_RmRunDir
941	#D-* Purpose: rm tmpdir (dummy function most of the time batch
942	#D- scheduler will do the job)
943	#D-* Examples:
944	#D-
945	function IGCM_sys_RmRunDir {
946	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
947	if ( $DEBUG_sys ) ; then
948	echo "IGCM_sys_RmRunDir :" $@
949	fi
950
951	typeset status
952
953	echo rm $@ > out_rsync 2>&1
954	\rm $@ >> out_rsync 2>&1
955	status=$?
956
957	if [ ${status} -gt 0 ] ; then
958	echo "IGCM_sys_RmRunDir : error code ${status}"
959	cat out_rsync
960	IGCM_debug_Exit "IGCM_sys_RmRunDir"
961	else
962	\rm out_rsync
963	fi
964	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
965	}
966
967	#D-#==================================================
968	#D-function IGCM_sys_Mv
969	#D-* Purpose: generic move
970	#D-* Examples:
971	#D-
972	function IGCM_sys_Mv {
973	IGCM_debug_PushStack "IGCM_sys_Mv" $@
974	if ( $DEBUG_sys ) ; then
975	echo "IGCM_sys_Mv :" $@
976	fi
977
978	if [ $DRYRUN = 0 ]; then
979
980	typeset status
981
982	echo mv $@ > out_rsync 2>&1
983	\mv $@ >> out_rsync 2>&1
984	status=$?
985
986	if [ ${status} -gt 0 ] ; then
987	echo "IGCM_sys_Mv : error code ${status}"
988	cat out_rsync
989	IGCM_debug_Exit "IGCM_sys_Mv"
990	else
991	\rm out_rsync
992	fi
993	else
994	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
995	fi
996
997	IGCM_debug_PopStack "IGCM_sys_Mv"
998	}
999
1000	#D-#==================================================
1001	#D-function IGCM_sys_Put_Dir
1002	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
1003	#D-* Examples:
1004	#D-
1005	function IGCM_sys_Put_Dir {
1006	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
1007	if ( $DEBUG_sys ) ; then
1008	echo "IGCM_sys_Put_Dir :" $@
1009	fi
1010	if [ $DRYRUN = 0 ]; then
1011	if [ ! -d ${1} ] ; then
1012	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
1013	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1014	return
1015	fi
1016
1017	typeset status
1018
1019	# Only if we use rsync
1020	#IGCM_sys_TestDirArchive $( dirname $2 )
1021	#
1022	#USUAL WAY
1023	\cp -r $1 $2 > out_rsync 2>&1
1024	status=$?
1025
1026	if [ ${status} -gt 0 ] ; then
1027	echo "IGCM_sys_Put_Dir : error code ${status}"
1028	cat out_rsync
1029	IGCM_debug_Exit "IGCM_sys_Put_Dir"
1030	else
1031	\rm out_rsync
1032	fi
1033	else
1034	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1035	fi
1036	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1037	}
1038
1039	#D-#==================================================
1040	#D-function IGCM_sys_Get_Dir
1041	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1042	#D-* Examples:
1043	#D-
1044	function IGCM_sys_Get_Dir {
1045	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1046	if ( $DEBUG_sys ) ; then
1047	echo "IGCM_sys_Get_Dir :" $@
1048	fi
1049	if [ $DRYRUN = 0 ]; then
1050	# if [ ! -d ${1} ] ; then
1051	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1052	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1053	# return
1054	# fi
1055
1056	typeset NB_ESSAI DELAI status i
1057	# number of tentative
1058	NB_ESSAI=3
1059	# time delay between tentative
1060	DELAI=2
1061
1062	# Only if we use rsync
1063	#IGCM_sys_TestDirArchive $( dirname $2 )
1064	#
1065	# USUAL WAY
1066	# add dmfind/dmget (to demigrate all offline files) :
1067	#dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget
1068	ccc_hsm get -r $1
1069
1070	i=0
1071	while [ $i -lt $NB_ESSAI ] ; do
1072	\cp -ur $1 $2 >> out_rsync 2>&1
1073	status=$?
1074	if [ ${status} -gt 0 ]; then
1075	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
1076	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1077	sleep $DELAI
1078	else
1079	break
1080	fi
1081	(( i = i + 1 ))
1082	done
1083
1084	if [ ${status} -gt 0 ] ; then
1085	echo "IGCM_sys_Get_Dir : error."
1086	cat out_rsync
1087	IGCM_debug_Exit "IGCM_sys_Get_Dir"
1088	else
1089	\rm out_rsync
1090	fi
1091	else
1092	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1093	fi
1094	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1095	}
1096
1097	#D-#==================================================
1098	#D-function IGCM_sys_Get_Master
1099	#D-* Purpose: Copy a complete directory from MASTER filesystem
1100	#D-* Examples:
1101	#D-
1102	function IGCM_sys_Get_Master {
1103	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1104	if ( $DEBUG_sys ) ; then
1105	echo "IGCM_sys_Get_Master :" $@
1106	fi
1107	if [ $DRYRUN = 0 ]; then
1108	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1109	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1110	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1111	return
1112	fi
1113
1114	typeset NB_ESSAI DELAI status i
1115	# number of tentative
1116	NB_ESSAI=3
1117	# time delay between tentative
1118	DELAI=2
1119
1120	i=0
1121	while [ $i -lt $NB_ESSAI ] ; do
1122	\cp -urL $1 $2 > out_rsync 2>&1
1123	status=$?
1124	if [ ${status} -gt 0 ]; then
1125	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
1126	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1127	sleep $DELAI
1128	else
1129	break
1130	fi
1131	(( i = i + 1 ))
1132	done
1133
1134	if [ ${status} -gt 0 ] ; then
1135	echo "IGCM_sys_Get_Master : error."
1136	cat out_rsync
1137	IGCM_debug_Exit "IGCM_sys_Get_Master"
1138	else
1139	\rm out_rsync
1140	fi
1141	else
1142	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1143	fi
1144	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1145	}
1146
1147	#====================================================
1148	#- Call IGCM_sys_Mirror_libIGCM now !
1149	if ( $MirrorlibIGCM ) ; then
1150	IGCM_sys_Mirror_libIGCM
1151	fi
1152
1153	#D-#==================================================
1154	#D-function IGCM_sys_Put_Rest
1155	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1156	#D- File and target directory must exist.
1157	#D-* Examples:
1158	#D-
1159	function IGCM_sys_Put_Rest {
1160	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1161	if ( $DEBUG_sys ) ; then
1162	echo "IGCM_sys_Put_Rest :" $@
1163	fi
1164	if [ $DRYRUN = 0 ]; then
1165	if [ ! -f ${1} ] ; then
1166	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1167	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1168	fi
1169
1170	typeset status
1171	#
1172	if [ X${JobType} = XRUN ] ; then
1173	IGCM_sys_Chmod 444 ${1}
1174	fi
1175
1176	#
1177	# USUAL WAY
1178	\cp $1 $2 > out_rsync 2>&1
1179	status=$?
1180
1181	# #RSYNC WITH NETWORK SSH CALL
1182	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1183	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1184
1185	# #RSYNC WITH NFS USE
1186	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1187	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1188
1189	# status=$?
1190	# IGCM_sys_Rsync_out $status
1191
1192	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1193	# (( status=status+$? ))
1194
1195	if [ ${status} -gt 0 ] ; then
1196	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1197	cat out_rsync
1198	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1199	else
1200	\rm out_rsync
1201	fi
1202	else
1203	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1204	fi
1205	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1206	}
1207
1208	#D-#==================================================
1209	#D-function IGCM_sys_PutBuffer_Rest
1210	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1211	#D- File and target directory must exist.
1212	#D-* Examples:
1213	#D-
1214	function IGCM_sys_PutBuffer_Rest {
1215	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1216	if ( $DEBUG_sys ) ; then
1217	echo "IGCM_sys_PutBuffer_Rest :" $@
1218	fi
1219	if [ $DRYRUN = 0 ]; then
1220	if [ ! -f ${1} ] ; then
1221	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1222	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1223	fi
1224
1225	typeset status
1226	#
1227	if [ X${JobType} = XRUN ] ; then
1228	IGCM_sys_Chmod 444 ${1}
1229	fi
1230
1231	#
1232	# USUAL WAY
1233	\cp $1 $2 > out_rsync 2>&1
1234	status=$?
1235
1236	if [ ${status} -gt 0 ] ; then
1237	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1238	cat out_rsync
1239	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1240	else
1241	\rm out_rsync
1242	fi
1243	else
1244	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1245	fi
1246	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1247	}
1248
1249	#D-#==================================================
1250	#D-function IGCM_sys_Put_Out
1251	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1252	#D-* Examples:
1253	#D-
1254	function IGCM_sys_Put_Out {
1255	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1256	if ( $DEBUG_sys ) ; then
1257	echo "IGCM_sys_Put_Out :" $@
1258	fi
1259
1260	typeset NB_ESSAI DELAI status i exist skip
1261
1262	# number of tentative
1263	NB_ESSAI=3
1264	# time delay between tentative
1265	DELAI=2
1266
1267	if [ $DRYRUN = 0 ]; then
1268	if [ ! -f ${1} ] ; then
1269	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1270	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1271	return 1
1272	fi
1273	#
1274	IGCM_sys_MkdirArchive $( dirname $2 )
1275	#
1276
1277	#=====================================================
1278	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1279	#=====================================================
1280
1281	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1282	#if [ $? -eq 0 ] ; then
1283	# typeset WORKPATH FILEPATH
1284	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
1285	# IGCM_sys_MkdirWork ${WORKPATH}
1286	# FILEPATH=${WORKPATH}/$( basename $2 )
1287	# #
1288	# IGCM_sys_Cp ${1} ${FILEPATH}
1289	#fi
1290
1291	if [ X${JobType} = XRUN ] ; then
1292	if [ X${3} = X ] ; then
1293	IGCM_sys_Chmod 444 ${1}
1294	fi
1295	fi
1296	#
1297
1298	exist=false
1299	skip=false
1300	if [ -f $2 ] ; then
1301	IGCM_debug_Print 1 "$2 already exist"
1302	#dmget $2
1303	ccc_hsm get $2
1304	exist=true
1305	if [ "X$( diff $1 $2 )" = X ] ; then
1306	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1307	skip=true
1308	else
1309	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1310	skip=false
1311	fi
1312	fi
1313	#
1314	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1315	IGCM_sys_Chmod u+w $2
1316	fi
1317
1318	if [ X${skip} = Xfalse ] ; then
1319	i=0
1320	while [ $i -lt $NB_ESSAI ] ; do
1321	# USUAL WAY
1322	\cp $1 $2 > out_rsync 2>&1
1323	status=$?
1324	if [ ${status} -gt 0 ]; then
1325	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1326	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1327	if [ -f $2 ] ; then
1328	if [ "X$( diff $1 $2 )" = X ] ; then
1329	IGCM_debug_Print 2 "$1 and $2 are the same file"
1330	IGCM_debug_Print 2 "Cp gave an error but the file has been copied properly!"
1331	status=0
1332	\rm out_rsync
1333	break
1334	else
1335	IGCM_debug_Print 2 "$1 and $2 are not the same file, we continue to try"
1336	\rm $2
1337	fi
1338	fi
1339	sleep $DELAI
1340	else
1341	break
1342	fi
1343	(( i = i + 1 ))
1344	done
1345	fi
1346
1347	# #RSYNC WITH NETWORK SSH CALL
1348	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1349	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1350
1351	# #RSYNC WITH NFS USE
1352	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1353	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1354
1355	# status=$?
1356	# IGCM_sys_Rsync_out $status
1357
1358	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1359	# (( status=status+$? ))
1360
1361	if [ ${status} -gt 0 ] ; then
1362	echo "IGCM_sys_Put_Out : error."
1363	cat out_rsync
1364	IGCM_debug_Exit "IGCM_sys_Put_Out"
1365	else
1366	\rm out_rsync
1367	fi
1368	else
1369	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1370	fi
1371	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1372	return 0
1373	}
1374
1375	#D-#==================================================
1376	#D-function IGCM_sys_PutBuffer_Out
1377	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1378	#D-* Examples:
1379	#D-
1380	function IGCM_sys_PutBuffer_Out {
1381	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1382	if ( $DEBUG_sys ) ; then
1383	echo "IGCM_sys_PutBuffer_Out :" $@
1384	fi
1385
1386	typeset NB_ESSAI DELAI status i exist skip
1387
1388	# number of tentative
1389	NB_ESSAI=3
1390	# time delay between tentative
1391	DELAI=2
1392
1393	if [ $DRYRUN = 0 ]; then
1394	if [ ! -f ${1} ] ; then
1395	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1396	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1397	return 1
1398	fi
1399	#
1400	IGCM_sys_Mkdir $( dirname $2 )
1401	#
1402
1403	if [ X${JobType} = XRUN ] ; then
1404	if [ X${3} = X ] ; then
1405	IGCM_sys_Chmod 444 ${1}
1406	fi
1407	fi
1408
1409	exist=false
1410	skip=false
1411	if [ -f $2 ] ; then
1412	IGCM_debug_Print 1 "$2 already exist"
1413	exist=true
1414	if [ "X$( diff $1 $2 )" = X ] ; then
1415	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1416	skip=true
1417	else
1418	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1419	skip=false
1420	fi
1421	fi
1422	#
1423	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1424	IGCM_sys_Chmod u+w $2
1425	fi
1426
1427	if [ X${skip} = Xfalse ] ; then
1428	i=0
1429	while [ $i -lt $NB_ESSAI ] ; do
1430	# USUAL WAY
1431	\cp $1 $2 > out_rsync 2>&1
1432	status=$?
1433	if [ ${status} -gt 0 ]; then
1434	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1435	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1436	sleep $DELAI
1437	else
1438	break
1439	fi
1440	(( i = i + 1 ))
1441	done
1442	fi
1443
1444	if [ ${status} -gt 0 ] ; then
1445	echo "IGCM_sys_PutBuffer_Out : error."
1446	cat out_rsync
1447	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1448	else
1449	\rm out_rsync
1450	fi
1451	else
1452	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1453	fi
1454	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1455	return 0
1456	}
1457
1458	#D-#==================================================
1459	#D-function IGCM_sys_Get
1460	#D-* Purpose: Get a file from ${ARCHIVE}
1461	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1462	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1463	function IGCM_sys_Get {
1464	IGCM_debug_PushStack "IGCM_sys_Get" $@
1465
1466	typeset DEST dm_liste ifile target file_work
1467	typeset NB_ESSAI DELAI status i
1468
1469	if ( $DEBUG_sys ) ; then
1470	echo "IGCM_sys_Get :" $@
1471	fi
1472
1473	# number of tentative
1474	NB_ESSAI=3
1475	# time delay between tentative
1476	DELAI=2
1477
1478	if [ $DRYRUN -le 2 ]; then
1479	if [ X${1} = X'/l' ] ; then
1480	# test if the first file is present in the old computation :
1481	eval set +A dm_liste \${${2}}
1482	else
1483	eval set +A dm_liste ${1}
1484	fi
1485	eval DEST=\${${#}}
1486
1487	#=====================================================
1488	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1489	#=====================================================
1490
1491	# Is it an R_OUT file (not R_IN) ?
1492	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1493	#if [ $? -eq 0 ] ; then
1494	# # Yes ? then we try to get it in SCRATCHDIR
1495	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
1496	# if [ -f ${file_work[0]} ] ; then
1497	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1498	# IGCM_debug_PopStack "IGCM_sys_Get"
1499	# return
1500	# fi
1501	#fi
1502
1503	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1504	status=$?
1505	if [ ${status} -gt 0 ] ; then
1506	echo "WARNING IGCM_sys_Get : error code ${status}"
1507	cat out_rsync
1508	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1509	fi
1510
1511	#if [ ${status} -gt 0 ] ; then
1512	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1513	# cat out_rsync
1514	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1515	# sleep 30
1516	# echo "We try another time"
1517	## dmget ${dm_liste[*]} > out_rsync 2>&1
1518	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1519	# status=$?
1520	# if [ ${status} -gt 0 ] ; then
1521	# echo "ERROR IGCM_sys_Get : again demigration error :"
1522	# cat out_rsync
1523	# IGCM_debug_Exit "IGCM_sys_Get"
1524	# fi
1525	# else
1526	# echo "ERROR IGCM_sys_Get : demigration error :"
1527	# cat out_rsync
1528	# IGCM_debug_Exit "IGCM_sys_Get"
1529	# fi
1530	#fi
1531
1532	# #RSYNC WITH NETWORK SSH CALL
1533	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1534	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1535
1536	# #RSYNC WITH NFS USE
1537	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1538	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1539
1540	# status=$?
1541	# IGCM_sys_Rsync_out $status
1542
1543	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1544	# (( status=status+$? ))
1545
1546	#USUAL WAY
1547	if [ X${1} = X'/l' ] ; then
1548	for target in ${dm_liste[*]} ; do
1549	local_file=$( basename ${target} )
1550	# test if the target file is present before the loop
1551	IGCM_sys_TestFileArchive ${target}
1552	status=$?
1553	if [ ${status} -gt 0 ] ; then
1554	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
1555	IGCM_debug_Exit "IGCM_sys_Get"
1556	else
1557	i=0
1558	while [ $i -lt $NB_ESSAI ] ; do
1559	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1560	status=$?
1561	if [ ${status} -gt 0 ]; then
1562	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1563	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1564	sleep $DELAI
1565	else
1566	break
1567	fi
1568	(( i = i + 1 ))
1569	done
1570	if [ ${status} -gt 0 ] ; then
1571	echo "IGCM_sys_Get : error"
1572	cat out_rsync
1573	\rm out_rsync
1574	IGCM_debug_Exit "IGCM_sys_Get"
1575	else
1576	\rm out_rsync
1577	fi
1578	fi
1579	done
1580	else
1581	i=0
1582	while [ $i -lt $NB_ESSAI ] ; do
1583	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1584	status=$?
1585	if [ ${status} -gt 0 ]; then
1586	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1587	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1588	sleep $DELAI
1589	else
1590	break
1591	fi
1592	(( i = i + 1 ))
1593	done
1594	if [ ${status} -gt 0 ] ; then
1595	echo "IGCM_sys_Get : error"
1596	cat out_rsync
1597	\rm out_rsync
1598	IGCM_debug_Exit "IGCM_sys_Get"
1599	else
1600	\rm out_rsync
1601	fi
1602	fi
1603	else
1604	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1605	fi
1606	IGCM_debug_PopStack "IGCM_sys_Get"
1607	}
1608
1609	#D-#==================================================
1610	#D-function IGCM_sys_GetBuffer
1611	#D-* Purpose: Get a file from ${SCRATCHDIR}
1612	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1613	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1614	function IGCM_sys_GetBuffer {
1615	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1616
1617	typeset DEST buf_liste ifile target file_work
1618	typeset NB_ESSAI DELAI status i
1619
1620	if ( $DEBUG_sys ) ; then
1621	echo "IGCM_sys_GetBuffer :" $@
1622	fi
1623
1624	# number of tentative
1625	NB_ESSAI=3
1626	# time delay between tentative
1627	DELAI=2
1628
1629	if [ $DRYRUN -le 2 ]; then
1630	if [ X${1} = X'/l' ] ; then
1631	# test if the first file is present in the old computation :
1632	eval set +A buf_liste \${${2}}
1633	else
1634	eval set +A buf_liste ${1}
1635	fi
1636	eval DEST=\${${#}}
1637
1638	#USUAL WAY
1639	if [ X${1} = X'/l' ] ; then
1640	for target in ${buf_liste[*]} ; do
1641	local_file=$( basename ${target} )
1642	i=0
1643	while [ $i -lt $NB_ESSAI ] ; do
1644	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1645	status=$?
1646	if [ ${status} -gt 0 ]; then
1647	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1648	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1649	sleep $DELAI
1650	else
1651	break
1652	fi
1653	(( i = i + 1 ))
1654	done
1655	if [ ${status} -gt 0 ] ; then
1656	echo "IGCM_sys_Get : error"
1657	cat out_rsync
1658	\rm out_rsync
1659	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1660	else
1661	\rm out_rsync
1662	fi
1663	done
1664	else
1665	i=0
1666	while [ $i -lt $NB_ESSAI ] ; do
1667	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1668	status=$?
1669	if [ ${status} -gt 0 ]; then
1670	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1671	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1672	sleep $DELAI
1673	else
1674	break
1675	fi
1676	(( i = i + 1 ))
1677	done
1678	if [ ${status} -gt 0 ] ; then
1679	echo "IGCM_sys_Get : error"
1680	cat out_rsync
1681	\rm out_rsync
1682	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1683	else
1684	\rm out_rsync
1685	fi
1686	fi
1687	else
1688	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1689	fi
1690	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1691	}
1692
1693	#D-#==================================================
1694	#D-function IGCM_sys_GetDate_FichWork
1695	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1696	#D-* Examples:
1697	#D-
1698	function IGCM_sys_GetDate_FichWork {
1699	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1700	if ( $DEBUG_sys ) ; then
1701	echo "IGCM_sys_GetDate_FichWork :" $@
1702	fi
1703	typeset dateF
1704	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1705	eval ${2}=${dateF[5]}
1706
1707	# donne la date filesys d'un fichier sur la machine work
1708	IGCM_debug_PopStack "IGCM_sys_FichWork"
1709	}
1710
1711	#D-#==================================================
1712	#D-function IGCM_sys_GetDate_FichArchive
1713	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1714	#D-* Examples:
1715	#D-
1716	function IGCM_sys_GetDate_FichArchive {
1717	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1718	if ( $DEBUG_sys ) ; then
1719	echo "IGCM_sys_GetDate_FichArchive :" $@
1720	fi
1721	typeset dateF
1722	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1723	eval ${2}=${dateF[5]}
1724
1725	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1726	}
1727
1728	#D-#==================================================
1729	#D-function IGCM_sys_Dods_Rm
1730	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1731	#D-* Examples:
1732	#D-
1733	function IGCM_sys_Dods_Rm {
1734	if ( $DEBUG_sys ) ; then
1735	echo "IGCM_sys_Dods_Rm :" $@
1736	fi
1737	typeset status
1738	if [ $DRYRUN = 0 ]; then
1739
1740	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1741	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1742	# echo "Nothing has been done."
1743	# return
1744	# fi
1745
1746	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1747	status=$?
1748
1749	# if [ ${status} -gt 0 ] ; then
1750	# echo "IGCM_sys_Dods_Rm : error."
1751	# cat out_dods_rm
1752	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1753	# else
1754	# rm out_dods_rm
1755	# fi
1756
1757	else
1758	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1759	fi
1760	return $status
1761	}
1762
1763	#D-#==================================================
1764	#D-function IGCM_sys_Dods_Cp
1765	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1766	#D-* Examples:
1767	#D-
1768	function IGCM_sys_Dods_Cp {
1769	if ( $DEBUG_sys ) ; then
1770	echo "IGCM_sys_Dods_Cp :" $@
1771	fi
1772	typeset status
1773	if [ $DRYRUN = 0 ]; then
1774
1775	# if [ ! -d ${R_SAVE}/${1} ] ; then
1776	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1777	# echo "Nothing has been done."
1778	# return
1779	# fi
1780
1781	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1782	status=$?
1783
1784	# if [ ${status} -gt 0 ] ; then
1785	# echo "IGCM_sys_Dods_Cp : error."
1786	# cat out_dods_cp
1787	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1788	# else
1789	# rm out_dods_cp
1790	# fi
1791
1792	else
1793	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1794	fi
1795	return $status
1796	}
1797
1798	#D-#==================================================
1799	#D-function IGCM_sys_Put_Dods
1800	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1801	#D-* Examples:
1802	#D-
1803	function IGCM_sys_Put_Dods {
1804	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1805	if ( $DEBUG_sys ) ; then
1806	echo "IGCM_sys_Put_Dods :" $@
1807	fi
1808	#set -vx
1809	typeset status
1810	if [ $DRYRUN = 0 ]; then
1811	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1812	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1813	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1814	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1815	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1816	return
1817	fi
1818
1819	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1820	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1821	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1822	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1823	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1824	return
1825	fi
1826	#
1827	if [ -d ${R_SAVE}/${1} ] ; then
1828	cd ${R_SAVE}
1829	elif [ -d ${R_FIGR}/${1} ] ; then
1830	cd ${R_FIGR}
1831	fi
1832
1833	IGCM_sys_Dods_Rm ${1}
1834	IGCM_sys_Dods_Cp ${1}
1835	status=0
1836
1837	if [ ${status} -gt 0 ] ; then
1838	echo "IGCM_sys_Put_Dods : error."
1839	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1840	fi
1841	else
1842	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1843	fi
1844	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1845	}
1846
1847	##############################################################
1848	# REBUILD OPERATOR
1849
1850	function IGCM_sys_rebuild {
1851	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1852	if ( $DEBUG_sys ) ; then
1853	echo "IGCM_sys_rebuild :" $@
1854	fi
1855
1856	typeset NB_ESSAI DELAI status i firstArg
1857	# number of tentative
1858	NB_ESSAI=3
1859	# time delay between tentative
1860	DELAI=2
1861
1862	i=0
1863	while [ $i -lt $NB_ESSAI ] ; do
1864	/home/cont003/p86ipsl/X64/bin/rebuild -f -o $@ > out_rsync 2>&1
1865	status=$?
1866	if [ ${status} -gt 0 ] ; then
1867	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1868	cat out_rsync
1869	\rm out_rsync
1870	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1871	firstArg=${1}
1872	\rm ${firstArg}
1873	sleep $DELAI
1874	else
1875	\rm out_rsync
1876	break
1877	fi
1878	(( i = i + 1 ))
1879	done
1880
1881	if [ ${status} -gt 0 ] ; then
1882	echo "IGCM_sys_rebuild : rebuild error"
1883	IGCM_debug_Exit "rebuild"
1884	fi
1885
1886	IGCM_debug_PopStack "IGCM_sys_rebuild"
1887	}
1888
1889	function IGCM_sys_rebuild_station {
1890	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1891	typeset i list_opt file_in file_out prefix_invert list_invert
1892	if ( $DEBUG_sys ) ; then
1893	echo "IGCM_sys_rebuild_station :" $@
1894	fi
1895	list_opt=$@
1896
1897	# Invert Axis : t,x -> x,t
1898	# t,pres,x -> x,t,pres
1899	# So that we can concatenate along x
1900	i=0
1901	for file_in in ${list_opt} ; do
1902	(( i = i + 1))
1903	[ ${i} = 1 ] && file_out=${file_in} && continue
1904	prefix_invert=$( basename ${file_in} .nc )
1905	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1906	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1907	done
1908
1909	# Concatenate
1910	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1911
1912	# Re-ivert file
1913	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1914
1915	# Station re-ordering is too expansive to be run within libICGM
1916	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1917	# This re-ordering must be done "in memory" by the cmorization process
1918	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1919	# BEGIN reordering
1920
1921	# Only LMDZ text output contains the exact ordering of the station.
1922	# We isolate this in the code below:
1923	# 0 38 -157.5000000000000 70.98591549295774
1924	# 0 54 27.49999999999999 67.18309859154928
1925	# 0 56 -62.50000000000001 82.39436619718309
1926	# 0 79 12.49999999999999 78.59154929577466
1927	# 0 116 -165.0000000000000 76.05633802816901
1928	# 0 117 130.0000000000000 70.98591549295774
1929	# 0 118 110.0000000000000 87.46478873239437
1930	# 1 40 4.999999999999995 51.97183098591550
1931	# typeset iStation iProc list_opt file_in file_out prefix_invert
1932	# typeset -Z4 j4
1933	# typeset -Z3 j3
1934
1935	# unset list_opt
1936	# set +A list_opt $@
1937
1938	# Filename after rebuild
1939	# file_out=${list_opt[0]}
1940	# Prefix of output files
1941	# prefix_invert=$( basename ${file_out} .nc )
1942	# Number of procs
1943	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
1944
1945	# iProc=0
1946	# while [ ${iProc} -lt ${num_proc} ] ; do
1947	# Array containing Station as a number
1948	# unset proc_stn
1949	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| awk ' {print $2}' )
1950	# Number of stations produced by processor proc
1951	# stationLast=${#proc_stn[*]}
1952	# Proc number on 4 digits
1953	# j4=${iProc}
1954	# Init
1955	# iStation=0
1956	# while [ ${iStation} -lt ${stationLast} ] ; do
1957	# Station number on 3 digits
1958	# j3=${proc_stn[${iStation}]}
1959	# Extract station
1960	# Invert Axis : t,x -> x,t
1961	# t,pres,x -> x,t,pres
1962	# So that we can concatenate along x
1963	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1964	# (( iStation = iStation + 1 ))
1965	# done
1966	# (( iProc = iProc + 1 ))
1967	# done
1968
1969	# Concatenate all station along x
1970	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1971
1972	# Re-invert file
1973	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1974
1975	# END reordering
1976
1977	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1978	}
1979
1980	############################################################
1981	# Activate Running Environnment Variables
1982
1983	function IGCM_sys_activ_variables {
1984	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1985	if ( $DEBUG_sys ) ; then
1986	echo "IGCM_sys_activ_variables"
1987	fi
1988
1989	ulimit -s unlimited
1990
1991	export KMP_STACKSIZE=3g
1992	export KMP_LIBRARY=turnaround
1993	export MKL_SERIAL=YES
1994	unset MPI_PLACING_MODE
1995
1996	# add module load nb of threads for core placement
1997	(( max_omp = 0 ))
1998
1999	for comp in ${config_ListOfComponents[*]} ; do
2000	eval ExeNameIn=\${config_Executable_${comp}[0]}
2001	eval ExeNameOut=\${config_Executable_${comp}[1]}
2002
2003	# Only if we really have an executable for the component :
2004	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2005	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2006	if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
2007	(( max_omp = comp_proc_omp_loc ))
2008	fi
2009	fi
2010	done
2011
2012	( [ X${max_omp} = X4 ] \|\| [ X${max_omp} = X8 ] ) && { module load openmp/${max_omp}thds ; IGCM_debug_Print 1 "module load openmp/${max_omp}thds" ; }
2013
2014	IGCM_debug_PopStack "IGCM_sys_activ_variables"
2015	}
2016
2017	############################################################
2018	# Desactivate Running Environnment Variables
2019
2020	function IGCM_sys_desactiv_variables {
2021	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
2022	if ( $DEBUG_sys ) ; then
2023	echo "IGCM_sys_desactiv_variables"
2024	fi
2025	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
2026	}
2027
2028	############################################################
2029	# Build MPI/OMP scripts run file (dummy function)
2030
2031	function IGCM_sys_build_run_file {
2032
2033	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
2034
2035	}
2036
2037	############################################################
2038	# Build MPI/OMP scripts
2039	function IGCM_sys_build_execution_scripts
2040	{
2041	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
2042	if ( $DEBUG_sys ) ; then
2043	echo "IGCM_sys_build_execution_scripts " $@
2044	fi
2045	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
2046	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
2047	typeset num_corempi nombre_restant_node nombre_restant_comp
2048	typeset IGCM_OMP
2049
2050	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
2051	IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
2052	fi
2053
2054	if ( ${OK_PARA_MPMD} ) ; then
2055
2056	if [ -f run_file ] ; then
2057	IGCM_sys_Rm -f run_file
2058	fi
2059	touch run_file
2060
2061	if ( ${OK_PARA_OMP} ) ; then
2062
2063	# Hosts treatment
2064
2065	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
2066
2067	i=0
2068	rm -f hosts
2069	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
2070	for nodes in `cat hosts.tmp` ; do
2071	host[$i]=$nodes
2072	echo "${host[$i]} slots=8 max_slots=8" >> hosts
2073	IGCM_debug_Print 1 ${host[$i]}
2074	i=$((i+1))
2075	done
2076	rm -f hosts.tmp
2077
2078	listnodes=${host[*]}
2079
2080	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
2081
2082	# Initialisation
2083
2084	init_node=y
2085	node_num_current=0
2086	start_num=0
2087	init_exec=n
2088
2089	# Test : if oasis is there, we put it at the first position
2090
2091	for comp in ${config_ListOfComponents[*]} ; do
2092
2093	if [ "X${comp}" = "XCPL" ] ; then
2094
2095	eval ExeNameIn=\${config_Executable_${comp}[0]}
2096	eval ExeNameOut=\${config_Executable_${comp}[1]}
2097
2098	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2099	echo "" >> script_${ExeNameOut}.ksh
2100	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2101	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2102	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2103	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2104	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2105
2106	init_node=n
2107
2108	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
2109	node_num_current=0
2110	node_current=${host[${node_num_current}]}
2111
2112	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
2113
2114	init_exec=y
2115	start_num=1
2116
2117	fi
2118
2119	done
2120
2121	# Then loop on the components (except for oasis)
2122
2123	for comp in ${config_ListOfComponents[*]} ; do
2124
2125	eval ExeNameIn=\${config_Executable_${comp}[0]}
2126	eval ExeNameOut=\${config_Executable_${comp}[1]}
2127
2128	# Only if we really have an executable for the component :
2129	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2130
2131	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2132	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2133
2134	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2135	# echo "set -vx" >> script_${ExeNameOut}.ksh
2136	echo "" >> script_${ExeNameOut}.ksh
2137	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2138	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2139	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2140	echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2141	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
2142	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
2143	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2144
2145	node_num=0
2146
2147	# We define the number of MPI process to be assigned for the component
2148
2149	nombre_restant_comp=${comp_proc_mpi_loc}
2150
2151	# Loop on the allocated nodes
2152
2153	for node in ${listnodes} ; do
2154
2155	# We go to the current node
2156	if [ ${node_num} = ${node_num_current} ] ; then
2157
2158	node_current=${host[${node_num_current}]}
2159
2160	# If first time on the node : initialisation
2161
2162	if [ ${init_node} = y ] ; then
2163	nombre_restant_node=${NUM_COREPERNODE}
2164	fi
2165
2166	# Test on the number of OMP threads
2167
2168	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
2169	(( node_num = node_num + 1 ))
2170	node_num_current=${node_num}
2171	init_node=y
2172	continue
2173	fi
2174
2175	# Number of MPI process to assign
2176
2177	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
2178
2179	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
2180	num_corempi=${nombre_restant_comp}
2181	fi
2182
2183	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
2184	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
2185
2186	if [ ${init_exec} = y ] ; then
2187	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
2188	else
2189	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
2190	init_exec=y
2191	fi
2192
2193	(( start_num = num_corempi + start_num ))
2194
2195	else
2196
2197	(( node_num = node_num + 1 ))
2198	continue
2199	fi
2200
2201	# Test on the number of core/process remaining on the node/component
2202
2203	if [ ${nombre_restant_node} = 0 ] ; then
2204	(( node_num = node_num + 1 ))
2205	node_num_current=${node_num}
2206	init_node=y
2207
2208	if [ ${nombre_restant_comp} = 0 ] ; then
2209	break 1
2210	fi
2211	else
2212
2213	node_num_current=${node_num}
2214	init_node=n
2215
2216	if [ ${nombre_restant_comp} = 0 ] ; then
2217	break 1
2218	fi
2219	fi
2220	done
2221	fi
2222	done
2223
2224	else
2225
2226	# Then first loop on the components for the coupler ie oasis
2227
2228	## the coupler ie oasis must be the first one
2229	for comp in ${config_ListOfComponents[*]} ; do
2230
2231	eval ExeNameOut=\${config_Executable_${comp}[1]}
2232
2233	# for CPL component only
2234	if [ "X${comp}" = "XCPL" ] ; then
2235	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2236	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
2237	fi
2238	done
2239
2240	# Then second loop on the components
2241
2242	for comp in ${config_ListOfComponents[*]} ; do
2243
2244	eval ExeNameOut=\${config_Executable_${comp}[1]}
2245
2246	# Only if we really have an executable for the component and not the coupler ie oasis:
2247	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2248	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2249	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2250	fi
2251	done
2252	IGCM_sys_Chmod u+x run_file
2253
2254	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
2255
2256	fi
2257
2258	else # Only one executable.
2259	# Check has to be added with :
2260	# Job :
2261	# #MSUB -n 8 # reservation des processeurs pour le job
2262	# #MSUB -N 4 # reservation des nodes pour le job
2263	# BATCH_NUM_PROC_TOT=32
2264	#
2265	# config.card :
2266	# ATM= (gcm.e, lmdz.x, 8MPI, 4OMP)
2267	#
2268	# tested with :
2269	# Job :
2270	# #MSUB -n 32
2271	# BATCH_NUM_PROC_TOT=32
2272	#
2273	# config.card :
2274	# ATM= (gcm.e, lmdz.x)
2275
2276
2277	for comp in ${config_ListOfComponents[*]} ; do
2278
2279	# Only if we really have an executable for the component :
2280	eval ExeNameOut=\${config_Executable_${comp}[1]}
2281	### if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
2282	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2283
2284	IGCM_OMP=1
2285
2286	if ( ${OK_PARA_OMP} ) ; then
2287	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2288	export OMP_NUM_THREADS=${comp_proc_omp_loc}
2289	IGCMG_OMP=${comp_proc_omp_loc}
2290	fi
2291	if ( ${OK_PARA_MPI} ) ; then
2292	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2293	# Default : mpirun used if nb_proc gt 1
2294	# pour sortie out/err par process ? more test required
2295	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
2296	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
2297	else
2298	# Check if BATCH_NUM_PROC_TOT = IGCMG_OMP
2299	if ( [ ${IGCM_OMP} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
2300	IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> ${comp}_PROC_OMP : ${BATCH_NUM_PROC_TOT} <> ${BRIDGE_MSUB_NPROC} "
2301	IGCM_debug_Print 1 "sys Titane : Check #MSUB -n xx into Job"
2302	IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
2303	fi
2304	# Default : mpirun is NOT used if nb_proc eq 1
2305	# pour sortie out/err par process ? more test required
2306	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2307	EXECUTION="/usr/bin/time ./${ExeNameOut}"
2308	fi
2309	fi
2310	done
2311
2312	fi
2313
2314	IGCM_debug_Print 1 "sys Titane : La commande d execution est "
2315	IGCM_debug_Print 1 "$EXECUTION"
2316
2317	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
2318	}
2319
2320	############################################################
2321	# Check of space available on temporary filesytems
2322	function IGCM_sys_check_quota {
2323	IGCM_debug_PushStack "IGCM_sys_check_quota"
2324	if ( $DEBUG_sys ) ; then
2325	echo "IGCM_sys_check_quota "
2326	fi
2327	# Limit of quota (in %)
2328	limit_quota=90
2329
2330	# Check of the volume
2331	volume_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $2}')
2332	volume_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $3}')
2333
2334	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2335
2336	unit_avail=${volume_avail: -1}
2337	unit_quota=${volume_quota: -1}
2338
2339	if [ "${unit_quota}" = "*" ] ; then
2340	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2341	IGCM_debug_Print 1 "More than 100% of your quota is used"
2342	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2343	IGCM_debug_Print 1 "You must have more than 10% available to run"
2344	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2345	IGCM_debug_Verif_Exit
2346	fi
2347
2348	temp_avail=${volume_avail%%${volume_avail: -1}*}
2349	temp_quota=${volume_quota%%${volume_quota: -1}*}
2350
2351	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2352
2353	# Convertion
2354	if [ ${volume_avail: -1} = "T" ] ; then
2355	(( temp_avail = temp_avail * 1000000000000 ))
2356	elif [ ${volume_avail: -1} = "G" ] ; then
2357	(( temp_avail = temp_avail * 1000000000 ))
2358	elif [ ${volume_avail: -1} = "M" ] ; then
2359	(( temp_avail = temp_avail * 1000000 ))
2360	elif [ ${volume_avail: -1} = "k" ] ; then
2361	(( temp_avail = temp_avail * 1000 ))
2362	else
2363	(( temp_avail = volume_avail ))
2364	fi
2365	if [ ${volume_quota: -1} = "T" ] ; then
2366	(( temp_quota = temp_quota * 1000000000000 ))
2367	elif [ ${volume_quota: -1} = "G" ] ; then
2368	(( temp_quota = temp_quota * 1000000000 ))
2369	elif [ ${volume_quota: -1} = "M" ] ; then
2370	(( temp_quota = temp_quota * 1000000 ))
2371	elif [ ${volume_quota: -1} = "k" ] ; then
2372	(( temp_quota = temp_quota * 1000 ))
2373	else
2374	(( temp_quota = volume_quota ))
2375	fi
2376	fi
2377
2378	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2379	# echo "volume ratio is " $quota_volume
2380
2381	if [ ${quota_volume} -ge ${limit_quota} ] ; then
2382	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2383	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2384	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2385	IGCM_debug_Print 1 "You must have more than 10% available to run"
2386	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2387	IGCM_debug_Verif_Exit
2388	fi
2389
2390	fi
2391
2392	# Check of the number of inodes
2393
2394	inode_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $6}')
2395	inode_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $7}')
2396
2397	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2398
2399	unit_avail=${inode_avail: -1}
2400	unit_quota=${inode_quota: -1}
2401
2402	if [ "${unit_quota}" = "*" ] ; then
2403	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2404	IGCM_debug_Print 1 "More than 100% of your quota is used"
2405	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2406	IGCM_debug_Print 1 "You must have more than 10% available to run"
2407	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2408	IGCM_debug_Verif_Exit
2409	fi
2410
2411	temp_avail=${inode_avail%%${inode_avail: -1}*}
2412	temp_quota=${inode_quota%%${inode_quota: -1}*}
2413
2414	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2415
2416	# Convertion
2417	if [ ${inode_avail: -1} = "T" ] ; then
2418	(( temp_avail = temp_avail * 1000000000000 ))
2419	elif [ ${inode_avail: -1} = "G" ] ; then
2420	(( temp_avail = temp_avail * 1000000000 ))
2421	elif [ ${inode_avail: -1} = "M" ] ; then
2422	(( temp_avail = temp_avail * 1000000 ))
2423	elif [ ${inode_avail: -1} = "k" ] ; then
2424	(( temp_avail = temp_avail * 1000 ))
2425	else
2426	(( temp_avail = inode_avail ))
2427	fi
2428
2429	if [ ${inode_quota: -1} = "T" ] ; then
2430	(( temp_quota = temp_quota * 1000000000000 ))
2431	elif [ ${inode_quota: -1} = "G" ] ; then
2432	(( temp_quota = temp_quota * 1000000000 ))
2433	elif [ ${inode_quota: -1} = "M" ] ; then
2434	(( temp_quota = temp_quota * 1000000 ))
2435	elif [ ${inode_quota: -1} = "k" ] ; then
2436	(( temp_quota = temp_quota * 1000 ))
2437	else
2438	(( temp_quota = inode_quota ))
2439	fi
2440	fi
2441	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2442	# echo "inode ratio is " $quota_inode
2443
2444	if [ ${quota_inode} -ge ${limit_quota} ] ; then
2445	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2446	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2447	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2448	IGCM_debug_Print 1 "You must have more than 10% available to run"
2449	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2450	IGCM_debug_Verif_Exit
2451	fi
2452
2453	fi
2454	IGCM_debug_PopStack "IGCM_sys_check_quota"
2455	}
2456
2457	##############################################################
2458	# NCO OPERATOR
2459
2460	function IGCM_sys_ncap2 {
2461	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2462	if ( $DEBUG_sys ) ; then
2463	echo "IGCM_sys_ncap2 :" $@
2464	fi
2465
2466	typeset NB_ESSAI DELAI status i
2467	# number of tentative
2468	NB_ESSAI=3
2469	# time delay between tentative
2470	DELAI=2
2471
2472	i=0
2473	while [ $i -lt $NB_ESSAI ] ; do
2474	ncap2 "$@" > out_rsync 2>&1
2475	status=$?
2476	if [ ${status} -gt 0 ] ; then
2477	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
2478	cat out_rsync
2479	\rm out_rsync
2480	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2481	sleep $DELAI
2482	else
2483	\rm out_rsync
2484	break
2485	fi
2486	(( i = i + 1 ))
2487	done
2488
2489	if [ ${status} -gt 0 ] ; then
2490	echo "IGCM_sys_ncap2 : ncap2 error"
2491	IGCM_debug_Exit "ncap2"
2492	fi
2493
2494	IGCM_debug_PopStack "IGCM_sys_ncap2"
2495	}
2496
2497	function IGCM_sys_ncatted {
2498	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2499	if ( $DEBUG_sys ) ; then
2500	echo "IGCM_sys_ncatted :" $@
2501	fi
2502
2503	typeset NB_ESSAI DELAI status i
2504	# number of tentative
2505	NB_ESSAI=3
2506	# time delay between tentative
2507	DELAI=2
2508
2509	i=0
2510	while [ $i -lt $NB_ESSAI ] ; do
2511	ncatted "$@" > out_rsync 2>&1
2512	status=$?
2513	if [ ${status} -gt 0 ] ; then
2514	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2515	cat out_rsync
2516	\rm out_rsync
2517	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2518	sleep $DELAI
2519	else
2520	\rm out_rsync
2521	break
2522	fi
2523	(( i = i + 1 ))
2524	done
2525
2526	if [ ${status} -gt 0 ] ; then
2527	echo "IGCM_sys_ncatted : ncatted error"
2528	IGCM_debug_Exit "ncatted"
2529	fi
2530
2531	IGCM_debug_PopStack "IGCM_sys_ncatted"
2532	}
2533
2534	function IGCM_sys_ncbo {
2535	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2536	if ( $DEBUG_sys ) ; then
2537	echo "IGCM_sys_ncbo :" $@
2538	fi
2539
2540	typeset NB_ESSAI DELAI status i
2541	# number of tentative
2542	NB_ESSAI=3
2543	# time delay between tentative
2544	DELAI=2
2545
2546	i=0
2547	while [ $i -lt $NB_ESSAI ] ; do
2548	ncbo $@ > out_rsync 2>&1
2549	status=$?
2550	if [ ${status} -gt 0 ] ; then
2551	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2552	cat out_rsync
2553	\rm out_rsync
2554	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2555	sleep $DELAI
2556	else
2557	\rm out_rsync
2558	break
2559	fi
2560	(( i = i + 1 ))
2561	done
2562
2563	if [ ${status} -gt 0 ] ; then
2564	echo "IGCM_sys_ncbo : ncbo error"
2565	IGCM_debug_Exit "ncbo"
2566	fi
2567
2568	IGCM_debug_PopStack "IGCM_sys_ncbo"
2569	}
2570
2571	function IGCM_sys_ncdiff {
2572	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2573	if ( $DEBUG_sys ) ; then
2574	echo "IGCM_sys_ncdiff :" $@
2575	fi
2576
2577	typeset NB_ESSAI DELAI status i
2578	# number of tentative
2579	NB_ESSAI=3
2580	# time delay between tentative
2581	DELAI=2
2582
2583	i=0
2584	while [ $i -lt $NB_ESSAI ] ; do
2585	ncdiff $@ > out_rsync 2>&1
2586	status=$?
2587	if [ ${status} -gt 0 ] ; then
2588	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2589	cat out_rsync
2590	\rm out_rsync
2591	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2592	sleep $DELAI
2593	else
2594	\rm out_rsync
2595	break
2596	fi
2597	(( i = i + 1 ))
2598	done
2599
2600	if [ ${status} -gt 0 ] ; then
2601	echo "IGCM_sys_ncdiff : ncdiff error"
2602	IGCM_debug_Exit "ncdiff"
2603	fi
2604
2605	IGCM_debug_PopStack "IGCM_sys_ncdiff"
2606	}
2607
2608	function IGCM_sys_ncea {
2609	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2610	if ( $DEBUG_sys ) ; then
2611	echo "IGCM_sys_ncea :" $@
2612	fi
2613
2614	typeset NB_ESSAI DELAI status i
2615	# number of tentative
2616	NB_ESSAI=3
2617	# time delay between tentative
2618	DELAI=2
2619
2620	i=0
2621	while [ $i -lt $NB_ESSAI ] ; do
2622	ncea $@ > out_rsync 2>&1
2623	status=$?
2624	if [ ${status} -gt 0 ] ; then
2625	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2626	cat out_rsync
2627	\rm out_rsync
2628	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2629	sleep $DELAI
2630	else
2631	\rm out_rsync
2632	break
2633	fi
2634	(( i = i + 1 ))
2635	done
2636
2637	if [ ${status} -gt 0 ] ; then
2638	echo "IGCM_sys_ncea : ncea error"
2639	IGCM_debug_Exit "ncea"
2640	fi
2641
2642	IGCM_debug_PopStack "IGCM_sys_ncea"
2643	}
2644
2645	function IGCM_sys_ncecat {
2646	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2647	if ( $DEBUG_sys ) ; then
2648	echo "IGCM_sys_ncecat :" $@
2649	fi
2650
2651	typeset NB_ESSAI DELAI status i
2652	# number of tentative
2653	NB_ESSAI=3
2654	# time delay between tentative
2655	DELAI=2
2656
2657	i=0
2658	while [ $i -lt $NB_ESSAI ] ; do
2659	ncecat $@ > out_rsync 2>&1
2660	status=$?
2661	if [ ${status} -gt 0 ] ; then
2662	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2663	cat out_rsync
2664	\rm out_rsync
2665	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2666	sleep $DELAI
2667	else
2668	\rm out_rsync
2669	break
2670	fi
2671	(( i = i + 1 ))
2672	done
2673
2674	if [ ${status} -gt 0 ] ; then
2675	echo "IGCM_sys_ncecat : ncecat error"
2676	IGCM_debug_Exit "ncecat"
2677	fi
2678
2679	IGCM_debug_PopStack "IGCM_sys_ncecat"
2680	}
2681
2682	function IGCM_sys_ncflint {
2683	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2684	if ( $DEBUG_sys ) ; then
2685	echo "IGCM_sys_ncflint :" $@
2686	fi
2687
2688	typeset NB_ESSAI DELAI status i
2689	# number of tentative
2690	NB_ESSAI=3
2691	# time delay between tentative
2692	DELAI=2
2693
2694	i=0
2695	while [ $i -lt $NB_ESSAI ] ; do
2696	ncflint $@ > out_rsync 2>&1
2697	status=$?
2698	if [ ${status} -gt 0 ] ; then
2699	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2700	cat out_rsync
2701	\rm out_rsync
2702	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2703	sleep $DELAI
2704	else
2705	\rm out_rsync
2706	break
2707	fi
2708	(( i = i + 1 ))
2709	done
2710
2711	if [ ${status} -gt 0 ] ; then
2712	echo "IGCM_sys_ncflint : ncflint error"
2713	IGCM_debug_Exit "ncflint"
2714	fi
2715
2716	IGCM_debug_PopStack "IGCM_sys_ncflint"
2717	}
2718
2719	function IGCM_sys_ncks {
2720	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2721	if ( $DEBUG_sys ) ; then
2722	echo "IGCM_sys_ncks :" $@
2723	fi
2724
2725	typeset NB_ESSAI DELAI status i
2726	# number of tentative
2727	NB_ESSAI=3
2728	# time delay between tentative
2729	DELAI=2
2730
2731	i=0
2732	while [ $i -lt $NB_ESSAI ] ; do
2733	ncks $@ > out_rsync 2>&1
2734	status=$?
2735	if [ ${status} -gt 0 ] ; then
2736	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2737	cat out_rsync
2738	\rm out_rsync
2739	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2740	sleep $DELAI
2741	else
2742	\rm out_rsync
2743	break
2744	fi
2745	(( i = i + 1 ))
2746	done
2747
2748	if [ ${status} -gt 0 ] ; then
2749	echo "IGCM_sys_ncks : ncks error"
2750	IGCM_debug_Exit "ncks"
2751	fi
2752
2753	IGCM_debug_PopStack "IGCM_sys_ncks"
2754	}
2755
2756	function IGCM_sys_ncpdq {
2757	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2758	if ( $DEBUG_sys ) ; then
2759	echo "IGCM_sys_ncpdq :" $@
2760	fi
2761
2762	typeset NB_ESSAI DELAI status i
2763	# number of tentative
2764	NB_ESSAI=3
2765	# time delay between tentative
2766	DELAI=2
2767
2768	i=0
2769	while [ $i -lt $NB_ESSAI ] ; do
2770	ncpdq $@ > out_rsync 2>&1
2771	status=$?
2772	if [ ${status} -gt 0 ] ; then
2773	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2774	cat out_rsync
2775	\rm out_rsync
2776	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2777	sleep $DELAI
2778	else
2779	\rm out_rsync
2780	break
2781	fi
2782	(( i = i + 1 ))
2783	done
2784
2785	if [ ${status} -gt 0 ] ; then
2786	echo "IGCM_sys_ncpdq : ncpdq error"
2787	IGCM_debug_Exit "ncpdq"
2788	fi
2789
2790	IGCM_debug_PopStack "IGCM_sys_ncpdq"
2791	}
2792
2793	function IGCM_sys_ncra {
2794	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2795	if ( $DEBUG_sys ) ; then
2796	echo "IGCM_sys_ncra :" $@
2797	fi
2798
2799	typeset NB_ESSAI DELAI status i
2800	# number of tentative
2801	NB_ESSAI=3
2802	# time delay between tentative
2803	DELAI=2
2804
2805	i=0
2806	while [ $i -lt $NB_ESSAI ] ; do
2807	ncra $@ > out_rsync 2>&1
2808	status=$?
2809	if [ ${status} -gt 0 ] ; then
2810	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2811	cat out_rsync
2812	\rm out_rsync
2813	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2814	sleep $DELAI
2815	else
2816	\rm out_rsync
2817	break
2818	fi
2819	(( i = i + 1 ))
2820	done
2821
2822	if [ ${status} -gt 0 ] ; then
2823	echo "IGCM_sys_ncra : ncra error"
2824	IGCM_debug_Exit "ncra"
2825	fi
2826
2827	IGCM_debug_PopStack "IGCM_sys_ncra"
2828	}
2829
2830	function IGCM_sys_ncrcat {
2831	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2832	if ( $DEBUG_sys ) ; then
2833	echo "IGCM_sys_ncrcat :" $@
2834	fi
2835
2836	typeset NB_ESSAI DELAI status i
2837	# number of tentative
2838	NB_ESSAI=3
2839	# time delay between tentative
2840	DELAI=2
2841
2842	i=0
2843	while [ $i -lt $NB_ESSAI ] ; do
2844	ncrcat $@ > out_rsync 2>&1
2845	status=$?
2846	if [ ${status} -gt 0 ] ; then
2847	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2848	cat out_rsync
2849	\rm out_rsync
2850	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2851	sleep $DELAI
2852	else
2853	\rm out_rsync
2854	break
2855	fi
2856	(( i = i + 1 ))
2857	done
2858
2859	if [ ${status} -gt 0 ] ; then
2860	echo "IGCM_sys_ncrcat : ncrcat error"
2861	#IGCM_debug_Exit "ncrcat"
2862	fi
2863
2864	IGCM_debug_PopStack "IGCM_sys_ncrcat"
2865	}
2866
2867	function IGCM_sys_ncrename {
2868	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2869	if ( $DEBUG_sys ) ; then
2870	echo "IGCM_sys_ncrename :" $@
2871	fi
2872
2873	typeset NB_ESSAI DELAI status i
2874	# number of tentative
2875	NB_ESSAI=3
2876	# time delay between tentative
2877	DELAI=2
2878
2879	i=0
2880	while [ $i -lt $NB_ESSAI ] ; do
2881	ncrename $@ > out_rsync 2>&1
2882	status=$?
2883	if [ ${status} -gt 0 ] ; then
2884	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2885	cat out_rsync
2886	\rm out_rsync
2887	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2888	sleep $DELAI
2889	else
2890	\rm out_rsync
2891	break
2892	fi
2893	(( i = i + 1 ))
2894	done
2895
2896	if [ ${status} -gt 0 ] ; then
2897	echo "IGCM_sys_ncrename : ncrename error"
2898	IGCM_debug_Exit "ncrename"
2899	fi
2900
2901	IGCM_debug_PopStack "IGCM_sys_ncrename"
2902	}
2903
2904	function IGCM_sys_ncwa {
2905	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2906	if ( $DEBUG_sys ) ; then
2907	echo "IGCM_sys_ncwa :" $@
2908	fi
2909
2910	typeset NB_ESSAI DELAI status i
2911	# number of tentative
2912	NB_ESSAI=3
2913	# time delay between tentative
2914	DELAI=2
2915
2916	i=0
2917	while [ $i -lt $NB_ESSAI ] ; do
2918	ncwa $@ > out_rsync 2>&1
2919	status=$?
2920	if [ ${status} -gt 0 ] ; then
2921	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2922	cat out_rsync
2923	\rm out_rsync
2924	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2925	sleep $DELAI
2926	else
2927	\rm out_rsync
2928	break
2929	fi
2930	(( i = i + 1 ))
2931	done
2932
2933	if [ ${status} -gt 0 ] ; then
2934	echo "IGCM_sys_ncwa : ncwa error"
2935	IGCM_debug_Exit "ncwa"
2936	fi
2937
2938	IGCM_debug_PopStack "IGCM_sys_ncwa"
2939	}
2940
2941	##############################################################
2942	# CDO OPERATOR
2943
2944	function IGCM_sys_cdo {
2945	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2946	if ( $DEBUG_sys ) ; then
2947	echo "IGCM_sys_cdo :" $@
2948	fi
2949
2950	typeset status
2951
2952	\cdo $@ > out_rsync 2>&1
2953	status=$?
2954	if [ ${status} -gt 0 ] ; then
2955	echo "IGCM_sys_cdo : error code ${status}"
2956	cat out_rsync
2957	\rm out_rsync
2958	IGCM_debug_PopStack "IGCM_sys_cdo"
2959	return 1
2960	else
2961	IGCM_debug_PopStack "IGCM_sys_cdo"
2962	return 0
2963	fi
2964
2965	IGCM_debug_PopStack "IGCM_sys_cdo"
2966	}
2967
2968	#D-#==================================================
2969	#D-function IGCM_sys_CountJobInQueue
2970	#D-* Purpose: Check if job_name is currently
2971	#D- running or in queue
2972	#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2973	#D-
2974
2975	function IGCM_sys_CountJobInQueue {
2976	IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2977	if ( $DEBUG_sys ) ; then
2978	echo "IGCM_sys_CountJobInQueue"
2979	fi
2980
2981	#NbRun=$( ccc_mstat -f \| grep -c ${JobName} )
2982
2983	# With -f option, the full job name is given in the last column
2984	NbRun=$( ccc_mstat -f \| gawk -v JobName=$1 'BEGIN { x=0 } ($NF~String) { x=x+1 } END { print x }' )
2985
2986	eval ${2}=${NbRun}
2987
2988	IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2989	}
2990

Note: See TracBrowser for help on using the repository browser.

Download in other formats: