Context Navigation

source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh @ 729

Last change on this file since 729 was 729, checked in by sdipsl, 12 years ago
special for O. Boucher. cp command says copy failed "permission denied" ; error code = 1 ; destination file exist after the copy and not before. ?!*% Strange
Property svn:keywords set to `Revision Author Date`
File size: 83.9 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Titane
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe param files \| \| Chmod \| \|
46	# \| \| Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=titane
71
72	#D-
73	#D-#==================================================
74	#D-Program used in libIGCM
75	#D-#==================================================
76
77	# rsync with path
78	typeset -r RSYNC=/usr/bin/rsync
79	# RSYNC_opt args to rsync
80	typeset -r RSYNC_opt="-va"
81	# ie storage filesystem
82	typeset -r STOREHOST=${MASTER}
83
84	#====================================================
85	# Source default environment
86	#====================================================
87	. /etc/profile
88
89	#====================================================
90	# Set environment tools (ferret, nco, cdo)
91	#====================================================
92	. /home/cont003/p86ipsl/.atlas_env_titane_ksh
93	if ( [ X${LSB_QUEUE} = Xmono ] \|\| [ X${LSB_QUEUE} = Xmonoext ] ) ; then
94	module switch nco/3.9.4 nco/3.9.4_netcdf4
95	fi
96
97	#====================================================
98	# Specific for ocean additionnal diagnostic
99	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
100	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
101
102	#====================================================
103	# Host specific DIRECTORIES
104	#====================================================
105
106	# ============ CESIUM START ============ #
107
108	#====================================================
109	#- Mirror libIGCM from titane to cesium if needed
110	#ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $3}' )
111	#if [ ! ${ROOTSYS} = "home" ] ; then
112	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
113	#else
114	# typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
115	#fi
116
117	#====================================================
118	#- libIGCM_POST
119	#if ( ${MirrorlibIGCM} ) ; then
120	# PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
121	# typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
122	#else
123	# typeset -r libIGCM_POST=${libIGCM}
124	#fi
125
126	# ============ CESIUM END ============ #
127
128	#====================================================
129	#- MirrorlibIGCM for frontend
130	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
131
132	#====================================================
133	#- libIGCM_POST for frontend
134	typeset -r libIGCM_POST=${libIGCM}
135
136	#====================================================
137	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
138	typeset -r R_EXE="${MODIPSL}/bin"
139
140	#====================================================
141	#- SUBMIT_DIR : submission dir
142	typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
143
144	#====================================================
145	#- IN
146	typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
147	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
148
149	#====================================================
150	#- ARCHIVE (dedicated to large files)
151	typeset ARCHIVE=${CCCSTOREDIR}
152
153	#====================================================
154	#- STORAGE (dedicated to small/medium files)
155	typeset STORAGE=${CCCWORKDIR}
156
157	#====================================================
158	#- R_OUT
159	typeset R_OUT=${ARCHIVE}/IGCM_OUT
160
161	#====================================================
162	#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
163	typeset R_FIG=${STORAGE}/IGCM_OUT
164
165	#====================================================
166	#- R_BUF (ONLY FOR double copy an scratch)
167	typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
168
169	#====================================================
170	#- BIG_DIR : BIG_DIR to store files waiting for rebuild
171	typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
172
173	#====================================================
174	#- OUT_POST
175	typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
176
177	#====================================================
178	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
179	if ( [ ! X${LSB_QUEUE} = Xmono ] && [ ! X${LSB_QUEUE} = Xmonoext ] ) ; then
180	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
181	else
182	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
183	fi
184
185	#====================================================
186	#- HOST_MPIRUN_COMMAND
187	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
188
189	#====================================================
190	#- Max number of arguments passed to nco operator or demigration command
191	UNIX_MAX_LIMIT=360
192
193	#====================================================
194	#- set PackDefault true on titane
195	PackDefault=true
196
197	#====================================================
198	#- Number of core per node (max number of OpenMP task)
199	NUM_COREPERNODE=8
200
201	#====================================================
202	#- Default number of MPI task for IPSL coupled model
203	#- required for backward compatibility
204	#-
205	DEFAULT_NUM_PROC_OCE=5
206	DEFAULT_NUM_PROC_CPL=1
207	(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
208	DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
209
210	#D-#==================================================
211	#D-function IGCM_sys_ChangeArchive
212	#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
213	#D-* Examples:
214	#D-
215	function IGCM_sys_ChangeArchive {
216	IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
217
218	ARCHIVE=${SCRATCHDIR}
219	STORAGE=${SCRATCHDIR}
220	R_OUT=${ARCHIVE}/IGCM_OUT
221	R_FIG=${STORAGE}/IGCM_OUT
222
223	IGCM_debug_Print 1 "ARCHIVE has been redefined = ${ARCHIVE}"
224	IGCM_debug_Print 1 "STORAGE has been redefined = ${STORAGE}"
225	IGCM_debug_Print 1 "R_OUT has been redefined = ${R_OUT}"
226	IGCM_debug_Print 1 "R_FIG has been redefined = ${R_FIG}"
227
228	IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
229	}
230
231	#D-#==================================================
232	#D-function IGCM_sys_RshMaster
233	#D-* Purpose: Connection to frontend machine.
234	#D-* Examples:
235	#D-
236	function IGCM_sys_RshMaster {
237	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
238	/bin/ksh <<-EOF
239	export libIGCM=${libIGCM}
240	export DEBUG_debug=${DEBUG_debug}
241	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
242	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
243	${@}
244	EOF
245	if [ $? -gt 0 ] ; then
246	echo "IGCM_sys_RshMaster : erreur."
247	IGCM_debug_Exit "IGCM_sys_RshMaster"
248	fi
249	IGCM_debug_PopStack "IGCM_sys_RshMaster"
250	}
251
252	#D-#==================================================
253	#D-function IGCM_sys_RshArchive
254	#D-* Purpose: Archive rsh command
255	#D-* Examples:
256	#D-
257	function IGCM_sys_RshArchive {
258	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
259	/bin/ksh <<-EOF
260	${@}
261	EOF
262	if [ $? -gt 0 ] ; then
263	echo "IGCM_sys_RshArchive : erreur."
264	IGCM_debug_Exit "IGCM_sys_RshArchive"
265	fi
266	IGCM_debug_PopStack "IGCM_sys_RshArchive"
267	}
268
269	#D-#==================================================
270	#D-function IGCM_sys_RshPost
271	#D-* Purpose: Post-process rsh command
272	#D-* Examples:
273	#D-
274	function IGCM_sys_RshPost {
275	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
276	if ( $DEBUG_sys ) ; then
277	echo "IGCM_sys_RshPost :" $@
278	fi
279
280	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
281	#cat tmp_IGCM_sys_RshPost_$$
282	# keep standard input (stdin) for the loop onto temporary file
283	cat >tmp_IGCM_sys_RshPost_$$
284
285	# ============ FRONTEND START ============ #
286
287	/bin/ksh <tmp_IGCM_sys_RshPost_$$
288	if [ $? -gt 0 ] ; then
289	echo "IGCM_sys_RshPost : erreur."
290	IGCM_debug_Exit "IGCM_sys_RshPost"
291	fi
292	# delete temporary file
293	\rm tmp_IGCM_sys_RshPost_$$
294
295	# ============ FRONTEND END ============ #
296
297	# ============ CESIUM START ============ #
298	# typeset NB_ESSAI DELAI status i
299	# if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
300	# #little hack so that rebuild submission is done on titane not an cesium
301	#
302	# libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
303	# POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
304	# sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
305	# sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
306	# sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
307	# sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
308	# \mv tmp.txt tmp_IGCM_sys_RshPost_$$
309	#
310	# echo cat tmp_IGCM_sys_RshPost_$$ AFTER
311	# cat tmp_IGCM_sys_RshPost_$$
312	#
313	# /bin/ksh <tmp_IGCM_sys_RshPost_$$
314	# if [ $? -gt 0 ] ; then
315	# echo "IGCM_sys_RshPost : erreur."
316	# IGCM_debug_Exit "IGCM_sys_RshPost"
317	# fi
318	# # delete temporary file
319	# \rm tmp_IGCM_sys_RshPost_$$
320	#
321	# else
322	# # number of tentative
323	# NB_ESSAI=10
324	# # time delay between tentative
325	# DELAI=10
326	# i=0
327	# while [ $i -ne $NB_ESSAI ] ; do
328	# ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
329	# status=$?
330	# if [ ${status} -ne 0 ];
331	# then
332	# sleep $DELAI
333	# else
334	# break
335	# fi
336	# let i=$i+1
337	# done
338	# # delete temporary file
339	# \rm tmp_IGCM_sys_RshPost_$$
340	#
341	# if [ ${status} -gt 0 ] ; then
342	# echo "IGCM_sys_RshPost : erreur."
343	# IGCM_debug_Exit "IGCM_sys_RshPost"
344	# fi
345	# fi
346
347	# ============ CESIUM END ============ #
348
349	IGCM_debug_PopStack "IGCM_sys_RshPost"
350	}
351
352	#D-#==================================================
353	#D-function IGCM_sys_SendMail
354	#D-* Purpose: Send mail when simulation is over
355	#D-* Examples:
356	#D-
357	function IGCM_sys_SendMail {
358	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
359	if ( $DEBUG_sys ) ; then
360	echo "IGCM_sys_SendMail :" $@
361	fi
362
363	if ( ${ExitFlag} ) ; then
364	status=failed
365	else
366	status=completed
367	fi
368
369	cat << END_MAIL > job_end.mail
370	Dear ${LOGIN},
371
372	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
373	Job started : ${DateBegin}
374	Job ended : ${DateEnd}
375	Output files are available in ${R_SAVE}
376	Files to be rebuild are temporarily available in ${REBUILD_DIR}
377	Pre-packed files are temporarily available in ${R_BUFR}
378	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
379	END_MAIL
380
381	if [ ! -z ${config_UserChoices_MailName} ] ; then
382	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
383	elif [ -f ~/.forward ] ; then
384	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
385	fi
386	sleep 10
387	rm -f job_end.mail
388
389	if [ $? -gt 0 ] ; then
390	echo "IGCM_sys_SendMail : erreur."
391	IGCM_debug_Exit "IGCM_sys_SendMail"
392	fi
393	IGCM_debug_PopStack "IGCM_sys_SendMail"
394	}
395
396	#D-#==================================================
397	#D-function IGCM_sys_Mkdir
398	#D-* Purpose: Master locale mkdir command
399	#D-* Examples:
400	#D-
401	function IGCM_sys_Mkdir {
402	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
403	if ( $DEBUG_sys ) ; then
404	echo "IGCM_sys_Mkdir :" $@
405	fi
406	if [ ! -d ${1} ]; then
407	\mkdir -p $1
408	if [ $? -gt 0 ] ; then
409	echo "IGCM_sys_Mkdir : erreur."
410	IGCM_debug_Exit "IGCM_sys_Mkdir"
411	fi
412	fi
413	# vérification :
414	if [ ! -d ${1} ] ; then
415	echo "IGCM_sys_Mkdir : erreur."
416	IGCM_debug_Exit "IGCM_sys_Mkdir"
417	fi
418	IGCM_debug_PopStack "IGCM_sys_Mkdir"
419	}
420
421	#D-#==================================================
422	#D-function IGCM_sys_MkdirArchive
423	#D-* Purpose: Mkdir on Archive
424	#D-* Examples:
425	#D-
426	function IGCM_sys_MkdirArchive {
427	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
428	if ( $DEBUG_sys ) ; then
429	echo "IGCM_sys_MkdirArchive :" $@
430	fi
431	#- creation de repertoire sur le serveur fichier
432	if [ ! -d ${1} ]; then
433	\mkdir -p $1
434	if [ $? -gt 0 ] ; then
435	echo "IGCM_sys_MkdirArchive : erreur."
436	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
437	fi
438	fi
439	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
440	}
441
442	#D-#==================================================
443	#D-function IGCM_sys_MkdirWork
444	#D-* Purpose: Mkdir on Work
445	#D-* Examples:
446	#D-
447	function IGCM_sys_MkdirWork {
448	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
449	if ( $DEBUG_sys ) ; then
450	echo "IGCM_sys_MkdirWork :" $@
451	fi
452	#- creation de repertoire sur le serveur fichier
453	if [ ! -d ${1} ]; then
454	\mkdir -p $1
455	if [ $? -gt 0 ] ; then
456	echo "IGCM_sys_MkdirWork : erreur."
457	IGCM_debug_Exit "IGCM_sys_MkdirWork"
458	fi
459	fi
460	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
461	}
462
463	#D-#==================================================
464	#D-function IGCM_sys_Cd
465	#D-* Purpose: master cd command
466	#D-* Examples:
467	#D-
468	function IGCM_sys_Cd {
469	IGCM_debug_PushStack "IGCM_sys_Cd" $@
470	if ( $DEBUG_sys ) ; then
471	echo "IGCM_sys_Cd :" $@
472	fi
473	\cd $1
474	if [ $? -gt 0 ] ; then
475	echo "IGCM_sys_Cd : erreur."
476	IGCM_debug_Exit "IGCM_sys_Cd"
477	fi
478	IGCM_debug_PopStack "IGCM_sys_Cd"
479	}
480
481	#D-#==================================================
482	#D-function IGCM_sys_Chmod
483	#D-* Purpose: Chmod
484	#D-* Examples:
485	#D-
486	function IGCM_sys_Chmod {
487	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
488	if ( $DEBUG_sys ) ; then
489	echo "IGCM_sys_Chmod :" $@
490	fi
491	if [ $DRYRUN -le 1 ]; then
492	\chmod $@
493	if [ $? -gt 0 ] ; then
494	echo "IGCM_sys_Chmod : erreur."
495	IGCM_debug_Exit "IGCM_sys_Chmod"
496	fi
497	else
498	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
499	fi
500	IGCM_debug_PopStack "IGCM_sys_Chmod"
501	}
502
503	#D-#==================================================
504	#D-function IGCM_sys_FileSize
505	#D-* Purpose: Filesize
506	#D-* Examples:
507	#D-
508	function IGCM_sys_FileSize {
509	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
510
511	typeset sizeF
512	set +A sizeF -- $( ls -la ${1} )
513	if [ $? -gt 0 ] ; then
514	IGCM_debug_Exit "IGCM_sys_FileSize"
515	fi
516	eval ${2}=${sizeF[4]}
517
518	IGCM_debug_PopStack "IGCM_sys_FileSize"
519	}
520
521	#D-#==================================================
522	#D-function IGCM_sys_TestDir
523	#D-* Purpose: Test Directory that must exists
524	#D-* Examples:
525	#D-
526	function IGCM_sys_TestDir {
527	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
528	if ( $DEBUG_sys ) ; then
529	echo "IGCM_sys_TestDir :" $@
530	fi
531	typeset ExistFlag
532	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
533	IGCM_debug_PopStack "IGCM_sys_TestDir"
534
535	return ${ExistFlag}
536	}
537
538	#D-#==================================================
539	#D-function IGCM_sys_TestDirArchive
540	#D-* Purpose: Test Directory that must exists on Archive
541	#D-* Examples:
542	#D-
543	function IGCM_sys_TestDirArchive {
544	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
545	if ( $DEBUG_sys ) ; then
546	echo "IGCM_sys_TestDirArchive :" $@
547	fi
548	typeset ExistFlag
549	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
550	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
551
552	return ${ExistFlag}
553	}
554
555	#D-#==================================================
556	#D-function IGCM_sys_IsFileArchived
557	#D-* Purpose: Test file that must NOT EXISTS on Archive
558	#D-* Examples:
559	#D-
560	function IGCM_sys_IsFileArchived {
561	IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
562	if ( $DEBUG_sys ) ; then
563	echo "IGCM_sys_IsFileArchived :" $@
564	fi
565	typeset IsArchivedFlag
566	IsArchivedFlag=$( [ X$( echo $1 \| grep ^\/ccc\/store ) != X ] && echo 0 \|\| echo 1 )
567	IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
568
569	return ${IsArchivedFlag}
570	}
571
572	#D-#==================================================
573	#D-function IGCM_sys_TestFileArchive
574	#D-* Purpose: Test file that must NOT EXISTS on Archive
575	#D-* Examples:
576	#D-
577	function IGCM_sys_TestFileArchive {
578	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
579	typeset ExistFlag
580	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
581	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
582
583	return ${ExistFlag}
584	}
585
586	#D-#==================================================
587	#D-function IGCM_sys_TestFileBuffer
588	#D-* Purpose: Test file that must NOT EXISTS on Buffer
589	#D-* Examples:
590	#D-
591	function IGCM_sys_TestFileBuffer {
592	IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
593	typeset ExistFlag
594	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
595	IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
596
597	return ${ExistFlag}
598	}
599
600	#D-#==================================================
601	#D-function IGCM_sys_CountFileArchive
602	#D-* Purpose: Count files on Archive filesystem
603	#D-* Examples:
604	#D-
605	function IGCM_sys_CountFileArchive {
606	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
607	ls ${@} 2>/dev/null \| wc -l
608	if [ $? -gt 0 ] ; then
609	echo "IGCM_sys_CountFileArchive : erreur."
610	fi
611	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
612	}
613
614	#D-#==================================================
615	#D-function IGCM_sys_CountFileBuffer
616	#D-* Purpose: Count files on Scratch filesystem
617	#D-* Examples:
618	#D-
619	function IGCM_sys_CountFileBuffer {
620	IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
621	ls ${@} 2>/dev/null \| wc -l
622	if [ $? -gt 0 ] ; then
623	echo "IGCM_sys_CountFileBuffer : erreur."
624	fi
625	IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
626	}
627
628	#D-#==================================================
629	#D-function IGCM_sys_Tree
630	#D-* Purpose: Tree directories with files on ${ARCHIVE}
631	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
632	#D-
633	function IGCM_sys_Tree {
634	IGCM_debug_PushStack "IGCM_sys_Tree" $@
635	if ( $DEBUG_sys ) ; then
636	echo "IGCM_sys_Tree :" $@
637	fi
638
639	\ls -lR ${@}
640
641	IGCM_debug_PopStack "IGCM_sys_Tree"
642	}
643
644	#D-#==================================================
645	#D-function IGCM_sys_Tar
646	#D-* Purpose: master tar command
647	#D-* Examples:
648	#D-
649	function IGCM_sys_Tar {
650	IGCM_debug_PushStack "IGCM_sys_Tar" $@
651	if ( $DEBUG_sys ) ; then
652	echo "IGCM_sys_Tar :" $@
653	fi
654	\tar cf $@
655	if [ $? -gt 0 ] ; then
656	echo "IGCM_sys_Tar : erreur."
657	IGCM_debug_Exit "IGCM_sys_Tar"
658	fi
659	IGCM_debug_PopStack "IGCM_sys_Tar"
660	}
661
662	#D-#==================================================
663	#D-function IGCM_sys_UnTar
664	#D-* Purpose: master un-tar command
665	#D-* Examples:
666	#D-
667	function IGCM_sys_UnTar {
668	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
669	if ( $DEBUG_sys ) ; then
670	echo "IGCM_sys_UnTar :" $@
671	fi
672	\tar xvf $1
673	if [ $? -gt 0 ] ; then
674	echo "IGCM_sys_UnTar : erreur."
675	IGCM_debug_Exit "IGCM_sys_UnTar"
676	fi
677	IGCM_debug_PopStack "IGCM_sys_UnTar"
678	}
679
680	#D-#==================================================
681	#D-function IGCM_sys_Qsub
682	#D-* Purpose: Qsub new job
683	#D-* Examples:
684	#D-
685	function IGCM_sys_Qsub {
686	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
687	if ( $DEBUG_sys ) ; then
688	echo "IGCM_sys_Qsub :" $@
689	fi
690	typeset options
691	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
692	options=" -o ${Script_Output} -e ${Script_Output}"
693	else
694	options=" -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod}"
695	fi
696
697	/usr/local/bin/ccc_msub ${options} < $1
698	if [ $? -gt 0 ] ; then
699	echo "IGCM_sys_Qsub : erreur ${options} $1"
700	IGCM_debug_Exit "IGCM_sys_Qsub"
701	fi
702	IGCM_debug_PopStack "IGCM_sys_Qsub"
703	}
704
705	#D-#==================================================
706	#D-function IGCM_sys_QsubPost
707	#D-* Purpose: Qsub new job on scalaire
708	#D-* Examples:
709	#D-
710	function IGCM_sys_QsubPost {
711	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
712	if ( $DEBUG_sys ) ; then
713	echo "IGCM_sys_QsubPost :" $@
714	fi
715
716	# Need to export listVarEnv for cesium
717	# NO Need to export listVarEnv for titane
718	# ============ FRONTEND START ============ #
719
720	/usr/local/bin/ccc_msub -p ${BRIDGE_MSUB_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
721
722	# ============ FRONTEND END ============ #
723
724	# ============ CESIUM START ============ #
725	# typeset NB_ESSAI DELAI status i
726	# # number of tentative
727	# NB_ESSAI=10
728	# # time delay between tentative
729	# DELAI=10
730	# (( i = 0 ))
731	# while [ $i -lt $NB_ESSAI ] ; do
732	# /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
733	# status=$?
734	# if [ ${status} -gt 0 ]; then
735	# sleep $DELAI
736	# else
737	# break
738	# fi
739	# (( i = i + 1 ))
740	# done
741	# ============ CESIUM END ============ #
742
743	if [ $? -gt 0 ] ; then
744	echo "IGCM_sys_QsubPost : erreur " $@
745	IGCM_debug_Exit "IGCM_sys_QsubPost"
746	fi
747	IGCM_debug_PopStack "IGCM_sys_QsubPost"
748	}
749
750	#D-*************************
751	#D- File transfer functions
752	#D-*************************
753	#D-
754
755	#D-#==================================================
756	#D-function IGCM_sys_Rsync_out
757	#D-* Purpose: treat return val of rsync
758	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
759	#D- Error values and explanations can depend on your system version.
760	function IGCM_sys_Rsync_out {
761	status=$1
762	if [ ! $status ] ; then
763	echo "rsync error !"
764	fi
765
766	if [ $MYLANG = "fr" ]; then
767	case $status in
768	0) return ;;
769	1) echo "Erreur de rsync ; RERR_SYNTAX : "
770	echo "Erreur de syntaxe ou d'utilisation."
771	return;;
772	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
773	echo "Incompatibilité de protocole."
774	return;;
775	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
776	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
777	echo "répertoires"
778	return;;
779	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
780	echo "Action demandée non supportée : une tentative de manipulation de"
781	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
782	echo "été faite ; ou une option qui est supportée par le client mais"
783	echo "pas par le serveur a été spécifiée."
784	return;;
785	10) echo "Erreur de rsync ; RERR_SOCKETIO"
786	echo "Erreur dans le socket d'entrée sortie"
787	return;;
788	11) echo "Erreur de rsync ; RERR_FILEIO"
789	echo "Erreur d'entrée sortie fichier"
790	return;;
791	12) echo "Erreur de rsync ; RERR_STREAMIO"
792	echo "Erreur dans flux de donnée du protocole rsync"
793	return;;
794	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
795	echo "Erreur avec les diagnostics du programme"
796	return;;
797	14) echo "Erreur de rsync ; RERR_IPC"
798	echo "Erreur dans le code IPC"
799	return;;
800	20) echo "Erreur de rsync ; RERR_SIGNAL"
801	echo "SIGUSR1 ou SIGINT reçu"
802	return;;
803	21) echo "Erreur de rsync ; RERR_WAITCHILD"
804	echo "Une erreur retournée par waitpid()"
805	return;;
806	22) echo "Erreur de rsync ; RERR_MALLOC"
807	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
808	return;;
809	23) echo ""
810	echo "Erreur fichier inexistant"
811	return;;
812	30) echo "Erreur de rsync ; RERR_TIMEOUT"
813	echo "Temps d'attente écoulé dans l'envoi/réception de données"
814	return;;
815	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $status
816	return;;
817	esac
818	elif [ $MYLANG = "en" ] ; then
819	case $status in
820	0) return;;
821	1) echo "rsync error : Syntax or usage error "
822	return;;
823	2) echo "rsync error : Protocol incompatibility "
824	return;;
825	3) echo "rsync error : Errors selecting input/output files, dirs"
826	return;;
827	4) echo "rsync error : Requested action not supported: an attempt"
828	echo "was made to manipulate 64-bit files on a platform that cannot support"
829	echo "them; or an option was specified that is supported by the client and"
830	echo "not by the server."
831	return;;
832	5) echo "rsync error : Error starting client-server protocol"
833	return;;
834	10) echo "rsync error : Error in socket I/O "
835	return;;
836	11) echo "rsync error : Error in file I/O "
837	return;;
838	12) echo "rsync error : Error in rsync protocol data stream "
839	return;;
840	13) echo "rsync error : Errors with program diagnostics "
841	return;;
842	14) echo "rsync error : Error in IPC code "
843	return;;
844	20) echo "rsync error : Received SIGUSR1 or SIGINT "
845	return;;
846	21) echo "rsync error : Some error returned by waitpid() "
847	return;;
848	22) echo "rsync error : Error allocating core memory buffers "
849	return;;
850	23) echo "rsync error : Partial transfer due to error"
851	return;;
852	24) echo "rsync error : Partial transfer due to vanished source files"
853	return;;
854	30) echo "rsync error : Timeout in data send/receive "
855	return;;
856	*) echo "rsync error : return code of rsync unknown :" $status
857	return;;
858	esac
859	else
860	echo "unknown language $MYLANG."
861	return
862	fi
863	}
864
865	#D-#==================================================
866	#D-function IGCM_sys_Miror_libIGCM
867	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
868	#D-* Examples:
869	#D-
870	function IGCM_sys_Mirror_libIGCM {
871	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
872	if ( $DEBUG_sys ) ; then
873	echo "IGCM_sys_Mirror_libIGCM"
874	fi
875
876	typeset status DEST
877
878	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
879
880	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
881	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
882	status=$?
883
884	if [ ${status} -gt 0 ] ; then
885	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
886	cat out_rsync
887	fi
888	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
889	}
890
891	#D-#==================================================
892	#D-function IGCM_sys_Cp
893	#D-* Purpose: generic cp
894	#D-* Examples:
895	#D-
896	function IGCM_sys_Cp {
897	IGCM_debug_PushStack "IGCM_sys_Cp" $@
898	if ( $DEBUG_sys ) ; then
899	echo "IGCM_sys_Cp :" $@
900	fi
901
902	typeset status
903
904	echo cp $@ > out_rsync 2>&1
905	\cp $@ >> out_rsync 2>&1
906	status=$?
907
908	if [ ${status} -gt 0 ] ; then
909	echo "IGCM_sys_Cp : error code ${status}"
910	cat out_rsync
911	IGCM_debug_Exit "IGCM_sys_Cp"
912	else
913	\rm out_rsync
914	fi
915	IGCM_debug_PopStack "IGCM_sys_Cp"
916	}
917
918	#D-#==================================================
919	#D-function IGCM_sys_Rm
920	#D-* Purpose: generic rm
921	#D-* Examples:
922	#D-
923	function IGCM_sys_Rm {
924	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
925	if ( $DEBUG_sys ) ; then
926	echo "IGCM_sys_Rm :" $@
927	fi
928
929	typeset status
930
931	echo rm $@ > out_rsync 2>&1
932	\rm $@ >> out_rsync 2>&1
933	status=$?
934
935	if [ ${status} -gt 0 ] ; then
936	echo "IGCM_sys_Rm : error code ${status}"
937	cat out_rsync
938	IGCM_debug_Exit "IGCM_sys_Rm"
939	else
940	\rm out_rsync
941	fi
942	IGCM_debug_PopStack "IGCM_sys_Rm"
943	}
944
945	#D-#==================================================
946	#D-function IGCM_sys_RmRunDir
947	#D-* Purpose: rm tmpdir (dummy function most of the time batch
948	#D- scheduler will do the job)
949	#D-* Examples:
950	#D-
951	function IGCM_sys_RmRunDir {
952	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
953	if ( $DEBUG_sys ) ; then
954	echo "IGCM_sys_RmRunDir :" $@
955	fi
956
957	typeset status
958
959	echo rm $@ > out_rsync 2>&1
960	\rm $@ >> out_rsync 2>&1
961	status=$?
962
963	if [ ${status} -gt 0 ] ; then
964	echo "IGCM_sys_RmRunDir : error code ${status}"
965	cat out_rsync
966	IGCM_debug_Exit "IGCM_sys_RmRunDir"
967	else
968	\rm out_rsync
969	fi
970	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
971	}
972
973	#D-#==================================================
974	#D-function IGCM_sys_Mv
975	#D-* Purpose: generic move
976	#D-* Examples:
977	#D-
978	function IGCM_sys_Mv {
979	IGCM_debug_PushStack "IGCM_sys_Mv" $@
980	if ( $DEBUG_sys ) ; then
981	echo "IGCM_sys_Mv :" $@
982	fi
983
984	if [ $DRYRUN = 0 ]; then
985
986	typeset status
987
988	echo mv $@ > out_rsync 2>&1
989	\mv $@ >> out_rsync 2>&1
990	status=$?
991
992	if [ ${status} -gt 0 ] ; then
993	echo "IGCM_sys_Mv : error code ${status}"
994	cat out_rsync
995	IGCM_debug_Exit "IGCM_sys_Mv"
996	else
997	\rm out_rsync
998	fi
999	else
1000	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1001	fi
1002
1003	IGCM_debug_PopStack "IGCM_sys_Mv"
1004	}
1005
1006	#D-#==================================================
1007	#D-function IGCM_sys_Put_Dir
1008	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
1009	#D-* Examples:
1010	#D-
1011	function IGCM_sys_Put_Dir {
1012	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
1013	if ( $DEBUG_sys ) ; then
1014	echo "IGCM_sys_Put_Dir :" $@
1015	fi
1016	if [ $DRYRUN = 0 ]; then
1017	if [ ! -d ${1} ] ; then
1018	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
1019	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1020	return
1021	fi
1022
1023	typeset status
1024
1025	# Only if we use rsync
1026	#IGCM_sys_TestDirArchive $( dirname $2 )
1027	#
1028	#USUAL WAY
1029	\cp -r $1 $2 > out_rsync 2>&1
1030	status=$?
1031
1032	if [ ${status} -gt 0 ] ; then
1033	echo "IGCM_sys_Put_Dir : error code ${status}"
1034	cat out_rsync
1035	IGCM_debug_Exit "IGCM_sys_Put_Dir"
1036	else
1037	\rm out_rsync
1038	fi
1039	else
1040	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1041	fi
1042	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1043	}
1044
1045	#D-#==================================================
1046	#D-function IGCM_sys_Get_Dir
1047	#D-* Purpose: Copy a complete directory from ${ARCHIVE}
1048	#D-* Examples:
1049	#D-
1050	function IGCM_sys_Get_Dir {
1051	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1052	if ( $DEBUG_sys ) ; then
1053	echo "IGCM_sys_Get_Dir :" $@
1054	fi
1055	if [ $DRYRUN = 0 ]; then
1056	# if [ ! -d ${1} ] ; then
1057	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1058	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1059	# return
1060	# fi
1061
1062	typeset NB_ESSAI DELAI status i
1063	# number of tentative
1064	NB_ESSAI=3
1065	# time delay between tentative
1066	DELAI=2
1067
1068	# Only if we use rsync
1069	#IGCM_sys_TestDirArchive $( dirname $2 )
1070	#
1071	# USUAL WAY
1072	# add dmfind/dmget (to demigrate all offline files) :
1073	#dmfind $1 -state MIG -o -state OFL -o -state PAR \| dmget
1074	ccc_hsm get -r $1
1075
1076	i=0
1077	while [ $i -lt $NB_ESSAI ] ; do
1078	\cp -ur $1 $2 >> out_rsync 2>&1
1079	status=$?
1080	if [ ${status} -gt 0 ]; then
1081	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
1082	IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1083	sleep $DELAI
1084	else
1085	break
1086	fi
1087	(( i = i + 1 ))
1088	done
1089
1090	if [ ${status} -gt 0 ] ; then
1091	echo "IGCM_sys_Get_Dir : error."
1092	cat out_rsync
1093	IGCM_debug_Exit "IGCM_sys_Get_Dir"
1094	else
1095	\rm out_rsync
1096	fi
1097	else
1098	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1099	fi
1100	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1101	}
1102
1103	#D-#==================================================
1104	#D-function IGCM_sys_Get_Master
1105	#D-* Purpose: Copy a complete directory from MASTER filesystem
1106	#D-* Examples:
1107	#D-
1108	function IGCM_sys_Get_Master {
1109	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1110	if ( $DEBUG_sys ) ; then
1111	echo "IGCM_sys_Get_Master :" $@
1112	fi
1113	if [ $DRYRUN = 0 ]; then
1114	if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
1115	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1116	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1117	return
1118	fi
1119
1120	typeset NB_ESSAI DELAI status i
1121	# number of tentative
1122	NB_ESSAI=3
1123	# time delay between tentative
1124	DELAI=2
1125
1126	i=0
1127	while [ $i -lt $NB_ESSAI ] ; do
1128	\cp -urL $1 $2 > out_rsync 2>&1
1129	status=$?
1130	if [ ${status} -gt 0 ]; then
1131	IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
1132	IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1133	sleep $DELAI
1134	else
1135	break
1136	fi
1137	(( i = i + 1 ))
1138	done
1139
1140	if [ ${status} -gt 0 ] ; then
1141	echo "IGCM_sys_Get_Master : error."
1142	cat out_rsync
1143	IGCM_debug_Exit "IGCM_sys_Get_Master"
1144	else
1145	\rm out_rsync
1146	fi
1147	else
1148	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1149	fi
1150	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1151	}
1152
1153	#====================================================
1154	#- Call IGCM_sys_Mirror_libIGCM now !
1155	if ( $MirrorlibIGCM ) ; then
1156	IGCM_sys_Mirror_libIGCM
1157	fi
1158
1159	#D-#==================================================
1160	#D-function IGCM_sys_Put_Rest
1161	#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1162	#D- File and target directory must exist.
1163	#D-* Examples:
1164	#D-
1165	function IGCM_sys_Put_Rest {
1166	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1167	if ( $DEBUG_sys ) ; then
1168	echo "IGCM_sys_Put_Rest :" $@
1169	fi
1170	if [ $DRYRUN = 0 ]; then
1171	if [ ! -f ${1} ] ; then
1172	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1173	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1174	fi
1175
1176	typeset status
1177	#
1178	if [ X${JobType} = XRUN ] ; then
1179	IGCM_sys_Chmod 444 ${1}
1180	fi
1181
1182	#
1183	# USUAL WAY
1184	\cp $1 $2 > out_rsync 2>&1
1185	status=$?
1186
1187	# #RSYNC WITH NETWORK SSH CALL
1188	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1189	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1190
1191	# #RSYNC WITH NFS USE
1192	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1193	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1194
1195	# status=$?
1196	# IGCM_sys_Rsync_out $status
1197
1198	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1199	# (( status=status+$? ))
1200
1201	if [ ${status} -gt 0 ] ; then
1202	echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1203	cat out_rsync
1204	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1205	else
1206	\rm out_rsync
1207	fi
1208	else
1209	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1210	fi
1211	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1212	}
1213
1214	#D-#==================================================
1215	#D-function IGCM_sys_PutBuffer_Rest
1216	#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1217	#D- File and target directory must exist.
1218	#D-* Examples:
1219	#D-
1220	function IGCM_sys_PutBuffer_Rest {
1221	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1222	if ( $DEBUG_sys ) ; then
1223	echo "IGCM_sys_PutBuffer_Rest :" $@
1224	fi
1225	if [ $DRYRUN = 0 ]; then
1226	if [ ! -f ${1} ] ; then
1227	echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1228	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1229	fi
1230
1231	typeset status
1232	#
1233	if [ X${JobType} = XRUN ] ; then
1234	IGCM_sys_Chmod 444 ${1}
1235	fi
1236
1237	#
1238	# USUAL WAY
1239	\cp $1 $2 > out_rsync 2>&1
1240	status=$?
1241
1242	if [ ${status} -gt 0 ] ; then
1243	echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1244	cat out_rsync
1245	IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1246	else
1247	\rm out_rsync
1248	fi
1249	else
1250	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1251	fi
1252	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1253	}
1254
1255	#D-#==================================================
1256	#D-function IGCM_sys_Put_Out
1257	#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1258	#D-* Examples:
1259	#D-
1260	function IGCM_sys_Put_Out {
1261	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1262	if ( $DEBUG_sys ) ; then
1263	echo "IGCM_sys_Put_Out :" $@
1264	fi
1265
1266	typeset NB_ESSAI DELAI status i exist skip
1267
1268	# number of tentative
1269	NB_ESSAI=3
1270	# time delay between tentative
1271	DELAI=2
1272
1273	if [ $DRYRUN = 0 ]; then
1274	if [ ! -f ${1} ] ; then
1275	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1276	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1277	return 1
1278	fi
1279	#
1280	IGCM_sys_MkdirArchive $( dirname $2 )
1281	#
1282
1283	#=====================================================
1284	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1285	#=====================================================
1286
1287	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1288	#if [ $? -eq 0 ] ; then
1289	# typeset WORKPATH FILEPATH
1290	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_BUF}\|" )
1291	# IGCM_sys_MkdirWork ${WORKPATH}
1292	# FILEPATH=${WORKPATH}/$( basename $2 )
1293	# #
1294	# IGCM_sys_Cp ${1} ${FILEPATH}
1295	#fi
1296
1297	if [ X${JobType} = XRUN ] ; then
1298	if [ X${3} = X ] ; then
1299	IGCM_sys_Chmod 444 ${1}
1300	fi
1301	fi
1302	#
1303
1304	exist=false
1305	skip=false
1306	if [ -f $2 ] ; then
1307	IGCM_debug_Print 1 "$2 already exist"
1308	#dmget $2
1309	ccc_hsm get $2
1310	exist=true
1311	if [ "X$( diff $1 $2 )" = X ] ; then
1312	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1313	skip=true
1314	else
1315	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1316	skip=false
1317	fi
1318	fi
1319	#
1320	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1321	IGCM_sys_Chmod u+w $2
1322	fi
1323
1324	if [ X${skip} = Xfalse ] ; then
1325	i=0
1326	while [ $i -lt $NB_ESSAI ] ; do
1327	# USUAL WAY
1328	\cp $1 $2 > out_rsync 2>&1
1329	status=$?
1330	if [ ${status} -gt 0 ]; then
1331	IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1332	IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1333	if [ -f $2 ] ; then
1334	if [ "X$( diff $1 $2 )" = X ] ; then
1335	IGCM_debug_Print 2 "$1 and $2 are the same file"
1336	IGCM_debug_Print 2 "Cp gave an error but the file has been copied properly!"
1337	break
1338	else
1339	IGCM_debug_Print 2 "$1 and $2 are not the same file, we continue to try"
1340	fi
1341	fi
1342	sleep $DELAI
1343	else
1344	break
1345	fi
1346	(( i = i + 1 ))
1347	done
1348	fi
1349
1350	# #RSYNC WITH NETWORK SSH CALL
1351	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1352	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1353
1354	# #RSYNC WITH NFS USE
1355	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1356	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1357
1358	# status=$?
1359	# IGCM_sys_Rsync_out $status
1360
1361	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1362	# (( status=status+$? ))
1363
1364	if [ ${status} -gt 0 ] ; then
1365	echo "IGCM_sys_Put_Out : error."
1366	cat out_rsync
1367	IGCM_debug_Exit "IGCM_sys_Put_Out"
1368	else
1369	\rm out_rsync
1370	fi
1371	else
1372	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1373	fi
1374	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1375	return 0
1376	}
1377
1378	#D-#==================================================
1379	#D-function IGCM_sys_PutBuffer_Out
1380	#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1381	#D-* Examples:
1382	#D-
1383	function IGCM_sys_PutBuffer_Out {
1384	IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1385	if ( $DEBUG_sys ) ; then
1386	echo "IGCM_sys_PutBuffer_Out :" $@
1387	fi
1388
1389	typeset NB_ESSAI DELAI status i exist skip
1390
1391	# number of tentative
1392	NB_ESSAI=3
1393	# time delay between tentative
1394	DELAI=2
1395
1396	if [ $DRYRUN = 0 ]; then
1397	if [ ! -f ${1} ] ; then
1398	echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1399	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1400	return 1
1401	fi
1402	#
1403	IGCM_sys_Mkdir $( dirname $2 )
1404	#
1405
1406	if [ X${JobType} = XRUN ] ; then
1407	if [ X${3} = X ] ; then
1408	IGCM_sys_Chmod 444 ${1}
1409	fi
1410	fi
1411
1412	exist=false
1413	skip=false
1414	if [ -f $2 ] ; then
1415	IGCM_debug_Print 1 "$2 already exist"
1416	exist=true
1417	if [ "X$( diff $1 $2 )" = X ] ; then
1418	IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1419	skip=true
1420	else
1421	IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1422	skip=false
1423	fi
1424	fi
1425	#
1426	if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1427	IGCM_sys_Chmod u+w $2
1428	fi
1429
1430	if [ X${skip} = Xfalse ] ; then
1431	i=0
1432	while [ $i -lt $NB_ESSAI ] ; do
1433	# USUAL WAY
1434	\cp $1 $2 > out_rsync 2>&1
1435	status=$?
1436	if [ ${status} -gt 0 ]; then
1437	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1438	IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1439	sleep $DELAI
1440	else
1441	break
1442	fi
1443	(( i = i + 1 ))
1444	done
1445	fi
1446
1447	if [ ${status} -gt 0 ] ; then
1448	echo "IGCM_sys_PutBuffer_Out : error."
1449	cat out_rsync
1450	IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1451	else
1452	\rm out_rsync
1453	fi
1454	else
1455	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1456	fi
1457	IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1458	return 0
1459	}
1460
1461	#D-#==================================================
1462	#D-function IGCM_sys_Get
1463	#D-* Purpose: Get a file from ${ARCHIVE}
1464	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1465	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1466	function IGCM_sys_Get {
1467	IGCM_debug_PushStack "IGCM_sys_Get" $@
1468
1469	typeset DEST dm_liste ifile target file_work
1470	typeset NB_ESSAI DELAI status i
1471
1472	if ( $DEBUG_sys ) ; then
1473	echo "IGCM_sys_Get :" $@
1474	fi
1475
1476	# number of tentative
1477	NB_ESSAI=3
1478	# time delay between tentative
1479	DELAI=2
1480
1481	if [ $DRYRUN -le 2 ]; then
1482	if [ X${1} = X'/l' ] ; then
1483	# test if the first file is present in the old computation :
1484	eval set +A dm_liste \${${2}}
1485	else
1486	eval set +A dm_liste ${1}
1487	fi
1488	eval DEST=\${${#}}
1489
1490	#=====================================================
1491	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1492	#=====================================================
1493
1494	# Is it an R_OUT file (not R_IN) ?
1495	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1496	#if [ $? -eq 0 ] ; then
1497	# # Yes ? then we try to get it in SCRATCHDIR
1498	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_BUF}\|g" )
1499	# if [ -f ${file_work[0]} ] ; then
1500	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1501	# IGCM_debug_PopStack "IGCM_sys_Get"
1502	# return
1503	# fi
1504	#fi
1505
1506	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1507	status=$?
1508	if [ ${status} -gt 0 ] ; then
1509	echo "WARNING IGCM_sys_Get : error code ${status}"
1510	cat out_rsync
1511	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1512	fi
1513
1514	#if [ ${status} -gt 0 ] ; then
1515	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1516	# cat out_rsync
1517	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1518	# sleep 30
1519	# echo "We try another time"
1520	## dmget ${dm_liste[*]} > out_rsync 2>&1
1521	# ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1522	# status=$?
1523	# if [ ${status} -gt 0 ] ; then
1524	# echo "ERROR IGCM_sys_Get : again demigration error :"
1525	# cat out_rsync
1526	# IGCM_debug_Exit "IGCM_sys_Get"
1527	# fi
1528	# else
1529	# echo "ERROR IGCM_sys_Get : demigration error :"
1530	# cat out_rsync
1531	# IGCM_debug_Exit "IGCM_sys_Get"
1532	# fi
1533	#fi
1534
1535	# #RSYNC WITH NETWORK SSH CALL
1536	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1537	# ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1538
1539	# #RSYNC WITH NFS USE
1540	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1541	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1542
1543	# status=$?
1544	# IGCM_sys_Rsync_out $status
1545
1546	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1547	# (( status=status+$? ))
1548
1549	#USUAL WAY
1550	if [ X${1} = X'/l' ] ; then
1551	for target in ${dm_liste[*]} ; do
1552	local_file=$( basename ${target} )
1553	# test if the target file is present before the loop
1554	IGCM_sys_TestFileArchive ${target}
1555	status=$?
1556	if [ ${status} -gt 0 ] ; then
1557	echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
1558	IGCM_debug_Exit "IGCM_sys_Get"
1559	else
1560	i=0
1561	while [ $i -lt $NB_ESSAI ] ; do
1562	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1563	status=$?
1564	if [ ${status} -gt 0 ]; then
1565	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1566	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1567	sleep $DELAI
1568	else
1569	break
1570	fi
1571	(( i = i + 1 ))
1572	done
1573	if [ ${status} -gt 0 ] ; then
1574	echo "IGCM_sys_Get : error"
1575	cat out_rsync
1576	\rm out_rsync
1577	IGCM_debug_Exit "IGCM_sys_Get"
1578	else
1579	\rm out_rsync
1580	fi
1581	fi
1582	done
1583	else
1584	i=0
1585	while [ $i -lt $NB_ESSAI ] ; do
1586	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1587	status=$?
1588	if [ ${status} -gt 0 ]; then
1589	IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1590	IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1591	sleep $DELAI
1592	else
1593	break
1594	fi
1595	(( i = i + 1 ))
1596	done
1597	if [ ${status} -gt 0 ] ; then
1598	echo "IGCM_sys_Get : error"
1599	cat out_rsync
1600	\rm out_rsync
1601	IGCM_debug_Exit "IGCM_sys_Get"
1602	else
1603	\rm out_rsync
1604	fi
1605	fi
1606	else
1607	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1608	fi
1609	IGCM_debug_PopStack "IGCM_sys_Get"
1610	}
1611
1612	#D-#==================================================
1613	#D-function IGCM_sys_GetBuffer
1614	#D-* Purpose: Get a file from ${SCRATCHDIR}
1615	#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1616	#D- IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1617	function IGCM_sys_GetBuffer {
1618	IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1619
1620	typeset DEST buf_liste ifile target file_work
1621	typeset NB_ESSAI DELAI status i
1622
1623	if ( $DEBUG_sys ) ; then
1624	echo "IGCM_sys_GetBuffer :" $@
1625	fi
1626
1627	# number of tentative
1628	NB_ESSAI=3
1629	# time delay between tentative
1630	DELAI=2
1631
1632	if [ $DRYRUN -le 2 ]; then
1633	if [ X${1} = X'/l' ] ; then
1634	# test if the first file is present in the old computation :
1635	eval set +A buf_liste \${${2}}
1636	else
1637	eval set +A buf_liste ${1}
1638	fi
1639	eval DEST=\${${#}}
1640
1641	#USUAL WAY
1642	if [ X${1} = X'/l' ] ; then
1643	for target in ${buf_liste[*]} ; do
1644	local_file=$( basename ${target} )
1645	i=0
1646	while [ $i -lt $NB_ESSAI ] ; do
1647	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1648	status=$?
1649	if [ ${status} -gt 0 ]; then
1650	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1651	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1652	sleep $DELAI
1653	else
1654	break
1655	fi
1656	(( i = i + 1 ))
1657	done
1658	if [ ${status} -gt 0 ] ; then
1659	echo "IGCM_sys_Get : error"
1660	cat out_rsync
1661	\rm out_rsync
1662	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1663	else
1664	\rm out_rsync
1665	fi
1666	done
1667	else
1668	i=0
1669	while [ $i -lt $NB_ESSAI ] ; do
1670	\cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1671	status=$?
1672	if [ ${status} -gt 0 ]; then
1673	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1674	IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1675	sleep $DELAI
1676	else
1677	break
1678	fi
1679	(( i = i + 1 ))
1680	done
1681	if [ ${status} -gt 0 ] ; then
1682	echo "IGCM_sys_Get : error"
1683	cat out_rsync
1684	\rm out_rsync
1685	IGCM_debug_Exit "IGCM_sys_GetBuffer"
1686	else
1687	\rm out_rsync
1688	fi
1689	fi
1690	else
1691	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1692	fi
1693	IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1694	}
1695
1696	#D-#==================================================
1697	#D-function IGCM_sys_GetDate_FichWork
1698	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1699	#D-* Examples:
1700	#D-
1701	function IGCM_sys_GetDate_FichWork {
1702	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1703	if ( $DEBUG_sys ) ; then
1704	echo "IGCM_sys_GetDate_FichWork :" $@
1705	fi
1706	typeset dateF
1707	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1708	eval ${2}=${dateF[5]}
1709
1710	# donne la date filesys d'un fichier sur la machine work
1711	IGCM_debug_PopStack "IGCM_sys_FichWork"
1712	}
1713
1714	#D-#==================================================
1715	#D-function IGCM_sys_GetDate_FichArchive
1716	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1717	#D-* Examples:
1718	#D-
1719	function IGCM_sys_GetDate_FichArchive {
1720	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1721	if ( $DEBUG_sys ) ; then
1722	echo "IGCM_sys_GetDate_FichArchive :" $@
1723	fi
1724	typeset dateF
1725	set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1726	eval ${2}=${dateF[5]}
1727
1728	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1729	}
1730
1731	#D-#==================================================
1732	#D-function IGCM_sys_Dods_Rm
1733	#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1734	#D-* Examples:
1735	#D-
1736	function IGCM_sys_Dods_Rm {
1737	if ( $DEBUG_sys ) ; then
1738	echo "IGCM_sys_Dods_Rm :" $@
1739	fi
1740	typeset status
1741	if [ $DRYRUN = 0 ]; then
1742
1743	# if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1744	# echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1745	# echo "Nothing has been done."
1746	# return
1747	# fi
1748
1749	/ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1750	status=$?
1751
1752	# if [ ${status} -gt 0 ] ; then
1753	# echo "IGCM_sys_Dods_Rm : error."
1754	# cat out_dods_rm
1755	# IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1756	# else
1757	# rm out_dods_rm
1758	# fi
1759
1760	else
1761	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1762	fi
1763	return $status
1764	}
1765
1766	#D-#==================================================
1767	#D-function IGCM_sys_Dods_Cp
1768	#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1769	#D-* Examples:
1770	#D-
1771	function IGCM_sys_Dods_Cp {
1772	if ( $DEBUG_sys ) ; then
1773	echo "IGCM_sys_Dods_Cp :" $@
1774	fi
1775	typeset status
1776	if [ $DRYRUN = 0 ]; then
1777
1778	# if [ ! -d ${R_SAVE}/${1} ] ; then
1779	# echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1780	# echo "Nothing has been done."
1781	# return
1782	# fi
1783
1784	/ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1785	status=$?
1786
1787	# if [ ${status} -gt 0 ] ; then
1788	# echo "IGCM_sys_Dods_Cp : error."
1789	# cat out_dods_cp
1790	# IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1791	# else
1792	# rm out_dods_cp
1793	# fi
1794
1795	else
1796	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1797	fi
1798	return $status
1799	}
1800
1801	#D-#==================================================
1802	#D-function IGCM_sys_Put_Dods
1803	#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1804	#D-* Examples:
1805	#D-
1806	function IGCM_sys_Put_Dods {
1807	IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1808	if ( $DEBUG_sys ) ; then
1809	echo "IGCM_sys_Put_Dods :" $@
1810	fi
1811	#set -vx
1812	typeset status
1813	if [ $DRYRUN = 0 ]; then
1814	if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1815	echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1816	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1817	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1818	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1819	return
1820	fi
1821
1822	if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1823	echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1824	echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1825	echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1826	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1827	return
1828	fi
1829	#
1830	if [ -d ${R_SAVE}/${1} ] ; then
1831	cd ${R_SAVE}
1832	elif [ -d ${R_FIGR}/${1} ] ; then
1833	cd ${R_FIGR}
1834	fi
1835
1836	IGCM_sys_Dods_Rm ${1}
1837	IGCM_sys_Dods_Cp ${1}
1838	status=0
1839
1840	if [ ${status} -gt 0 ] ; then
1841	echo "IGCM_sys_Put_Dods : error."
1842	IGCM_debug_Exit "IGCM_sys_Put_Dods"
1843	fi
1844	else
1845	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1846	fi
1847	IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1848	}
1849
1850	##############################################################
1851	# REBUILD OPERATOR
1852
1853	function IGCM_sys_rebuild {
1854	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1855	if ( $DEBUG_sys ) ; then
1856	echo "IGCM_sys_rebuild :" $@
1857	fi
1858
1859	typeset NB_ESSAI DELAI status i lastArg
1860	# number of tentative
1861	NB_ESSAI=3
1862	# time delay between tentative
1863	DELAI=2
1864
1865	i=0
1866	while [ $i -lt $NB_ESSAI ] ; do
1867	/home/cont003/p86ipsl/X64/bin/rebuild -f -o $@ > out_rsync 2>&1
1868	status=$?
1869	if [ ${status} -gt 0 ] ; then
1870	IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1871	cat out_rsync
1872	\rm out_rsync
1873	IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1874	eval lastArg=\$$#
1875	\rm ${lastArg}
1876	else
1877	\rm out_rsync
1878	break
1879	fi
1880	(( i = i + 1 ))
1881	done
1882
1883	if [ ${status} -gt 0 ] ; then
1884	echo "IGCM_sys_rebuild : rebuild error"
1885	IGCM_debug_Exit "rebuild"
1886	fi
1887
1888	IGCM_debug_PopStack "IGCM_sys_rebuild"
1889	}
1890
1891	function IGCM_sys_rebuild_station {
1892	IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1893	typeset i list_opt file_in file_out prefix_invert list_invert
1894	if ( $DEBUG_sys ) ; then
1895	echo "IGCM_sys_rebuild_station :" $@
1896	fi
1897	list_opt=$@
1898
1899	# Invert Axis : t,x -> x,t
1900	# t,pres,x -> x,t,pres
1901	# So that we can concatenate along x
1902	i=0
1903	for file_in in ${list_opt} ; do
1904	(( i = i + 1))
1905	[ ${i} = 1 ] && file_out=${file_in} && continue
1906	prefix_invert=$( basename ${file_in} .nc )
1907	IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1908	list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1909	done
1910
1911	# Concatenate
1912	IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1913
1914	# Re-ivert file
1915	IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1916
1917	# Station re-ordering is too expansive to be run within libICGM
1918	# This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1919	# This re-ordering must be done "in memory" by the cmorization process
1920	# Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1921	# BEGIN reordering
1922
1923	# Only LMDZ text output contains the exact ordering of the station.
1924	# We isolate this in the code below:
1925	# 0 38 -157.5000000000000 70.98591549295774
1926	# 0 54 27.49999999999999 67.18309859154928
1927	# 0 56 -62.50000000000001 82.39436619718309
1928	# 0 79 12.49999999999999 78.59154929577466
1929	# 0 116 -165.0000000000000 76.05633802816901
1930	# 0 117 130.0000000000000 70.98591549295774
1931	# 0 118 110.0000000000000 87.46478873239437
1932	# 1 40 4.999999999999995 51.97183098591550
1933	# typeset iStation iProc list_opt file_in file_out prefix_invert
1934	# typeset -Z4 j4
1935	# typeset -Z3 j3
1936
1937	# unset list_opt
1938	# set +A list_opt $@
1939
1940	# Filename after rebuild
1941	# file_out=${list_opt[0]}
1942	# Prefix of output files
1943	# prefix_invert=$( basename ${file_out} .nc )
1944	# Number of procs
1945	# num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} \| wc -l )
1946
1947	# iProc=0
1948	# while [ ${iProc} -lt ${num_proc} ] ; do
1949	# Array containing Station as a number
1950	# unset proc_stn
1951	# set +A proc_stn $( grep "iophy_mpi rank ip lon lat $iProc" ${PREFIX}_${Exe_Output} \| sed -e "s/iophy_mpi rank ip lon lat //g" \| awk ' {print $2}' )
1952	# Number of stations produced by processor proc
1953	# stationLast=${#proc_stn[*]}
1954	# Proc number on 4 digits
1955	# j4=${iProc}
1956	# Init
1957	# iStation=0
1958	# while [ ${iStation} -lt ${stationLast} ] ; do
1959	# Station number on 3 digits
1960	# j3=${proc_stn[${iStation}]}
1961	# Extract station
1962	# Invert Axis : t,x -> x,t
1963	# t,pres,x -> x,t,pres
1964	# So that we can concatenate along x
1965	# IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1966	# (( iStation = iStation + 1 ))
1967	# done
1968	# (( iProc = iProc + 1 ))
1969	# done
1970
1971	# Concatenate all station along x
1972	# IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1973
1974	# Re-invert file
1975	# IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1976
1977	# END reordering
1978
1979	IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1980	}
1981
1982	############################################################
1983	# Activate Running Environnment Variables
1984
1985	function IGCM_sys_activ_variables {
1986	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1987	if ( $DEBUG_sys ) ; then
1988	echo "IGCM_sys_activ_variables"
1989	fi
1990
1991	ulimit -s unlimited
1992
1993	export KMP_STACKSIZE=3g
1994	export KMP_LIBRARY=turnaround
1995	export MKL_SERIAL=YES
1996	unset MPI_PLACING_MODE
1997
1998	# add module load nb of threads for core placement
1999	(( max_omp = 0 ))
2000
2001	for comp in ${config_ListOfComponents[*]} ; do
2002	eval ExeNameIn=\${config_Executable_${comp}[0]}
2003	eval ExeNameOut=\${config_Executable_${comp}[1]}
2004
2005	# Only if we really have an executable for the component :
2006	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2007	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2008	if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
2009	(( max_omp = comp_proc_omp_loc ))
2010	fi
2011	fi
2012	done
2013
2014	( [ X${max_omp} = X4 ] \|\| [ X${max_omp} = X8 ] ) && { module load openmp/${max_omp}thds ; IGCM_debug_Print 1 "module load openmp/${max_omp}thds" ; }
2015
2016	IGCM_debug_PopStack "IGCM_sys_activ_variables"
2017	}
2018
2019	############################################################
2020	# Desactivate Running Environnment Variables
2021
2022	function IGCM_sys_desactiv_variables {
2023	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
2024	if ( $DEBUG_sys ) ; then
2025	echo "IGCM_sys_desactiv_variables"
2026	fi
2027	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
2028	}
2029
2030	############################################################
2031	# Build MPI/OMP scripts run file (dummy function)
2032
2033	function IGCM_sys_build_run_file {
2034
2035	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
2036
2037	}
2038
2039	############################################################
2040	# Build MPI/OMP scripts
2041	function IGCM_sys_build_execution_scripts
2042	{
2043	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
2044	if ( $DEBUG_sys ) ; then
2045	echo "IGCM_sys_build_execution_scripts " $@
2046	fi
2047	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
2048	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
2049	typeset num_corempi nombre_restant_node nombre_restant_comp
2050	typeset IGCM_OMP
2051
2052	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
2053	IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
2054	fi
2055
2056	if ( ${OK_PARA_MPMD} ) ; then
2057
2058	if [ -f run_file ] ; then
2059	IGCM_sys_Rm -f run_file
2060	fi
2061	touch run_file
2062
2063	if ( ${OK_PARA_OMP} ) ; then
2064
2065	# Hosts treatment
2066
2067	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
2068
2069	i=0
2070	rm -f hosts
2071	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
2072	for nodes in `cat hosts.tmp` ; do
2073	host[$i]=$nodes
2074	echo "${host[$i]} slots=8 max_slots=8" >> hosts
2075	IGCM_debug_Print 1 ${host[$i]}
2076	i=$((i+1))
2077	done
2078	rm -f hosts.tmp
2079
2080	listnodes=${host[*]}
2081
2082	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
2083
2084	# Initialisation
2085
2086	init_node=y
2087	node_num_current=0
2088	start_num=0
2089	init_exec=n
2090
2091	# Test : if oasis is there, we put it at the first position
2092
2093	for comp in ${config_ListOfComponents[*]} ; do
2094
2095	if [ "X${comp}" = "XCPL" ] ; then
2096
2097	eval ExeNameIn=\${config_Executable_${comp}[0]}
2098	eval ExeNameOut=\${config_Executable_${comp}[1]}
2099
2100	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2101	echo "" >> script_${ExeNameOut}.ksh
2102	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2103	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2104	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2105	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2106	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2107
2108	init_node=n
2109
2110	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
2111	node_num_current=0
2112	node_current=${host[${node_num_current}]}
2113
2114	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
2115
2116	init_exec=y
2117	start_num=1
2118
2119	fi
2120
2121	done
2122
2123	# Then loop on the components (except for oasis)
2124
2125	for comp in ${config_ListOfComponents[*]} ; do
2126
2127	eval ExeNameIn=\${config_Executable_${comp}[0]}
2128	eval ExeNameOut=\${config_Executable_${comp}[1]}
2129
2130	# Only if we really have an executable for the component :
2131	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2132
2133	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2134	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2135
2136	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2137	# echo "set -vx" >> script_${ExeNameOut}.ksh
2138	echo "" >> script_${ExeNameOut}.ksh
2139	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
2140	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
2141	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
2142	echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2143	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
2144	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
2145	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2146
2147	node_num=0
2148
2149	# We define the number of MPI process to be assigned for the component
2150
2151	nombre_restant_comp=${comp_proc_mpi_loc}
2152
2153	# Loop on the allocated nodes
2154
2155	for node in ${listnodes} ; do
2156
2157	# We go to the current node
2158	if [ ${node_num} = ${node_num_current} ] ; then
2159
2160	node_current=${host[${node_num_current}]}
2161
2162	# If first time on the node : initialisation
2163
2164	if [ ${init_node} = y ] ; then
2165	nombre_restant_node=${NUM_COREPERNODE}
2166	fi
2167
2168	# Test on the number of OMP threads
2169
2170	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
2171	(( node_num = node_num + 1 ))
2172	node_num_current=${node_num}
2173	init_node=y
2174	continue
2175	fi
2176
2177	# Number of MPI process to assign
2178
2179	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
2180
2181	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
2182	num_corempi=${nombre_restant_comp}
2183	fi
2184
2185	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
2186	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
2187
2188	if [ ${init_exec} = y ] ; then
2189	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
2190	else
2191	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
2192	init_exec=y
2193	fi
2194
2195	(( start_num = num_corempi + start_num ))
2196
2197	else
2198
2199	(( node_num = node_num + 1 ))
2200	continue
2201	fi
2202
2203	# Test on the number of core/process remaining on the node/component
2204
2205	if [ ${nombre_restant_node} = 0 ] ; then
2206	(( node_num = node_num + 1 ))
2207	node_num_current=${node_num}
2208	init_node=y
2209
2210	if [ ${nombre_restant_comp} = 0 ] ; then
2211	break 1
2212	fi
2213	else
2214
2215	node_num_current=${node_num}
2216	init_node=n
2217
2218	if [ ${nombre_restant_comp} = 0 ] ; then
2219	break 1
2220	fi
2221	fi
2222	done
2223	fi
2224	done
2225
2226	else
2227
2228	# Then first loop on the components for the coupler ie oasis
2229
2230	## the coupler ie oasis must be the first one
2231	for comp in ${config_ListOfComponents[*]} ; do
2232
2233	eval ExeNameOut=\${config_Executable_${comp}[1]}
2234
2235	# for CPL component only
2236	if [ "X${comp}" = "XCPL" ] ; then
2237	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2238	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
2239	fi
2240	done
2241
2242	# Then second loop on the components
2243
2244	for comp in ${config_ListOfComponents[*]} ; do
2245
2246	eval ExeNameOut=\${config_Executable_${comp}[1]}
2247
2248	# Only if we really have an executable for the component and not the coupler ie oasis:
2249	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
2250	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2251	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2252	fi
2253	done
2254	IGCM_sys_Chmod u+x run_file
2255
2256	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
2257
2258	fi
2259
2260	else # Only one executable.
2261	# Check has to be added with :
2262	# Job :
2263	# #MSUB -n 8 # reservation des processeurs pour le job
2264	# #MSUB -N 4 # reservation des nodes pour le job
2265	# BATCH_NUM_PROC_TOT=32
2266	#
2267	# config.card :
2268	# ATM= (gcm.e, lmdz.x, 8MPI, 4OMP)
2269	#
2270	# tested with :
2271	# Job :
2272	# #MSUB -n 32
2273	# BATCH_NUM_PROC_TOT=32
2274	#
2275	# config.card :
2276	# ATM= (gcm.e, lmdz.x)
2277
2278
2279	for comp in ${config_ListOfComponents[*]} ; do
2280
2281	# Only if we really have an executable for the component :
2282	eval ExeNameOut=\${config_Executable_${comp}[1]}
2283	### if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
2284	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
2285
2286	IGCM_OMP=1
2287
2288	if ( ${OK_PARA_OMP} ) ; then
2289	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2290	export OMP_NUM_THREADS=${comp_proc_omp_loc}
2291	IGCMG_OMP=${comp_proc_omp_loc}
2292	fi
2293	if ( ${OK_PARA_MPI} ) ; then
2294	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2295	# Default : mpirun used if nb_proc gt 1
2296	# pour sortie out/err par process ? more test required
2297	# echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
2298	EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
2299	else
2300	# Check if BATCH_NUM_PROC_TOT = IGCMG_OMP
2301	if ( [ ${IGCM_OMP} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
2302	IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> ${comp}_PROC_OMP : ${BATCH_NUM_PROC_TOT} <> ${BRIDGE_MSUB_NPROC} "
2303	IGCM_debug_Print 1 "sys Titane : Check #MSUB -n xx into Job"
2304	IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
2305	fi
2306	# Default : mpirun is NOT used if nb_proc eq 1
2307	# pour sortie out/err par process ? more test required
2308	# echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2309	EXECUTION="/usr/bin/time ./${ExeNameOut}"
2310	fi
2311	fi
2312	done
2313
2314	fi
2315
2316	IGCM_debug_Print 1 "sys Titane : La commande d execution est "
2317	IGCM_debug_Print 1 "$EXECUTION"
2318
2319	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
2320	}
2321
2322	############################################################
2323	# Check of space available on temporary filesytems
2324	function IGCM_sys_check_quota {
2325	IGCM_debug_PushStack "IGCM_sys_check_quota"
2326	if ( $DEBUG_sys ) ; then
2327	echo "IGCM_sys_check_quota "
2328	fi
2329	# Limit of quota (in %)
2330	limit_quota=90
2331
2332	# Check of the volume
2333	volume_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $2}')
2334	volume_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $3}')
2335
2336	if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2337
2338	unit_avail=${volume_avail: -1}
2339	unit_quota=${volume_quota: -1}
2340
2341	if [ "${unit_quota}" = "*" ] ; then
2342	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2343	IGCM_debug_Print 1 "More than 100% of your quota is used"
2344	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2345	IGCM_debug_Print 1 "You must have more than 10% available to run"
2346	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2347	IGCM_debug_Verif_Exit
2348	fi
2349
2350	temp_avail=${volume_avail%%${volume_avail: -1}*}
2351	temp_quota=${volume_quota%%${volume_quota: -1}*}
2352
2353	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2354
2355	# Convertion
2356	if [ ${volume_avail: -1} = "T" ] ; then
2357	(( temp_avail = temp_avail * 1000000000000 ))
2358	elif [ ${volume_avail: -1} = "G" ] ; then
2359	(( temp_avail = temp_avail * 1000000000 ))
2360	elif [ ${volume_avail: -1} = "M" ] ; then
2361	(( temp_avail = temp_avail * 1000000 ))
2362	elif [ ${volume_avail: -1} = "k" ] ; then
2363	(( temp_avail = temp_avail * 1000 ))
2364	else
2365	(( temp_avail = volume_avail ))
2366	fi
2367	if [ ${volume_quota: -1} = "T" ] ; then
2368	(( temp_quota = temp_quota * 1000000000000 ))
2369	elif [ ${volume_quota: -1} = "G" ] ; then
2370	(( temp_quota = temp_quota * 1000000000 ))
2371	elif [ ${volume_quota: -1} = "M" ] ; then
2372	(( temp_quota = temp_quota * 1000000 ))
2373	elif [ ${volume_quota: -1} = "k" ] ; then
2374	(( temp_quota = temp_quota * 1000 ))
2375	else
2376	(( temp_quota = volume_quota ))
2377	fi
2378	fi
2379
2380	quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2381	# echo "volume ratio is " $quota_volume
2382
2383	if [ ${quota_volume} -ge ${limit_quota} ] ; then
2384	IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2385	IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2386	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2387	IGCM_debug_Print 1 "You must have more than 10% available to run"
2388	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2389	IGCM_debug_Verif_Exit
2390	fi
2391
2392	fi
2393
2394	# Check of the number of inodes
2395
2396	inode_quota=$(ccc_quota \| grep ' scratch' \| awk '{print $6}')
2397	inode_avail=$(ccc_quota \| grep ' scratch' \| awk '{print $7}')
2398
2399	if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2400
2401	unit_avail=${inode_avail: -1}
2402	unit_quota=${inode_quota: -1}
2403
2404	if [ "${unit_quota}" = "*" ] ; then
2405	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2406	IGCM_debug_Print 1 "More than 100% of your quota is used"
2407	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2408	IGCM_debug_Print 1 "You must have more than 10% available to run"
2409	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2410	IGCM_debug_Verif_Exit
2411	fi
2412
2413	temp_avail=${inode_avail%%${inode_avail: -1}*}
2414	temp_quota=${inode_quota%%${inode_quota: -1}*}
2415
2416	if [ ! ${unit_avail} = ${unit_quota} ] ; then
2417
2418	# Convertion
2419	if [ ${inode_avail: -1} = "T" ] ; then
2420	(( temp_avail = temp_avail * 1000000000000 ))
2421	elif [ ${inode_avail: -1} = "G" ] ; then
2422	(( temp_avail = temp_avail * 1000000000 ))
2423	elif [ ${inode_avail: -1} = "M" ] ; then
2424	(( temp_avail = temp_avail * 1000000 ))
2425	elif [ ${inode_avail: -1} = "k" ] ; then
2426	(( temp_avail = temp_avail * 1000 ))
2427	else
2428	(( temp_avail = inode_avail ))
2429	fi
2430
2431	if [ ${inode_quota: -1} = "T" ] ; then
2432	(( temp_quota = temp_quota * 1000000000000 ))
2433	elif [ ${inode_quota: -1} = "G" ] ; then
2434	(( temp_quota = temp_quota * 1000000000 ))
2435	elif [ ${inode_quota: -1} = "M" ] ; then
2436	(( temp_quota = temp_quota * 1000000 ))
2437	elif [ ${inode_quota: -1} = "k" ] ; then
2438	(( temp_quota = temp_quota * 1000 ))
2439	else
2440	(( temp_quota = inode_quota ))
2441	fi
2442	fi
2443	quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" \| bc)
2444	# echo "inode ratio is " $quota_inode
2445
2446	if [ ${quota_inode} -ge ${limit_quota} ] ; then
2447	IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2448	IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2449	IGCM_debug_Print 1 "Use the ccc_quota command to check"
2450	IGCM_debug_Print 1 "You must have more than 10% available to run"
2451	IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2452	IGCM_debug_Verif_Exit
2453	fi
2454
2455	fi
2456	IGCM_debug_PopStack "IGCM_sys_check_quota"
2457	}
2458
2459	##############################################################
2460	# NCO OPERATOR
2461
2462	function IGCM_sys_ncap2 {
2463	IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
2464	if ( $DEBUG_sys ) ; then
2465	echo "IGCM_sys_ncap2 :" $@
2466	fi
2467
2468	typeset NB_ESSAI DELAI status i
2469	# number of tentative
2470	NB_ESSAI=3
2471	# time delay between tentative
2472	DELAI=2
2473
2474	i=0
2475	while [ $i -lt $NB_ESSAI ] ; do
2476	ncap2 "$@" > out_rsync 2>&1
2477	status=$?
2478	if [ ${status} -gt 0 ] ; then
2479	IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
2480	cat out_rsync
2481	\rm out_rsync
2482	IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2483	else
2484	\rm out_rsync
2485	break
2486	fi
2487	(( i = i + 1 ))
2488	done
2489
2490	if [ ${status} -gt 0 ] ; then
2491	echo "IGCM_sys_ncap2 : ncap2 error"
2492	IGCM_debug_Exit "ncap2"
2493	fi
2494
2495	IGCM_debug_PopStack "IGCM_sys_ncap2"
2496	}
2497
2498	function IGCM_sys_ncatted {
2499	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
2500	if ( $DEBUG_sys ) ; then
2501	echo "IGCM_sys_ncatted :" $@
2502	fi
2503
2504	typeset NB_ESSAI DELAI status i
2505	# number of tentative
2506	NB_ESSAI=3
2507	# time delay between tentative
2508	DELAI=2
2509
2510	i=0
2511	while [ $i -lt $NB_ESSAI ] ; do
2512	ncatted "$@" > out_rsync 2>&1
2513	status=$?
2514	if [ ${status} -gt 0 ] ; then
2515	IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2516	cat out_rsync
2517	\rm out_rsync
2518	IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2519	else
2520	\rm out_rsync
2521	break
2522	fi
2523	(( i = i + 1 ))
2524	done
2525
2526	if [ ${status} -gt 0 ] ; then
2527	echo "IGCM_sys_ncatted : ncatted error"
2528	IGCM_debug_Exit "ncatted"
2529	fi
2530
2531	IGCM_debug_PopStack "IGCM_sys_ncatted"
2532	}
2533
2534	function IGCM_sys_ncbo {
2535	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
2536	if ( $DEBUG_sys ) ; then
2537	echo "IGCM_sys_ncbo :" $@
2538	fi
2539
2540	typeset NB_ESSAI DELAI status i
2541	# number of tentative
2542	NB_ESSAI=3
2543	# time delay between tentative
2544	DELAI=2
2545
2546	i=0
2547	while [ $i -lt $NB_ESSAI ] ; do
2548	ncbo $@ > out_rsync 2>&1
2549	status=$?
2550	if [ ${status} -gt 0 ] ; then
2551	IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2552	cat out_rsync
2553	\rm out_rsync
2554	IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2555	else
2556	\rm out_rsync
2557	break
2558	fi
2559	(( i = i + 1 ))
2560	done
2561
2562	if [ ${status} -gt 0 ] ; then
2563	echo "IGCM_sys_ncbo : ncbo error"
2564	IGCM_debug_Exit "ncbo"
2565	fi
2566
2567	IGCM_debug_PopStack "IGCM_sys_ncbo"
2568	}
2569
2570	function IGCM_sys_ncdiff {
2571	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
2572	if ( $DEBUG_sys ) ; then
2573	echo "IGCM_sys_ncdiff :" $@
2574	fi
2575
2576	typeset NB_ESSAI DELAI status i
2577	# number of tentative
2578	NB_ESSAI=3
2579	# time delay between tentative
2580	DELAI=2
2581
2582	i=0
2583	while [ $i -lt $NB_ESSAI ] ; do
2584	ncdiff $@ > out_rsync 2>&1
2585	status=$?
2586	if [ ${status} -gt 0 ] ; then
2587	IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2588	cat out_rsync
2589	\rm out_rsync
2590	IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2591	else
2592	\rm out_rsync
2593	break
2594	fi
2595	(( i = i + 1 ))
2596	done
2597
2598	if [ ${status} -gt 0 ] ; then
2599	echo "IGCM_sys_ncdiff : ncdiff error"
2600	IGCM_debug_Exit "ncdiff"
2601	fi
2602
2603	IGCM_debug_PopStack "IGCM_sys_ncdiff"
2604	}
2605
2606	function IGCM_sys_ncea {
2607	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
2608	if ( $DEBUG_sys ) ; then
2609	echo "IGCM_sys_ncea :" $@
2610	fi
2611
2612	typeset NB_ESSAI DELAI status i
2613	# number of tentative
2614	NB_ESSAI=3
2615	# time delay between tentative
2616	DELAI=2
2617
2618	i=0
2619	while [ $i -lt $NB_ESSAI ] ; do
2620	ncea $@ > out_rsync 2>&1
2621	status=$?
2622	if [ ${status} -gt 0 ] ; then
2623	IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2624	cat out_rsync
2625	\rm out_rsync
2626	IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2627	else
2628	\rm out_rsync
2629	break
2630	fi
2631	(( i = i + 1 ))
2632	done
2633
2634	if [ ${status} -gt 0 ] ; then
2635	echo "IGCM_sys_ncea : ncea error"
2636	IGCM_debug_Exit "ncea"
2637	fi
2638
2639	IGCM_debug_PopStack "IGCM_sys_ncea"
2640	}
2641
2642	function IGCM_sys_ncecat {
2643	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
2644	if ( $DEBUG_sys ) ; then
2645	echo "IGCM_sys_ncecat :" $@
2646	fi
2647
2648	typeset NB_ESSAI DELAI status i
2649	# number of tentative
2650	NB_ESSAI=3
2651	# time delay between tentative
2652	DELAI=2
2653
2654	i=0
2655	while [ $i -lt $NB_ESSAI ] ; do
2656	ncecat $@ > out_rsync 2>&1
2657	status=$?
2658	if [ ${status} -gt 0 ] ; then
2659	IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2660	cat out_rsync
2661	\rm out_rsync
2662	IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2663	else
2664	\rm out_rsync
2665	break
2666	fi
2667	(( i = i + 1 ))
2668	done
2669
2670	if [ ${status} -gt 0 ] ; then
2671	echo "IGCM_sys_ncecat : ncecat error"
2672	IGCM_debug_Exit "ncecat"
2673	fi
2674
2675	IGCM_debug_PopStack "IGCM_sys_ncecat"
2676	}
2677
2678	function IGCM_sys_ncflint {
2679	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
2680	if ( $DEBUG_sys ) ; then
2681	echo "IGCM_sys_ncflint :" $@
2682	fi
2683
2684	typeset NB_ESSAI DELAI status i
2685	# number of tentative
2686	NB_ESSAI=3
2687	# time delay between tentative
2688	DELAI=2
2689
2690	i=0
2691	while [ $i -lt $NB_ESSAI ] ; do
2692	ncflint $@ > out_rsync 2>&1
2693	status=$?
2694	if [ ${status} -gt 0 ] ; then
2695	IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2696	cat out_rsync
2697	\rm out_rsync
2698	IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2699	else
2700	\rm out_rsync
2701	break
2702	fi
2703	(( i = i + 1 ))
2704	done
2705
2706	if [ ${status} -gt 0 ] ; then
2707	echo "IGCM_sys_ncflint : ncflint error"
2708	IGCM_debug_Exit "ncflint"
2709	fi
2710
2711	IGCM_debug_PopStack "IGCM_sys_ncflint"
2712	}
2713
2714	function IGCM_sys_ncks {
2715	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
2716	if ( $DEBUG_sys ) ; then
2717	echo "IGCM_sys_ncks :" $@
2718	fi
2719
2720	typeset NB_ESSAI DELAI status i
2721	# number of tentative
2722	NB_ESSAI=3
2723	# time delay between tentative
2724	DELAI=2
2725
2726	i=0
2727	while [ $i -lt $NB_ESSAI ] ; do
2728	ncks $@ > out_rsync 2>&1
2729	status=$?
2730	if [ ${status} -gt 0 ] ; then
2731	IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2732	cat out_rsync
2733	\rm out_rsync
2734	IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2735	else
2736	\rm out_rsync
2737	break
2738	fi
2739	(( i = i + 1 ))
2740	done
2741
2742	if [ ${status} -gt 0 ] ; then
2743	echo "IGCM_sys_ncks : ncks error"
2744	IGCM_debug_Exit "ncks"
2745	fi
2746
2747	IGCM_debug_PopStack "IGCM_sys_ncks"
2748	}
2749
2750	function IGCM_sys_ncpdq {
2751	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
2752	if ( $DEBUG_sys ) ; then
2753	echo "IGCM_sys_ncpdq :" $@
2754	fi
2755
2756	typeset NB_ESSAI DELAI status i
2757	# number of tentative
2758	NB_ESSAI=3
2759	# time delay between tentative
2760	DELAI=2
2761
2762	i=0
2763	while [ $i -lt $NB_ESSAI ] ; do
2764	ncpdq $@ > out_rsync 2>&1
2765	status=$?
2766	if [ ${status} -gt 0 ] ; then
2767	IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2768	cat out_rsync
2769	\rm out_rsync
2770	IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2771	else
2772	\rm out_rsync
2773	break
2774	fi
2775	(( i = i + 1 ))
2776	done
2777
2778	if [ ${status} -gt 0 ] ; then
2779	echo "IGCM_sys_ncpdq : ncpdq error"
2780	IGCM_debug_Exit "ncpdq"
2781	fi
2782
2783	IGCM_debug_PopStack "IGCM_sys_ncpdq"
2784	}
2785
2786	function IGCM_sys_ncra {
2787	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
2788	if ( $DEBUG_sys ) ; then
2789	echo "IGCM_sys_ncra :" $@
2790	fi
2791
2792	typeset NB_ESSAI DELAI status i
2793	# number of tentative
2794	NB_ESSAI=3
2795	# time delay between tentative
2796	DELAI=2
2797
2798	i=0
2799	while [ $i -lt $NB_ESSAI ] ; do
2800	ncra $@ > out_rsync 2>&1
2801	status=$?
2802	if [ ${status} -gt 0 ] ; then
2803	IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2804	cat out_rsync
2805	\rm out_rsync
2806	IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2807	else
2808	\rm out_rsync
2809	break
2810	fi
2811	(( i = i + 1 ))
2812	done
2813
2814	if [ ${status} -gt 0 ] ; then
2815	echo "IGCM_sys_ncra : ncra error"
2816	IGCM_debug_Exit "ncra"
2817	fi
2818
2819	IGCM_debug_PopStack "IGCM_sys_ncra"
2820	}
2821
2822	function IGCM_sys_ncrcat {
2823	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
2824	if ( $DEBUG_sys ) ; then
2825	echo "IGCM_sys_ncrcat :" $@
2826	fi
2827
2828	typeset NB_ESSAI DELAI status i
2829	# number of tentative
2830	NB_ESSAI=3
2831	# time delay between tentative
2832	DELAI=2
2833
2834	i=0
2835	while [ $i -lt $NB_ESSAI ] ; do
2836	ncrcat $@ > out_rsync 2>&1
2837	status=$?
2838	if [ ${status} -gt 0 ] ; then
2839	IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2840	cat out_rsync
2841	\rm out_rsync
2842	IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2843	else
2844	\rm out_rsync
2845	break
2846	fi
2847	(( i = i + 1 ))
2848	done
2849
2850	if [ ${status} -gt 0 ] ; then
2851	echo "IGCM_sys_ncrcat : ncrcat error"
2852	#IGCM_debug_Exit "ncrcat"
2853	fi
2854
2855	IGCM_debug_PopStack "IGCM_sys_ncrcat"
2856	}
2857
2858	function IGCM_sys_ncrename {
2859	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
2860	if ( $DEBUG_sys ) ; then
2861	echo "IGCM_sys_ncrename :" $@
2862	fi
2863
2864	typeset NB_ESSAI DELAI status i
2865	# number of tentative
2866	NB_ESSAI=3
2867	# time delay between tentative
2868	DELAI=2
2869
2870	i=0
2871	while [ $i -lt $NB_ESSAI ] ; do
2872	ncrename $@ > out_rsync 2>&1
2873	status=$?
2874	if [ ${status} -gt 0 ] ; then
2875	IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2876	cat out_rsync
2877	\rm out_rsync
2878	IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2879	else
2880	\rm out_rsync
2881	break
2882	fi
2883	(( i = i + 1 ))
2884	done
2885
2886	if [ ${status} -gt 0 ] ; then
2887	echo "IGCM_sys_ncrename : ncrename error"
2888	IGCM_debug_Exit "ncrename"
2889	fi
2890
2891	IGCM_debug_PopStack "IGCM_sys_ncrename"
2892	}
2893
2894	function IGCM_sys_ncwa {
2895	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
2896	if ( $DEBUG_sys ) ; then
2897	echo "IGCM_sys_ncwa :" $@
2898	fi
2899
2900	typeset NB_ESSAI DELAI status i
2901	# number of tentative
2902	NB_ESSAI=3
2903	# time delay between tentative
2904	DELAI=2
2905
2906	i=0
2907	while [ $i -lt $NB_ESSAI ] ; do
2908	ncwa $@ > out_rsync 2>&1
2909	status=$?
2910	if [ ${status} -gt 0 ] ; then
2911	IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2912	cat out_rsync
2913	\rm out_rsync
2914	IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2915	else
2916	\rm out_rsync
2917	break
2918	fi
2919	(( i = i + 1 ))
2920	done
2921
2922	if [ ${status} -gt 0 ] ; then
2923	echo "IGCM_sys_ncwa : ncwa error"
2924	IGCM_debug_Exit "ncwa"
2925	fi
2926
2927	IGCM_debug_PopStack "IGCM_sys_ncwa"
2928	}
2929
2930	##############################################################
2931	# CDO OPERATOR
2932
2933	function IGCM_sys_cdo {
2934	IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
2935	if ( $DEBUG_sys ) ; then
2936	echo "IGCM_sys_cdo :" $@
2937	fi
2938
2939	typeset status
2940
2941	\cdo $@ > out_rsync 2>&1
2942	status=$?
2943	if [ ${status} -gt 0 ] ; then
2944	echo "IGCM_sys_cdo : error code ${status}"
2945	cat out_rsync
2946	\rm out_rsync
2947	IGCM_debug_PopStack "IGCM_sys_cdo"
2948	return 1
2949	else
2950	IGCM_debug_PopStack "IGCM_sys_cdo"
2951	return 0
2952	fi
2953
2954	IGCM_debug_PopStack "IGCM_sys_cdo"
2955	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: