Context Navigation

source: branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh @ 570

Last change on this file since 570 was 570, checked in by aclsce, 12 years ago
Modified to run different configurations mono-executable (sequential, MPI or hybrid MPI/OpenMP) and MPMD mode on titane. Warning : not tested !!!
Property svn:keywords set to `Revision Author Date`
File size: 49.5 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5	# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#=========================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#=========================================================
19
20	#D-#==================================================
21	#D-LibIGCM_sys for Titane
22	#D-#==================================================
23	#D-
24	#D- This ksh library if a layer under some usefull
25	#D-environment variables and shell commands.
26	#D-All those definitions depend on host particularities.
27	#D-It manages a stack mechanism and test validity of operations.
28	#D-All function described bellow must be prefixed by IGCM_sys.
29
30	#====================================================
31	# libIGCM_sys PARAMETERS
32	#====================================================
33
34	#====================================================
35	# set DEBUG_sys to true to output calls of function
36	typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38	#====================================================
39	# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40	typeset -r DRYRUN=${DRYRUN:=0}
41
42	# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43	# -------------------------------------------------------------------------------------
44	# \| DRYRUN= \| Date computations, \| sys_Get \| Exe \| sys_Put_Out; sys_Put_Rest \|
45	# \| \| Cp/Exe param files \| \| Chmod \| \|
46	# \| \| Qsub \| \| \| \|
47	# -------------------------------------------------------------------------------------
48	# \| 0 \| yes \| yes \| yes \| yes \|
49	# -------------------------------------------------------------------------------------
50	# \| 1 \| yes \| yes \| yes \| no \|
51	# -------------------------------------------------------------------------------------
52	# \| 2 \| yes \| yes \| no \| no \|
53	# -------------------------------------------------------------------------------------
54	# \| 3 \| yes \| no \| no \| no \|
55	# -------------------------------------------------------------------------------------
56
57	#=====================================================
58	# Global Variables :
59	#=====================================================
60	# Language : "fr" or "en"
61	typeset -r MYLANG="fr"
62
63	#=====================================================
64	# Host and user names
65	# $hostname ou hostname
66	typeset HOST=${HOST:=$( hostname )}
67	# $username ou whoami
68	typeset LOGIN=${LOGIN:=$( whoami )}
69	# $hostname of the MASTER job
70	typeset MASTER=titane
71
72	#D-
73	#D-#==================================================
74	#D-Program used in libIGCM
75	#D-#==================================================
76
77	# rsync with path
78	typeset -r RSYNC=/usr/bin/rsync
79	# RSYNC_opt args to rsync
80	typeset -r RSYNC_opt="-va"
81	# ie storage filesystem
82	typeset -r RHOST=titane
83
84	#====================================================
85	# Source default environment
86	#====================================================
87	. /etc/profile
88
89	#====================================================
90	# Set environment tools (ferret, nco, cdo)
91	#====================================================
92	. /home/cont003/p86ipsl/.atlas_env_titane_ksh
93
94	#====================================================
95	# Specific for ocean additionnal diagnostic
96	export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
97	export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
98
99	#====================================================
100	# Host specific DIRECTORIES
101	#====================================================
102
103	#====================================================
104	#- R_EXE (==> BIN_DIR = ${MODIPSL}/bin )
105	typeset -r R_EXE="${MODIPSL}/bin"
106
107	#====================================================
108	#- SUBMIT_DIR : submission dir
109	typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
110
111	#====================================================
112	#- ARCHIVE
113	typeset -r ARCHIVE=${DMFDIR}
114
115	#====================================================
116	#- Mirror libIGCM from titane to cesium if needed
117	ROOTSYS=$( echo ${libIGCM} \| gawk -F"/" '{print $2}' )
118	if [ ! ${ROOTSYS} = "home" ] ; then
119	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
120	else
121	typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
122	fi
123
124	#====================================================
125	#- libIGCM_POST
126	if ( ${MirrorlibIGCM} ) ; then
127	PATHlibIGCM=$( echo ${libIGCM} \| gawk -F"${LOGIN}/" '{print $2}' \| sed -e "s&/libIGCM&&" )
128	typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
129	else
130	typeset -r libIGCM_POST=${libIGCM}
131	fi
132
133	#====================================================
134	#- IN
135	typeset -r R_IN=${R_IN:=/dmnfs/cont003/p86ipsl/IGCM}
136	typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
137
138	#====================================================
139	#- OUT
140	typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
141
142	#====================================================
143	#- OUT_SCR (ONLY FOR double copy an scratch)
144	typeset -r R_OUT_SCR=${SCRATCHDIR}/IGCM_OUT
145
146	#====================================================
147	#- OUT_POST
148	typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
149
150	#====================================================
151	#- RUN_DIR_PATH : Temporary working directory (=> TMP)
152	if [ ! X${LSB_QUEUE} = Xmono ] ; then
153	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
154	else
155	typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
156	fi
157
158	#====================================================
159	#- BIG_DIR : BIG_DIR to store files waiting for rebuild
160	typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
161
162	#====================================================
163	#- HOST_MPIRUN_COMMAND
164	typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
165
166	#====================================================
167	#- Max number of arguments passed to nco operator or demigration command
168	UNIX_MAX_LIMIT=360
169
170	NUM_COREPERNODE=8
171
172	#====================================================
173	#- Default number of MPI task for IPSL coupled model
174	#- required for backward compatibility
175	#-
176	DEFAULT_NUM_PROC_OCE=5
177	DEFAULT_NUM_PROC_CPL=1
178	DEFAULT_NUM_PROC_ATM=26
179	DEFAULT_NUM_PROC_TOTAL=32
180
181	#D-#==================================================
182	#D-function IGCM_sys_RshMaster
183	#D-* Purpose: Just a fake command to wrapp
184	#D- IGCM_card call in post-treatment
185	#D- Ulam do not see brodie filesystem
186	#D- Cesium do not see all mercure filesystem
187	#D- That's why we need this hack.
188	#D-* Examples:
189	#D-
190	function IGCM_sys_RshMaster {
191	IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
192	/bin/ksh <<-EOF
193	export libIGCM=${libIGCM}
194	export DEBUG_debug=${DEBUG_debug}
195	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
196	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
197	${@}
198	EOF
199	if [ $? -gt 0 ] ; then
200	echo "IGCM_sys_RshMaster : erreur."
201	IGCM_debug_Exit "IGCM_sys_RshMaster"
202	fi
203	IGCM_debug_PopStack "IGCM_sys_RshMaster"
204	}
205
206	#D-#==================================================
207	#D-function IGCM_sys_RshArchive
208	#D-* Purpose: Archive rsh command
209	#D-* Examples:
210	#D-
211	function IGCM_sys_RshArchive {
212	IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
213	/bin/ksh <<-EOF
214	${@}
215	EOF
216	if [ $? -gt 0 ] ; then
217	echo "IGCM_sys_RshArchive : erreur."
218	IGCM_debug_Exit "IGCM_sys_RshArchive"
219	fi
220	IGCM_debug_PopStack "IGCM_sys_RshArchive"
221	}
222
223	#D-#==================================================
224	#D-function IGCM_sys_RshPost
225	#D-* Purpose: Post-process rsh command
226	#D-* Examples:
227	#D-
228	function IGCM_sys_RshPost {
229	IGCM_debug_PushStack "IGCM_sys_RshPost" $@
230	if ( $DEBUG_sys ) ; then
231	echo "IGCM_sys_RshPost :" $@
232	fi
233
234	typeset NB_ESSAI DELAI status i
235	# keep standard input for the loop onto temporary file
236	cat >tmp_IGCM_sys_RshPost_$$
237
238	#echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
239	#cat tmp_IGCM_sys_RshPost_$$
240
241	if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
242	# little hack so that rebuild submission is done on titane not an cesium
243	#
244	libIGCM_POST_sed=$( echo $libIGCM_POST \| sed 's/\//\\\//g' )
245	POST_DIR_sed=$( echo ${POST_DIR} \| sed 's/\//\\\//g' )
246	sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
247	sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
248	sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
249	sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
250	#\mv tmp.txt tmp_IGCM_sys_RshPost_$$
251	#
252	#echo cat tmp_IGCM_sys_RshPost_$$ AFTER
253	#cat tmp_IGCM_sys_RshPost_$$
254	/bin/ksh <tmp_IGCM_sys_RshPost_$$
255	if [ $? -gt 0 ] ; then
256	echo "IGCM_sys_RshPost : erreur."
257	IGCM_debug_Exit "IGCM_sys_RshPost"
258	fi
259	# delete temporary file
260	\rm tmp_IGCM_sys_RshPost_$$
261	else
262	# number of tentative
263	NB_ESSAI=10
264	# time delay between tentative
265	DELAI=10
266	i=0
267	while [ $i -ne $NB_ESSAI ] ; do
268	ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
269	status=$?
270	if [ ${status} -ne 0 ];
271	then
272	sleep $DELAI
273	else
274	break
275	fi
276	let i=$i+1
277	done
278	# delete temporary file
279	\rm tmp_IGCM_sys_RshPost_$$
280
281	if [ ${status} -gt 0 ] ; then
282	echo "IGCM_sys_RshPost : erreur."
283	IGCM_debug_Exit "IGCM_sys_RshPost"
284	fi
285	fi
286	IGCM_debug_PopStack "IGCM_sys_RshPost"
287	}
288
289	#D-#==================================================
290	#D-function IGCM_sys_SendMail
291	#D-* Purpose: Send mail when simulation is over
292	#D-* Examples:
293	#D-
294	function IGCM_sys_SendMail {
295	IGCM_debug_PushStack "IGCM_sys_SendMail" $@
296	if ( $DEBUG_sys ) ; then
297	echo "IGCM_sys_SendMail :" $@
298	fi
299
300	if ( ${ExitFlag} ) ; then
301	status=failed
302	else
303	status=completed
304	fi
305
306	ssh -t titane996 ssh platine /bin/ksh <<-EOF
307	export LOGIN=${LOGIN}
308	export config_UserChoices_JobName=${config_UserChoices_JobName}
309	export config_UserChoices_MailName=${config_UserChoices_MailName}
310	export DateBegin=${DateBegin}
311	export DateEnd=${DateEnd}
312	export R_SAVE=${R_SAVE}
313	export SUBMIT_DIR=${SUBMIT_DIR}
314	export status=${status}
315
316	cat << END_MAIL > job_end.mail
317	Dear ${LOGIN},
318
319	Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
320	Job started : ${DateBegin}
321	Job ended : ${DateEnd}
322	Output files are available in ${R_SAVE}
323	Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
324	END_MAIL
325
326	if [ ! -z ${config_UserChoices_MailName} ] ; then
327	mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
328	elif [ -f ~/.forward ] ; then
329	mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
330	fi
331
332	sleep 10
333	rm -f job_end.mail
334	EOF
335
336	if [ $? -gt 0 ] ; then
337	echo "IGCM_sys_SendMail : erreur."
338	IGCM_debug_Exit "IGCM_sys_SendMail"
339	fi
340	IGCM_debug_PopStack "IGCM_sys_SendMail"
341	}
342
343	#D-#==================================================
344	#D-function IGCM_sys_Mkdir
345	#D-* Purpose: Master locale mkdir command
346	#D-* Examples:
347	#D-
348	function IGCM_sys_Mkdir {
349	IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
350	if ( $DEBUG_sys ) ; then
351	echo "IGCM_sys_Mkdir :" $@
352	fi
353	if [ ! -d ${1} ]; then
354	\mkdir -p $1
355	if [ $? -gt 0 ] ; then
356	echo "IGCM_sys_Mkdir : erreur."
357	IGCM_debug_Exit "IGCM_sys_Mkdir"
358	fi
359	fi
360	# vérification :
361	if [ ! -d ${1} ] ; then
362	echo "IGCM_sys_Mkdir : erreur."
363	IGCM_debug_Exit "IGCM_sys_Mkdir"
364	fi
365	IGCM_debug_PopStack "IGCM_sys_Mkdir"
366	}
367
368	#D-#==================================================
369	#D-function IGCM_sys_MkdirArchive
370	#D-* Purpose: Mkdir on Archive
371	#D-* Examples:
372	#D-
373	function IGCM_sys_MkdirArchive {
374	IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
375	if ( $DEBUG_sys ) ; then
376	echo "IGCM_sys_MkdirArchive :" $@
377	fi
378	#- creation de repertoire sur le serveur fichier
379	if [ ! -d ${1} ]; then
380	\mkdir -p $1
381	if [ $? -gt 0 ] ; then
382	echo "IGCM_sys_MkdirArchive : erreur."
383	IGCM_debug_Exit "IGCM_sys_MkdirArchive"
384	fi
385	fi
386	IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
387	}
388
389	#D-#==================================================
390	#D-function IGCM_sys_MkdirWork
391	#D-* Purpose: Mkdir on Work
392	#D-* Examples:
393	#D-
394	function IGCM_sys_MkdirWork {
395	IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
396	if ( $DEBUG_sys ) ; then
397	echo "IGCM_sys_MkdirWork :" $@
398	fi
399	#- creation de repertoire sur le serveur fichier
400	if [ ! -d ${1} ]; then
401	\mkdir -p $1
402	if [ $? -gt 0 ] ; then
403	echo "IGCM_sys_MkdirWork : erreur."
404	IGCM_debug_Exit "IGCM_sys_MkdirWork"
405	fi
406	fi
407	IGCM_debug_PopStack "IGCM_sys_MkdirWork"
408	}
409
410	#D-#==================================================
411	#D-function IGCM_sys_Cd
412	#D-* Purpose: master cd command
413	#D-* Examples:
414	#D-
415	function IGCM_sys_Cd {
416	IGCM_debug_PushStack "IGCM_sys_Cd" $@
417	if ( $DEBUG_sys ) ; then
418	echo "IGCM_sys_Cd :" $@
419	fi
420	\cd $1
421	if [ $? -gt 0 ] ; then
422	echo "IGCM_sys_Cd : erreur."
423	IGCM_debug_Exit "IGCM_sys_Cd"
424	fi
425	IGCM_debug_PopStack "IGCM_sys_Cd"
426	}
427
428	#D-#==================================================
429	#D-function IGCM_sys_Chmod
430	#D-* Purpose: Chmod
431	#D-* Examples:
432	#D-
433	function IGCM_sys_Chmod {
434	IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
435	if ( $DEBUG_sys ) ; then
436	echo "IGCM_sys_Chmod :" $@
437	fi
438	if [ $DRYRUN -le 1 ]; then
439	\chmod $@
440	if [ $? -gt 0 ] ; then
441	echo "IGCM_sys_Chmod : erreur."
442	IGCM_debug_Exit "IGCM_sys_Chmod"
443	fi
444	else
445	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
446	fi
447	IGCM_debug_PopStack "IGCM_sys_Chmod"
448	}
449
450	#D-#==================================================
451	#D-function IGCM_sys_FileSize
452	#D-* Purpose: Filesize
453	#D-* Examples:
454	#D-
455	function IGCM_sys_FileSize {
456	IGCM_debug_PushStack "IGCM_sys_FileSize" $@
457
458	typeset sizeF
459	set +A sizeF -- $( ls -la ${1} )
460	if [ $? -gt 0 ] ; then
461	IGCM_debug_Exit "IGCM_sys_FileSize"
462	fi
463	eval ${2}=${sizeF[4]}
464
465	IGCM_debug_PopStack "IGCM_sys_FileSize"
466	}
467
468	#D-#==================================================
469	#D-function IGCM_sys_TestDir
470	#D-* Purpose: Test Directory that must exists
471	#D-* Examples:
472	#D-
473	function IGCM_sys_TestDir {
474	IGCM_debug_PushStack "IGCM_sys_TestDir" $@
475	if ( $DEBUG_sys ) ; then
476	echo "IGCM_sys_TestDir :" $@
477	fi
478	typeset ExistFlag
479	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
480	IGCM_debug_PopStack "IGCM_sys_TestDir"
481
482	return ${ExistFlag}
483	}
484
485	#D-#==================================================
486	#D-function IGCM_sys_TestDirArchive
487	#D-* Purpose: Test Directory that must exists on Archive
488	#D-* Examples:
489	#D-
490	function IGCM_sys_TestDirArchive {
491	IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
492	if ( $DEBUG_sys ) ; then
493	echo "IGCM_sys_TestDirArchive :" $@
494	fi
495	typeset ExistFlag
496	ExistFlag=$( [ -d $1 ] && echo 0 \|\| echo 1 )
497	IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
498
499	return ${ExistFlag}
500	}
501
502	#D-#==================================================
503	#D-function IGCM_sys_TestFileArchive
504	#D-* Purpose: Test file that must NOT EXISTS on Archive
505	#D-* Examples:
506	#D-
507	function IGCM_sys_TestFileArchive {
508	IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
509	typeset ExistFlag
510	ExistFlag=$( [ -f $1 ] && echo 0 \|\| echo 1 )
511	IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
512
513	return ${ExistFlag}
514	}
515
516	#D-#==================================================
517	#D-function IGCM_sys_CountFileArchive
518	#D-* Purpose: Count files on Archive filesystem
519	#D-* Examples:
520	#D-
521	function IGCM_sys_CountFileArchive {
522	IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
523	ls ${@} 2>/dev/null \| wc -l
524	if [ $? -gt 0 ] ; then
525	echo "IGCM_sys_CountFileArchive : erreur."
526	fi
527	IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
528	}
529
530	#D-#==================================================
531	#D-function IGCM_sys_Tree
532	#D-* Purpose: Tree directories with files on ${ARCHIVE}
533	#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
534	#D-
535	function IGCM_sys_Tree {
536	IGCM_debug_PushStack "IGCM_sys_Tree" $@
537	if ( $DEBUG_sys ) ; then
538	echo "IGCM_sys_Tree :" $@
539	fi
540
541	\ls -lR ${@}
542
543	IGCM_debug_PopStack "IGCM_sys_Tree"
544	}
545
546	#D-#==================================================
547	#D-function IGCM_sys_Tar
548	#D-* Purpose: master un-tar command
549	#D-* Examples:
550	#D-
551	function IGCM_sys_Tar {
552	IGCM_debug_PushStack "IGCM_sys_Tar" $@
553	if ( $DEBUG_sys ) ; then
554	echo "IGCM_sys_Tar :" $@
555	fi
556	\tar xvf $1
557	if [ $? -gt 0 ] ; then
558	echo "IGCM_sys_Tar : erreur."
559	IGCM_debug_Exit "IGCM_sys_Tar"
560	fi
561	IGCM_debug_PopStack "IGCM_sys_Tar"
562	}
563
564	#D-#==================================================
565	#D-function IGCM_sys_UnTar
566	#D-* Purpose: master un-tar command
567	#D-* Examples:
568	#D-
569	function IGCM_sys_UnTar {
570	IGCM_debug_PushStack "IGCM_sys_UnTar" $@
571	if ( $DEBUG_sys ) ; then
572	echo "IGCM_sys_UnTar :" $@
573	fi
574	\tar xvf $1
575	if [ $? -gt 0 ] ; then
576	echo "IGCM_sys_UnTar : erreur."
577	IGCM_debug_Exit "IGCM_sys_UnTar"
578	fi
579	IGCM_debug_PopStack "IGCM_sys_UnTar"
580	}
581
582	#D-#==================================================
583	#D-function IGCM_sys_Qsub
584	#D-* Purpose: Qsub new job
585	#D-* Examples:
586	#D-
587	function IGCM_sys_Qsub {
588	IGCM_debug_PushStack "IGCM_sys_Qsub" $@
589	if ( $DEBUG_sys ) ; then
590	echo "IGCM_sys_Qsub :" $@
591	fi
592
593	if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
594	/usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} < $1
595	else
596	/usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod} < $1
597	fi
598
599	if [ $? -gt 0 ] ; then
600	echo "IGCM_sys_Qsub : erreur " $@
601	IGCM_debug_Exit "IGCM_sys_Qsub"
602	fi
603	IGCM_debug_PopStack "IGCM_sys_Qsub"
604	}
605
606	#D-#==================================================
607	#D-function IGCM_sys_QsubPost
608	#D-* Purpose: Qsub new job on scalaire
609	#D-* Examples:
610	#D-
611	function IGCM_sys_QsubPost {
612	IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
613	if ( $DEBUG_sys ) ; then
614	echo "IGCM_sys_QsubPost :" $@
615	fi
616	/usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
617	if [ $? -gt 0 ] ; then
618	echo "IGCM_sys_QsubPost : erreur " $@
619	IGCM_debug_Exit "IGCM_sys_QsubPost"
620	fi
621	IGCM_debug_PopStack "IGCM_sys_QsubPost"
622	}
623
624	#D-*************************
625	#D- File transfer functions
626	#D-*************************
627	#D-
628
629	#D-#==================================================
630	#D-function IGCM_sys_Rsync_out
631	#D-* Purpose: treat return val of rsync
632	#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
633	#D- Error values and explanations can depend on your system version.
634	function IGCM_sys_Rsync_out {
635	RET=$1
636	if [ ! $RET ] ; then
637	echo "rsync error !"
638	fi
639
640	if [ $MYLANG = "fr" ]; then
641	case $RET in
642	0) return ;;
643	1) echo "Erreur de rsync ; RERR_SYNTAX : "
644	echo "Erreur de syntaxe ou d'utilisation."
645	return;;
646	2) echo "Erreur de rsync ; RERR_PROTOCOL : "
647	echo "Incompatibilité de protocole."
648	return;;
649	3) echo "Erreur de rsync ; RERR_FILESELECT 3"
650	echo "Erreurs lors de la sélection des fichiers d'entrée sortie et"
651	echo "répertoires"
652	return;;
653	4) echo "Erreur de rsync ; RERR_UNSUPPORTED"
654	echo "Action demandée non supportée : une tentative de manipulation de"
655	echo "fichiers 64-bits sur une plate-forme qui ne les supporte pas a"
656	echo "été faite ; ou une option qui est supportée par le client mais"
657	echo "pas par le serveur a été spécifiée."
658	return;;
659	10) echo "Erreur de rsync ; RERR_SOCKETIO"
660	echo "Erreur dans le socket d'entrée sortie"
661	return;;
662	11) echo "Erreur de rsync ; RERR_FILEIO"
663	echo "Erreur d'entrée sortie fichier"
664	return;;
665	12) echo "Erreur de rsync ; RERR_STREAMIO"
666	echo "Erreur dans flux de donnée du protocole rsync"
667	return;;
668	13) echo "Erreur de rsync ; RERR_MESSAGEIO"
669	echo "Erreur avec les diagnostics du programme"
670	return;;
671	14) echo "Erreur de rsync ; RERR_IPC"
672	echo "Erreur dans le code IPC"
673	return;;
674	20) echo "Erreur de rsync ; RERR_SIGNAL"
675	echo "SIGUSR1 ou SIGINT reçu"
676	return;;
677	21) echo "Erreur de rsync ; RERR_WAITCHILD"
678	echo "Une erreur retournée par waitpid()"
679	return;;
680	22) echo "Erreur de rsync ; RERR_MALLOC"
681	echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
682	return;;
683	23) echo ""
684	echo "Erreur fichier inexistant"
685	return;;
686	30) echo "Erreur de rsync ; RERR_TIMEOUT"
687	echo "Temps d'attente écoulé dans l'envoi/réception de données"
688	return;;
689	*) echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
690	return;;
691	esac
692	elif [ $MYLANG = "en" ] ; then
693	case $RET in
694	0) return;;
695	1) echo "rsync error : Syntax or usage error "
696	return;;
697	2) echo "rsync error : Protocol incompatibility "
698	return;;
699	3) echo "rsync error : Errors selecting input/output files, dirs"
700	return;;
701	4) echo "rsync error : Requested action not supported: an attempt"
702	echo "was made to manipulate 64-bit files on a platform that cannot support"
703	echo "them; or an option was specified that is supported by the client and"
704	echo "not by the server."
705	return;;
706	5) echo "rsync error : Error starting client-server protocol"
707	return;;
708	10) echo "rsync error : Error in socket I/O "
709	return;;
710	11) echo "rsync error : Error in file I/O "
711	return;;
712	12) echo "rsync error : Error in rsync protocol data stream "
713	return;;
714	13) echo "rsync error : Errors with program diagnostics "
715	return;;
716	14) echo "rsync error : Error in IPC code "
717	return;;
718	20) echo "rsync error : Received SIGUSR1 or SIGINT "
719	return;;
720	21) echo "rsync error : Some error returned by waitpid() "
721	return;;
722	22) echo "rsync error : Error allocating core memory buffers "
723	return;;
724	23) echo "rsync error : Partial transfer due to error"
725	return;;
726	24) echo "rsync error : Partial transfer due to vanished source files"
727	return;;
728	30) echo "rsync error : Timeout in data send/receive "
729	return;;
730	*) echo "rsync error : return code of rsync unknown :" $RET
731	return;;
732	esac
733	else
734	echo "unknown language $MYLANG."
735	return
736	fi
737	}
738
739
740	#D-#==================================================
741	#D-function IGCM_sys_Miror_libIGCM
742	#D-* Purpose: Mirror libIGCM PATH and lib to cesium
743	#D-* Examples:
744	#D-
745	function IGCM_sys_Mirror_libIGCM {
746	IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
747	if ( $DEBUG_sys ) ; then
748	echo "IGCM_sys_Mirror_libIGCM"
749	fi
750
751	typeset RET DEST
752
753	mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
754
755	echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
756	${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
757	RET=$?
758
759	if [ ${RET} -gt 0 ] ; then
760	echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
761	cat out_rsync
762	fi
763	IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
764	}
765
766	#====================================================
767	#- Call IGCM_sys_Mirror_libIGCM now !
768	if ( $MirrorlibIGCM ) ; then
769	IGCM_sys_Mirror_libIGCM
770	fi
771
772	#D-#==================================================
773	#D-function IGCM_sys_Cp
774	#D-* Purpose: generic cp
775	#D-* Examples:
776	#D-
777	function IGCM_sys_Cp {
778	IGCM_debug_PushStack "IGCM_sys_Cp" $@
779	if ( $DEBUG_sys ) ; then
780	echo "IGCM_sys_Cp :" $@
781	fi
782
783	typeset RET
784
785	echo cp $@ > out_rsync 2>&1
786	\cp $@ >> out_rsync 2>&1
787	RET=$?
788
789	if [ ${RET} -gt 0 ] ; then
790	echo "IGCM_sys_Cp : error."
791	cat out_rsync
792	IGCM_debug_Exit "IGCM_sys_Cp"
793	else
794	\rm out_rsync
795	fi
796	IGCM_debug_PopStack "IGCM_sys_Cp"
797	}
798
799	#D-#==================================================
800	#D-function IGCM_sys_Rm
801	#D-* Purpose: generic rm
802	#D-* Examples:
803	#D-
804	function IGCM_sys_Rm {
805	IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
806	if ( $DEBUG_sys ) ; then
807	echo "IGCM_sys_Rm :" $@
808	fi
809
810	typeset RET
811
812	echo rm $@ > out_rsync 2>&1
813	\rm $@ >> out_rsync 2>&1
814	RET=$?
815
816	if [ ${RET} -gt 0 ] ; then
817	echo "IGCM_sys_Rm : error."
818	cat out_rsync
819	IGCM_debug_Exit "IGCM_sys_Rm"
820	else
821	\rm out_rsync
822	fi
823	IGCM_debug_PopStack "IGCM_sys_Rm"
824	}
825
826	#D-#==================================================
827	#D-function IGCM_sys_RmRunDir
828	#D-* Purpose: rm tmpdir (dummy function most of the time batch
829	#D- scheduler will do the job)
830	#D-* Examples:
831	#D-
832	function IGCM_sys_RmRunDir {
833	IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
834	if ( $DEBUG_sys ) ; then
835	echo "IGCM_sys_RmRunDir :" $@
836	fi
837
838	typeset RET
839
840	echo rm $@ > out_rsync 2>&1
841	\rm $@ >> out_rsync 2>&1
842	RET=$?
843
844	if [ ${RET} -gt 0 ] ; then
845	echo "IGCM_sys_RmRunDir : error."
846	cat out_rsync
847	IGCM_debug_Exit "IGCM_sys_RmRunDir"
848	else
849	\rm out_rsync
850	fi
851
852	IGCM_debug_PopStack "IGCM_sys_RmRunDir"
853	}
854
855	#D-#==================================================
856	#D-function IGCM_sys_Mv
857	#D-* Purpose: generic move
858	#D-* Examples:
859	#D-
860	function IGCM_sys_Mv {
861	IGCM_debug_PushStack "IGCM_sys_Mv" $@
862	if ( $DEBUG_sys ) ; then
863	echo "IGCM_sys_Mv :" $@
864	fi
865
866	if [ $DRYRUN = 0 ]; then
867
868	typeset RET
869
870	echo mv $@ > out_rsync 2>&1
871	\mv $@ >> out_rsync 2>&1
872	RET=$?
873
874	if [ ${RET} -gt 0 ] ; then
875	echo "IGCM_sys_Mv : error in mv."
876	cat out_rsync
877	IGCM_debug_Exit "IGCM_sys_Mv"
878	else
879	\rm out_rsync
880	fi
881	else
882	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
883	fi
884
885	IGCM_debug_PopStack "IGCM_sys_Mv"
886	}
887
888	#D-#==================================================
889	#D-function IGCM_sys_Put_Dir
890	#D-* Purpose: Copy a complete directory on $(ARCHIVE)
891	#D-* Examples:
892	#D-
893	function IGCM_sys_Put_Dir {
894	IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
895	if ( $DEBUG_sys ) ; then
896	echo "IGCM_sys_Put_Dir :" $@
897	fi
898	if [ $DRYRUN = 0 ]; then
899	if [ ! -d ${1} ] ; then
900	echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
901	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
902	return
903	fi
904
905	typeset RET
906
907	# Only if we use rsync
908	#IGCM_sys_TestDirArchive $( dirname $2 )
909	#
910	#USUAL WAY
911	\cp -r $1 $2 > out_rsync 2>&1
912	RET=$?
913
914	if [ ${RET} -gt 0 ] ; then
915	echo "IGCM_sys_Put_Dir : error."
916	cat out_rsync
917	IGCM_debug_Exit "IGCM_sys_Put_Dir"
918	else
919	\rm out_rsync
920	fi
921	else
922	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
923	fi
924	IGCM_debug_PopStack "IGCM_sys_Put_Dir"
925	}
926
927	#D-#==================================================
928	#D-function IGCM_sys_Get_Dir
929	#D-* Purpose: Copy a complete directory from $(ARCHIVE)
930	#D-* Examples:
931	#D-
932	function IGCM_sys_Get_Dir {
933	IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
934	if ( $DEBUG_sys ) ; then
935	echo "IGCM_sys_Get_Dir :" $@
936	fi
937	if [ $DRYRUN = 0 ]; then
938	# if [ ! -d ${1} ] ; then
939	# echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
940	# IGCM_debug_PopStack "IGCM_sys_Get_Dir"
941	# return
942	# fi
943
944	typeset RET
945
946	# Only if we use rsync
947	#IGCM_sys_TestDirArchive $( dirname $2 )
948	#
949	#USUAL WAY
950	# add dmget (to demigrate all offline files) to reduce time of this command :
951	dmget $1/*
952	\cp -r $1 $2 > out_rsync 2>&1
953	RET=$?
954
955	if [ ${RET} -gt 0 ] ; then
956	echo "IGCM_sys_Get_Dir : error."
957	cat out_rsync
958	IGCM_debug_Exit "IGCM_sys_Get_Dir"
959	else
960	\rm out_rsync
961	fi
962	else
963	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
964	fi
965	IGCM_debug_PopStack "IGCM_sys_Get_Dir"
966	}
967
968	#D-#==================================================
969	#D-function IGCM_sys_Get_Master
970	#D-* Purpose: Copy a complete directory from MASTER filesystem
971	#D-* Examples:
972	#D-
973	function IGCM_sys_Get_Master {
974	IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
975	if ( $DEBUG_sys ) ; then
976	echo "IGCM_sys_Get_Master :" $@
977	fi
978	if [ $DRYRUN = 0 ]; then
979	if [ ! -d ${1} ] ; then
980	echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
981	IGCM_debug_PopStack "IGCM_sys_Get_Master"
982	return
983	fi
984
985	typeset RET
986
987	#USUAL WAY
988	\cp -r $1 $2 > out_rsync 2>&1
989	RET=$?
990
991	if [ ${RET} -gt 0 ] ; then
992	echo "IGCM_sys_Get_Master : error."
993	cat out_rsync
994	IGCM_debug_Exit "IGCM_sys_Get_Master"
995	fi
996	else
997	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
998	fi
999	IGCM_debug_PopStack "IGCM_sys_Get_Master"
1000	}
1001
1002	#D-#==================================================
1003	#D-function IGCM_sys_Put_Rest
1004	#D-* Purpose: Put computied restarts on $(ARCHIVE).
1005	#D- File and target directory must exist.
1006	#D-* Examples:
1007	#D-
1008	function IGCM_sys_Put_Rest {
1009	IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1010	if ( $DEBUG_sys ) ; then
1011	echo "IGCM_sys_Put_Rest :" $@
1012	fi
1013	if [ $DRYRUN = 0 ]; then
1014	if [ ! -f ${1} ] ; then
1015	echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1016	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1017	fi
1018
1019	typeset RET
1020	#
1021	if [ X${JobType} = XRUN ] ; then
1022	IGCM_sys_Chmod 444 ${1}
1023	fi
1024	#
1025	IGCM_sys_TestDirArchive $( dirname $2 )
1026	#
1027	# USUAL WAY
1028	\cp $1 $2 > out_rsync 2>&1
1029	RET=$?
1030
1031	# #RSYNC WITH NETWORK SSH CALL
1032	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} > out_rsync 2>&1
1033	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} >> out_rsync 2>&1
1034
1035	# #RSYNC WITH NFS USE
1036	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1037	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1038
1039	# RET=$?
1040	# IGCM_sys_Rsync_out $RET
1041
1042	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1043	# (( RET=RET+$? ))
1044
1045	if [ ${RET} -gt 0 ] ; then
1046	echo "IGCM_sys_Put_Rest : error."
1047	cat out_rsync
1048	IGCM_debug_Exit "IGCM_sys_Put_Rest"
1049	else
1050	\rm out_rsync
1051	fi
1052	else
1053	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1054	fi
1055	IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1056	}
1057
1058	#D-#==================================================
1059	#D-function IGCM_sys_Put_Out
1060	#D-* Purpose: Copy a file on $(ARCHIVE) after have chmod it in readonly
1061	#D-* Examples:
1062	#D-
1063	function IGCM_sys_Put_Out {
1064	IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1065	if ( $DEBUG_sys ) ; then
1066	echo "IGCM_sys_Put_Out :" $@
1067	fi
1068	if [ $DRYRUN = 0 ]; then
1069	if [ ! -f ${1} ] ; then
1070	echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1071	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1072	return 1
1073	fi
1074	#
1075	IGCM_sys_MkdirArchive $( dirname $2 )
1076	#
1077	typeset RET
1078
1079	#=====================================================
1080	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1081	#=====================================================
1082
1083	#echo ${2} \| grep "${R_OUT}" > /dev/null 2>&1
1084	#if [ $? -eq 0 ] ; then
1085	# typeset WORKPATH FILEPATH
1086	# WORKPATH=$( dirname $2 \| sed -e "s\|${R_OUT}\|${R_OUT_SCR}\|" )
1087	# IGCM_sys_MkdirWork ${WORKPATH}
1088	# FILEPATH=${WORKPATH}/$( basename $2 )
1089	# #
1090	# IGCM_sys_Cp ${1} ${FILEPATH}
1091	#fi
1092
1093	if [ X${JobType} = XRUN ] ; then
1094	if [ X${3} = X ] ; then
1095	IGCM_sys_Chmod 444 ${1}
1096	fi
1097	fi
1098	#
1099	# USUAL WAY
1100	\cp $1 $2 > out_rsync 2>&1
1101	RET=$?
1102
1103	# #RSYNC WITH NETWORK SSH CALL
1104	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} > out_rsync 2>&1
1105	# ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} >> out_rsync 2>&1
1106
1107	# #RSYNC WITH NFS USE
1108	# echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1109	# ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1110
1111	# RET=$?
1112	# IGCM_sys_Rsync_out $RET
1113
1114	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1115	# (( RET=RET+$? ))
1116
1117	if [ ${RET} -gt 0 ] ; then
1118	echo "IGCM_sys_Put_Out : error."
1119	cat out_rsync
1120	IGCM_debug_Exit "IGCM_sys_Put_Out"
1121	else
1122	\rm out_rsync
1123	fi
1124	else
1125	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1126	fi
1127	IGCM_debug_PopStack "IGCM_sys_Put_Out"
1128	return 0
1129	}
1130
1131	#D-#==================================================
1132	#D-function IGCM_sys_Get
1133	#D-* Purpose: Get a file from ${ARCHIVE}
1134	#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1135	#D- IGCM_sys_Get /l Array_contain_myfiles /destpath/
1136	function IGCM_sys_Get {
1137	IGCM_debug_PushStack "IGCM_sys_Get" $@
1138
1139	typeset DEST RET dm_liste ifile target file_work
1140
1141	if ( $DEBUG_sys ) ; then
1142	echo "IGCM_sys_Get :" $@
1143	fi
1144	if [ $DRYRUN -le 2 ]; then
1145	if [ X${1} = X'/l' ] ; then
1146	# test if the first file is present in the old computation :
1147	eval set +A dm_liste \${${2}}
1148	else
1149	eval set +A dm_liste ${1}
1150	fi
1151	eval DEST=\${${#}}
1152
1153	#=====================================================
1154	# COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1155	#=====================================================
1156
1157	# Is it an R_OUT file (not R_IN) ?
1158	#echo ${dm_liste[0]} \| grep "${R_OUT}" > /dev/null 2>&1
1159	#if [ $? -eq 0 ] ; then
1160	# # Yes ? then we try to get it in SCRATCHDIR
1161	# set +A file_work $( echo ${dm_liste[*]} \| sed -e "s\|${R_OUT}\|${R_OUT_SCR}\|g" )
1162	# if [ -f ${file_work[0]} ] ; then
1163	# IGCM_sys_Cp ${file_work[*]} ${DEST}
1164	# IGCM_debug_PopStack "IGCM_sys_Get"
1165	# return
1166	# fi
1167	#fi
1168
1169	# test if the (first) file is present in the old computation :
1170	IGCM_sys_TestFileArchive ${dm_liste[0]}
1171	RET=$?
1172	if [ ${RET} -gt 0 ] ; then
1173	echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1174	IGCM_debug_Exit "IGCM_sys_Get"
1175	fi
1176
1177	dmget ${dm_liste[*]} > out_rsync 2>&1
1178	RET=$?
1179
1180	if [ ${RET} -gt 0 ] ; then
1181	echo "WARNING IGCM_sys_Get : demigration error."
1182	cat out_rsync
1183	echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1184	fi
1185
1186	#if [ ${RET} -gt 0 ] ; then
1187	# if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1188	# cat out_rsync
1189	# echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1190	# sleep 30
1191	# echo "We try another time"
1192	# dmget ${dm_liste[*]} > out_rsync 2>&1
1193	# RET=$?
1194	# if [ ${RET} -gt 0 ] ; then
1195	# echo "ERROR IGCM_sys_Get : again demigration error :"
1196	# cat out_rsync
1197	# IGCM_debug_Exit "IGCM_sys_Get"
1198	# fi
1199	# else
1200	# echo "ERROR IGCM_sys_Get : demigration error :"
1201	# cat out_rsync
1202	# IGCM_debug_Exit "IGCM_sys_Get"
1203	# fi
1204	#fi
1205
1206	#USUAL WAY
1207	if [ X${1} = X'/l' ] ; then
1208	(( RET=0 ))
1209	for target in ${dm_liste[*]} ; do
1210	local_file=$( basename ${target} )
1211	\cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1212	(( RET = RET + $? ))
1213	done
1214	else
1215	\cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1216	RET=$?
1217	fi
1218
1219	# #RSYNC WITH NETWORK SSH CALL
1220	# echo ${RSYNC} ${RSYNC_opt} -e ssh ${RHOST}:"${dm_liste}" ${RHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1221	# ${RSYNC} ${RSYNC_opt} -e ssh ${RHOST}:"${dm_liste}" ${RHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1222
1223	# #RSYNC WITH NFS USE
1224	# echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1225	# ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1226
1227	# RET=$?
1228	# IGCM_sys_Rsync_out $RET
1229
1230	# ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1231	# (( RET=RET+$? ))
1232
1233	if [ ${RET} -gt 0 ] ; then
1234	echo "IGCM_sys_Get : copy error."
1235	cat out_rsync
1236	IGCM_debug_Exit "IGCM_sys_Get"
1237	else
1238	\rm out_rsync
1239	fi
1240	else
1241	( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1242	fi
1243	IGCM_debug_PopStack "IGCM_sys_Get"
1244	}
1245
1246	############################################################## A FINIR !!
1247
1248	#D-#==================================================
1249	#D-function IGCM_sys_GetDate_FichWork
1250	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1251	#D-* Examples:
1252	#D-
1253	function IGCM_sys_GetDate_FichWork {
1254	IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1255	if ( $DEBUG_sys ) ; then
1256	echo "IGCM_sys_GetDate_FichWork :" $@
1257	fi
1258	# donne la date filesys d'un fichier sur la machine work
1259	IGCM_debug_PopStack "IGCM_sys_FichWork"
1260	}
1261
1262	#D-#==================================================
1263	#D-function IGCM_sys_GetDate_FichArchive
1264	#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1265	#D-* Examples:
1266	#D-
1267	function IGCM_sys_GetDate_FichArchive {
1268	IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1269	if ( $DEBUG_sys ) ; then
1270	echo "IGCM_sys_GetDate_FichArchive :" $@
1271	fi
1272	IGCM_debug_PopStack "IGCM_sys_FichArchive"
1273	}
1274
1275	##############################################################
1276	# REBUILD OPERATOR
1277
1278	function IGCM_sys_rebuild {
1279	IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1280	if ( $DEBUG_sys ) ; then
1281	echo "IGCM_sys_rebuild :" $@
1282	fi
1283	/home/cont003/p86ipsl/X64/bin/rebuild -f -o $@
1284	if [ $? -gt 0 ] ; then
1285	echo "IGCM_sys_rebuild : erreur ${@}."
1286	IGCM_debug_Exit "rebuild"
1287	fi
1288
1289	IGCM_debug_PopStack "IGCM_sys_rebuild"
1290	}
1291
1292	############################################################
1293	# Activate Running Environnment Variables
1294
1295	function IGCM_sys_activ_variables {
1296	IGCM_debug_PushStack "IGCM_sys_activ_variables"
1297	if ( $DEBUG_sys ) ; then
1298	echo "IGCM_sys_activ_variables"
1299	fi
1300	# ulimit -s 2097152
1301
1302	typeset max_omp
1303
1304	ulimit -s unlimited
1305
1306	(( max_omp = 0 ))
1307	for comp in ${config_ListOfComponents[*]} ; do
1308
1309	eval ExeNameIn=\${config_Executable_${comp}[0]}
1310	eval ExeNameOut=\${config_Executable_${comp}[1]}
1311
1312
1313	# Only if we really have an executable for the component :
1314	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1315	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1316
1317	if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
1318
1319	(( max_omp = comp_proc_omp_loc ))
1320	fi
1321	fi
1322	done
1323	if [ ${max_omp} -gt 1 ] ; then
1324	module load openmp/${max_omp}thds
1325	fi
1326
1327	IGCM_debug_PopStack "IGCM_sys_activ_variables"
1328	}
1329
1330	############################################################
1331	# Desactivate Running Environnment Variables
1332
1333	function IGCM_sys_desactiv_variables {
1334	IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1335	if ( $DEBUG_sys ) ; then
1336	echo "IGCM_sys_desactiv_variables"
1337	fi
1338	IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1339	}
1340
1341	############################################################
1342	# Build MPI/OMP scripts run file (dummy function)
1343
1344	function IGCM_sys_build_run_file {
1345
1346	IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1347
1348	}
1349
1350	############################################################
1351	# Build MPI/OMP scripts
1352
1353	function IGCM_sys_build_execution_scripts
1354	{
1355	IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1356	if ( $DEBUG_sys ) ; then
1357	echo "IGCM_sys_build_execution_scripts " $@
1358	fi
1359	typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
1360	typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1361	typeset num_corempi nombre_restant_node nombre_restant_comp
1362
1363	if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
1364	IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1365	fi
1366
1367	if ( ${OK_PARA_MPMD} ) ; then
1368
1369	if [ -f run_file ] ; then
1370	IGCM_sys_Rm -f run_file
1371	fi
1372	touch run_file
1373
1374	if ( ${OK_PARA_OMP} ) ; then
1375
1376	# Hosts treatment
1377
1378	${HOST_MPIRUN_COMMAND} hostname \| sort \| uniq > hosts.tmp
1379
1380	i=0
1381	rm -f hosts
1382	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
1383	for nodes in `cat hosts.tmp`
1384	do
1385	host[$i]=$nodes
1386	echo "${host[$i]} slots=8 max_slots=8" >> hosts
1387	IGCM_debug_Print 1 ${host[$i]}
1388	i=$((i+1))
1389	done
1390	rm -f hosts.tmp
1391
1392	listnodes=${host[*]}
1393
1394	EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
1395	# fi
1396
1397	# Initialisation
1398
1399	init_node=y
1400	node_num_current=0
1401	start_num=0
1402	init_exec=n
1403
1404
1405	# Test : if oasis is there, we put it at the first position
1406
1407	for comp in ${config_ListOfComponents[*]} ; do
1408
1409	if [ "X${comp}" = "XCPL" ] ; then
1410
1411	eval ExeNameIn=\${config_Executable_${comp}[0]}
1412	eval ExeNameOut=\${config_Executable_${comp}[1]}
1413
1414	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1415	echo "" >> script_${ExeNameOut}.ksh
1416	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1417	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1418	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1419	echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1420	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1421
1422	init_node=n
1423
1424	(( nombre_restant_node = NUM_COREPERNODE - 1 ))
1425	node_num_current=0
1426	node_current=${host[${node_num_current}]}
1427
1428	EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1429
1430	init_exec=y
1431	start_num=1
1432
1433	fi
1434
1435	done
1436
1437	# Then loop on the components (except for oasis)
1438
1439	for comp in ${config_ListOfComponents[*]} ; do
1440
1441	eval ExeNameIn=\${config_Executable_${comp}[0]}
1442	eval ExeNameOut=\${config_Executable_${comp}[1]}
1443
1444
1445	# Only if we really have an executable for the component :
1446	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1447
1448	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1449	eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1450
1451	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1452	# echo "set -vx" >> script_${ExeNameOut}.ksh
1453	echo "" >> script_${ExeNameOut}.ksh
1454	echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh
1455	echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh
1456	echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh
1457	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1458	echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh
1459	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1460	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1461
1462	node_num=0
1463
1464	# We define the number of MPI process to be assigned for the component
1465
1466	nombre_restant_comp=${comp_proc_mpi_loc}
1467
1468	# Loop on the allocated nodes
1469
1470	for node in ${listnodes} ; do
1471
1472	# We go to the current node
1473
1474	if [ ${node_num} = ${node_num_current} ] ; then
1475
1476	node_current=${host[${node_num_current}]}
1477
1478	# If first time on the node : initialisation
1479
1480	if [ ${init_node} = y ] ; then
1481	nombre_restant_node=${NUM_COREPERNODE}
1482	fi
1483
1484	# Test on the number of OMP threads
1485
1486	if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1487	(( node_num = node_num + 1 ))
1488	node_num_current=${node_num}
1489	init_node=y
1490	continue
1491	fi
1492
1493	# Number of MPI process to assign
1494
1495	(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1496
1497	if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1498	num_corempi=${nombre_restant_comp}
1499	fi
1500
1501	(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1502	(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1503
1504	if [ ${init_exec} = y ] ; then
1505	EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1506	else
1507	EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1508	init_exec=y
1509	fi
1510
1511	(( start_num = num_corempi + start_num ))
1512
1513	else
1514
1515	(( node_num = node_num + 1 ))
1516	continue
1517	fi
1518
1519	# Test on the number of core/process remaining on the node/component
1520
1521	if [ ${nombre_restant_node} = 0 ] ; then
1522	(( node_num = node_num + 1 ))
1523	node_num_current=${node_num}
1524	init_node=y
1525
1526	if [ ${nombre_restant_comp} = 0 ] ; then
1527	break 1
1528	fi
1529	else
1530
1531	node_num_current=${node_num}
1532	init_node=n
1533
1534	if [ ${nombre_restant_comp} = 0 ] ; then
1535	break 1
1536	fi
1537	fi
1538	done
1539	fi
1540	done
1541
1542	else
1543
1544	# Then first loop on the components for the coupler ie oasis
1545
1546	### the coupler ie oasis must be the first one
1547	for comp in ${config_ListOfComponents[*]} ; do
1548
1549	eval ExeNameOut=\${config_Executable_${comp}[1]}
1550
1551	# for CPL component only
1552	if [ "X${comp}" = "XCPL" ] ; then
1553
1554	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1555
1556	if ( ${OK_PARA_MPI} ) ; then
1557	(( mpi_count = 1 ))
1558	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1559	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1560	(( mpi_count = mpi_count + 1 ))
1561	done
1562	else
1563	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1564	fi
1565	fi
1566	done
1567
1568	# Then second loop on the components
1569
1570	for comp in ${config_ListOfComponents[*]} ; do
1571
1572	eval ExeNameOut=\${config_Executable_${comp}[1]}
1573
1574	# Only if we really have an executable for the component and not the coupler ie oasis:
1575	if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1576
1577	eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1578
1579	if ( ${OK_PARA_MPI} ) ; then
1580
1581	(( mpi_count = 1 ))
1582	until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
1583	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1584	(( mpi_count = mpi_count + 1 ))
1585	done
1586	else
1587	echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1588	fi
1589	fi
1590	done
1591	IGCM_sys_Chmod u+x run_file
1592
1593	EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1594
1595	fi
1596
1597	else # Only one executable. launch it.
1598
1599	for comp in ${config_ListOfComponents[*]} ; do
1600
1601	# Only if we really have an executable for the component :
1602	eval ExeNameOut=\${config_Executable_${comp}[1]}
1603	if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1604
1605	echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1606	echo "" >> script_${ExeNameOut}.ksh
1607	if ( ${OK_PARA_OMP} ) ; then
1608	echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1609	fi
1610	if ( ${OK_PARA_MPI} ) ; then
1611	# Attention : a voir si sur 1 proc mais non //
1612	echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}" >> script_${ExeNameOut}.ksh
1613	IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1614	EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1615	else
1616	EXECUTION="time ./${ExeNameOut}"
1617	fi
1618	fi
1619	done
1620
1621	fi
1622
1623	IGCM_debug_Print 1 "sys Titane : La commande d execution est "
1624	IGCM_debug_Print 1 $EXECUTION
1625
1626	IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1627	}
1628
1629
1630
1631	##############################################################
1632	# NCO OPERATOR
1633
1634	function IGCM_sys_ncatted {
1635	IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1636	if ( $DEBUG_sys ) ; then
1637	echo "IGCM_sys_ncatted :" $@
1638	fi
1639	ncatted "$@"
1640	if [ $? -gt 0 ] ; then
1641	echo "IGCM_sys_ncatted : erreur ${@}."
1642	IGCM_debug_Exit "ncatted"
1643	fi
1644
1645	IGCM_debug_PopStack "IGCM_sys_ncatted"
1646	}
1647
1648	function IGCM_sys_ncbo {
1649	IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
1650	if ( $DEBUG_sys ) ; then
1651	echo "IGCM_sys_ncbo :" $@
1652	fi
1653	ncbo $@
1654	if [ $? -gt 0 ] ; then
1655	echo "IGCM_sys_ncbo : erreur ${@}."
1656	IGCM_debug_Exit "ncbo"
1657	fi
1658
1659	IGCM_debug_PopStack "IGCM_sys_ncbo"
1660	}
1661
1662	function IGCM_sys_ncdiff {
1663	IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
1664	if ( $DEBUG_sys ) ; then
1665	echo "IGCM_sys_ncdiff :" $@
1666	fi
1667	ncdiff $@
1668	if [ $? -gt 0 ] ; then
1669	echo "IGCM_sys_ncdiff : erreur ${@}."
1670	IGCM_debug_Exit "ncdiff"
1671	fi
1672
1673	IGCM_debug_PopStack "IGCM_sys_ncdiff"
1674	}
1675
1676	function IGCM_sys_ncea {
1677	IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
1678	if ( $DEBUG_sys ) ; then
1679	echo "IGCM_sys_ncea :" $@
1680	fi
1681	ncea $@
1682	if [ $? -gt 0 ] ; then
1683	echo "IGCM_sys_ncea : erreur ${@}."
1684	IGCM_debug_Exit "ncea"
1685	fi
1686
1687	IGCM_debug_PopStack "IGCM_sys_ncea"
1688	}
1689
1690	function IGCM_sys_ncecat {
1691	IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
1692	if ( $DEBUG_sys ) ; then
1693	echo "IGCM_sys_ncecat :" $@
1694	fi
1695	ncecat $@
1696	if [ $? -gt 0 ] ; then
1697	echo "IGCM_sys_ncecat : erreur ${@}."
1698	IGCM_debug_Exit "ncecat"
1699	fi
1700
1701	IGCM_debug_PopStack "IGCM_sys_ncecat"
1702	}
1703
1704	function IGCM_sys_ncflint {
1705	IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
1706	if ( $DEBUG_sys ) ; then
1707	echo "IGCM_sys_ncflint :" $@
1708	fi
1709	ncflint $@
1710	if [ $? -gt 0 ] ; then
1711	echo "IGCM_sys_ncflint : erreur ${@}."
1712	IGCM_debug_Exit "ncflint"
1713	fi
1714
1715	IGCM_debug_PopStack "IGCM_sys_ncflint"
1716	}
1717
1718	function IGCM_sys_ncks {
1719	IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
1720	if ( $DEBUG_sys ) ; then
1721	echo "IGCM_sys_ncks :" $@
1722	fi
1723	ncks $@
1724	if [ $? -gt 0 ] ; then
1725	echo "IGCM_sys_ncks : erreur ${@}."
1726	IGCM_debug_Exit "ncks"
1727	fi
1728
1729	IGCM_debug_PopStack "IGCM_sys_ncks"
1730	}
1731
1732	function IGCM_sys_ncpdq {
1733	IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
1734	if ( $DEBUG_sys ) ; then
1735	echo "IGCM_sys_ncpdq :" $@
1736	fi
1737	ncpdq $@
1738	if [ $? -gt 0 ] ; then
1739	echo "IGCM_sys_ncpdq : erreur ${@}."
1740	IGCM_debug_Exit "ncpdq"
1741	fi
1742
1743	IGCM_debug_PopStack "IGCM_sys_ncpdq"
1744	}
1745
1746	function IGCM_sys_ncra {
1747	IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
1748	if ( $DEBUG_sys ) ; then
1749	echo "IGCM_sys_ncra :" $@
1750	fi
1751	ncra $@
1752	if [ $? -gt 0 ] ; then
1753	echo "IGCM_sys_ncra : erreur ${@}."
1754	IGCM_debug_Exit "ncra"
1755	fi
1756
1757	IGCM_debug_PopStack "IGCM_sys_ncra"
1758	}
1759
1760	function IGCM_sys_ncrcat {
1761	IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
1762	if ( $DEBUG_sys ) ; then
1763	echo "IGCM_sys_ncrcat :" $@
1764	fi
1765	ncrcat $@
1766	if [ $? -gt 0 ] ; then
1767	echo "IGCM_sys_ncrcat : erreur ${@}."
1768	# IGCM_debug_Exit "ncrcat"
1769	fi
1770
1771	IGCM_debug_PopStack "IGCM_sys_ncrcat"
1772	}
1773
1774	function IGCM_sys_ncrename {
1775	IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
1776	if ( $DEBUG_sys ) ; then
1777	echo "IGCM_sys_ncrename :" $@
1778	fi
1779	ncrename $@
1780	if [ $? -gt 0 ] ; then
1781	echo "IGCM_sys_ncrename : erreur ${@}."
1782	IGCM_debug_Exit "ncrename"
1783	fi
1784
1785	IGCM_debug_PopStack "IGCM_sys_ncrename"
1786	}
1787
1788	function IGCM_sys_ncwa {
1789	IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
1790	if ( $DEBUG_sys ) ; then
1791	echo "IGCM_sys_ncwa :" $@
1792	fi
1793	ncwa $@
1794	if [ $? -gt 0 ] ; then
1795	echo "IGCM_sys_ncwa : erreur ${@}."
1796	IGCM_debug_Exit "ncwa"
1797	fi
1798
1799	IGCM_debug_PopStack "IGCM_sys_ncwa"
1800	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: