Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Blame
Revision Log

AA_create_ts @ 5899

Last change on this file since 5899 was 5899, checked in by aclsce, 3 years ago
Modified libIGCM and configuration to run XIOS ensembles.
File size: 31.7 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -o TS.out_%I
6	#-Q- curie #MSUB -e TS.out_%I
7	#-Q- curie #MSUB -n 1 # Reservation du processus
8	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
9	#-Q- curie #MSUB -q ::default_node::
10	#-Q- curie #MSUB -c ::default_core::
11	#-Q- curie #MSUB -Q normal
12	#-Q- curie #MSUB -A ::default_project::
13	#-Q- curie set +x
14	#-Q- irene ######################
15	#-Q- irene ## IRENE TGCC/CEA ##
16	#-Q- irene ######################
17	#-Q- irene #MSUB -r TS # Job name
18	#-Q- irene #MSUB -o TS.out_%I
19	#-Q- irene #MSUB -e TS.out_%I
20	#-Q- irene #MSUB -n 1 # Number of cores
21	#-Q- irene #MSUB -T 80000 # Maximum elapsed time
22	#-Q- irene #MSUB -q ::default_node::
23	#-Q- irene #MSUB -c ::default_core::
24	#-Q- irene #MSUB -Q normal
25	#-Q- irene #MSUB -A ::default_post_project::
26	#-Q- irene #MSUB -m store,work,scratch
27	#-Q- irene set +x
28	#-Q- irene-amd ######################
29	#-Q- irene-amd ## IRENE TGCC/CEA ##
30	#-Q- irene-amd ######################
31	#-Q- irene-amd #MSUB -r TS # Job name
32	#-Q- irene-amd #MSUB -o TS.out_%I
33	#-Q- irene-amd #MSUB -e TS.out_%I
34	#-Q- irene-amd #MSUB -n 1 # Number of cores
35	#-Q- irene-amd #MSUB -T 80000 # Maximum elapsed time
36	#-Q- irene-amd #MSUB -q ::default_node::
37	#-Q- irene-amd #MSUB -c ::default_core::
38	#-Q- irene-amd #MSUB -Q normal
39	#-Q- irene-amd #MSUB -A ::default_post_project::
40	#-Q- irene-amd #MSUB -m store,work,scratch
41	#-Q- irene-amd set +x
42	#-Q- jeanzay #!/bin/ksh
43	#-Q- jeanzay ######################
44	#-Q- jeanzay ## JEANZAY IDRIS ##
45	#-Q- jeanzay ######################
46	#-Q- jeanzay #SBATCH --job-name=TS # Job Name
47	#-Q- jeanzay #SBATCH --output=TS.out_%I # standard output
48	#-Q- jeanzay #SBATCH --error=TS.out_%I # error output
49	#-Q- jeanzay #SBATCH -N 1 # Number of core
50	#-Q- jeanzay #SBATCH --partition=prepost # Post-processing partition
51	#-Q- jeanzay #SBATCH --time=10:00:00 # Wall clock limit (seconds)
52	#-Q- jeanzay #SBATCH --account ::default_project::@cpu
53	#-Q- jeanzay set +x
54	#-Q- ada #!/bin/ksh
55	#-Q- ada #######################
56	#-Q- ada ## ADA IDRIS ##
57	#-Q- ada #######################
58	#-Q- ada # @ job_type = mpich
59	#-Q- ada # @ requirements = (Feature == "prepost")
60	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
61	#-Q- ada # @ wall_clock_limit = 20:00:00
62	#-Q- ada # Nom du travail LoadLeveler
63	#-Q- ada # @ job_name = TS
64	#-Q- ada # Fichier de sortie standard du travail
65	#-Q- ada # @ output = $(job_name).$(jobid)
66	#-Q- ada # Fichier de sortie d'erreur du travail
67	#-Q- ada # @ error = $(job_name).$(jobid)
68	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
69	#-Q- ada # @ notification = error
70	#-Q- ada # @ environment = $DEBUG_debug ; $BigBrother ; $postProcessingStopLevel ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $POST_DIR ; $Script_Post_Output ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateEnd ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; wall_clock_limit=$(wall_clock_limit)
71	#-Q- ada # @ queue
72	#-Q- lxiv8 ######################
73	#-Q- lxiv8 ## OBELIX LSCE ##
74	#-Q- lxiv8 ######################
75	#-Q- lxiv8 #PBS -N TS
76	#-Q- lxiv8 #PBS -m a
77	#-Q- lxiv8 #PBS -j oe
78	#-Q- lxiv8 #PBS -q medium
79	#-Q- lxiv8 #PBS -o TS.$$
80	#-Q- lxiv8 #PBS -S /bin/ksh
81	#-Q- ifort_CICLAD ######################
82	#-Q- ifort_CICLAD ## CICLAD IPSL ##
83	#-Q- ifort_CICLAD ######################
84	#-Q- ifort_CICLAD #PBS -N TS
85	#-Q- ifort_CICLAD #PBS -m a
86	#-Q- ifort_CICLAD #PBS -j oe
87	#-Q- ifort_CICLAD ###PBS -q h12 # Queue for 12 hours at ciclad only
88	#-Q- ifort_CICLAD #PBS -S /bin/ksh
89	#-Q- default #!/bin/ksh
90	#-Q- default ##################
91	#-Q- default ## DEFAULT HOST ##
92	#-Q- default ##################
93
94	#**************************************************************
95	# Author: Sebastien Denvil
96	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
97	# $Revision:: 1525 $ Revision of last commit
98	# $Author:: cetlod $ Author of last commit
99	# $Date:: 2020-04-27 10:39:22 +0200 (Mon, 27 Apr 2020) $ Date of last commit
100	# IPSL (2006)
101	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
102	#
103	#**************************************************************
104
105	#set -eu
106	#set -vx
107
108	date
109
110	#-Q- ada export OMP_NUM_THREADS=1
111
112	#D- Task type DO NOT CHANGE (computing, post-processing or checking)
113	TaskType=post-processing
114
115	########################################################################
116
117	#D- Flag to determine if this job in a standalone mode
118	#D- Default : value from AA_job if any
119	StandAlone=${StandAlone:=true}
120
121	#D- Path to libIGCM
122	#D- Default : value from AA_job if any
123	libIGCM=${libIGCM:=::modipsl::/libIGCM}
124
125	#D- Flag to determine allready produced time series. Empty if you start from the beginning
126	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
127	CompletedFlag=${CompletedFlag:=}
128
129	#D- Flag to determine task type this script will perform.
130	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
131	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
132	TsTask=${TsTask:=2D}
133	if [ X${TsTask} = Xempty ] ; then
134	TsTask=""
135	fi
136
137	#D- Flag to determine if rebuild process has been performed asynchronously
138	#D- Possible value true or false.
139	#D- If true raw files has allready been patched by rebuild job
140	#D- If false create_ts will patch the raw files
141	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
142	RebuildFrequency=${RebuildFrequency:=true}
143
144	#D- Flag to determine job's output directory
145	#D- Default : value from libIGCM_post.ksh if any
146	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
147
148	#D- Increased verbosity (1, 2, 3)
149	#D- Default : value from AA_job if any
150	Verbosity=${Verbosity:=3}
151
152	#D- Low level debug : to bypass lib test checks and stack construction
153	#D- Default : value from AA_job if any
154	DEBUG_debug=${DEBUG_debug:=false}
155
156	#D- TEMPORARY Flag to determine atmospheric resolution
157	#D- Default : value from atmospheric driver if any
158	RESOL_ATM=ALL
159
160	#D- Flag to determine surface resolution
161	#D- Default : value from surface driver if any
162	RESOL_SRF=ALL
163
164	#D- Flag to determine surface resolution
165	#D- Default : value from surface driver if any
166	RESOL_SBG=ALL
167
168	#D- TEMPORARY Flag to determine ocean resolution
169	#D- Default : value from ocean driver if any
170	RESOL_OCE=${RESOL_OCE:=ORCA2}
171
172	#D- TEMPORARY Flag to determine ice resolution
173	#D- Default : value from ice driver if any
174	RESOL_ICE=${RESOL_OCE:=ORCA2}
175
176	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
177	#D- Default : value from ice driver if any
178	RESOL_MBG=${RESOL_OCE:=ORCA2}
179
180	########################################################################
181
182	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
183	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
184	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
185	#-------
186	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
187	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
188	#-------
189	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
190	IGCM_sys_Cd ${RUN_DIR_PATH}
191	#-------
192	( ${DEBUG_debug} ) && IGCM_debug_Check
193	( ${DEBUG_debug} ) && IGCM_card_Check
194	( ${DEBUG_debug} ) && IGCM_date_Check
195
196	########################################################################
197
198	#set -vx
199
200	# Useful cleaning function
201	MENAGE ()
202	{
203	typeset ExistFlag
204	ExistFlag=$( IGCM_sys_RshArchive "[ -f $1/$3 ] && echo 0 \|\| echo 1" )
205	[ ${ExistFlag} -eq 0 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
206	}
207
208	########################################################################
209
210	# ------------------------------------------------------------------
211	# Test if all was right before proceeding further
212	# ------------------------------------------------------------------
213	IGCM_debug_Verif_Exit
214
215	if [ ${StandAlone} = true ] ; then
216	CARD_DIR=${SUBMIT_DIR}
217	else
218	CARD_DIR=${RUN_DIR_PATH}
219
220	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
221
222	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
223
224	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
225
226	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
227	fi
228
229	#==================================
230	# First of all
231	#
232	# Read libIGCM compatibility version in config.card
233	# Read UserChoices section
234	# Read Ensemble section
235	# Read Post section
236	# Define all netcdf output directories
237	#==================================
238	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
239
240	# ------------------------------------------------------------------
241	# Activate BigBrother so as to supervise this job
242	# ------------------------------------------------------------------
243	IGCM_debug_BigBro_Initialize
244
245	#==================================
246	# Define default value to keep compatibility with previous card: means before changes due to TGCC
247	if [ X${PackDefault} = Xtrue ] ; then
248	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
249	else
250	config_Post_PackFrequency=NONE
251	fi
252
253	#==================================
254	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
255	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
256	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
257	fi
258
259	#
260	# Determine component to take care of depending on kind of task create_ts will perform
261	#
262	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
263	#
264	# Determine period for time series. Default : value from AA_job or from launch_TS if any
265	#
266	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
267
268	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
269	#
270	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
271	#
272	# Determine Dimension of the produced time series : 2D or 3D
273	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
274	Dimension=2D
275	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
276	Dimension=3D
277	else
278	Dimension=""
279	fi
280
281	# ------------------------------------------------------------------
282	#D- Test if all was right before entering the loop
283	# ------------------------------------------------------------------
284	IGCM_debug_Verif_Exit
285
286	########################################################################
287	# Depending on requested time series fill following variables :
288	#
289	# - LISTE_FILE_${comp}[*]
290	# - LISTE_VARS_${file}[*]
291	# - LISTE_AXIS_${file}[*]
292	# - LISTE_PATCH_${file}[*]
293	########################################################################
294
295	#
296	# For each selected component determine which files need post-processing
297	#
298	DoJob=false
299	for comp in ${config_ListOfComponents[*]} ; do
300	#
301	ActiveComp=false
302	#
303	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
304	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
305	#
306	card=${CARD_DIR}/COMP/${compname}.card
307
308	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
309	#
310	ListFilesName=${compname}_OutputFiles_List
311	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
312	#
313	if [ X${FileName0} != X${NULL_STR} ] ; then
314	#
315	#IGCM_debug_Print 1 "Component : ${compname}"
316	#
317	# INITIALISATION
318	#
319	typeset i
320	if [ ${FlagToRead} ] ; then
321	((NbFiles=${FlagToRead}))
322	((i=${NbFiles}))
323	else
324	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
325	i=2
326	fi
327	#
328	until [ $i -gt $NbFiles ]; do
329	#
330	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
331	#
332	if [ X${flag_post} != XNONE ] ; then
333	#
334	# For each comp and each file who needs post-processing
335	# determine which variables need time series.
336	#
337	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
338	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
339	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
340	#
341	# If TimeSeriesVars list is empty we skip
342	#
343	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
344	#
345	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
346	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
347	(( i=i+3 ))
348	continue
349	#
350	fi
351	#
352	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
353	#
354	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
355	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
356	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
357	(( i=i+3 ))
358	continue
359	fi
360	#
361	# If we are not a Chunck type task, we skip files/variables handled by chunck task
362	#
363	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
364	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
365	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
366	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
367	(( i=i+3 ))
368	continue
369	fi
370	fi
371	#
372	# Now on we know we have things to do
373	#
374
375	#
376	# We need coordinates axis to properly fill produced time series
377	#
378	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
379	#
380	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
381	#
382	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
383	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
384	(( i=i+3 ))
385	continue
386	#
387	fi
388	#
389	# We need LIST of variables not allready produced (useful for standalone mode)
390	#
391	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
392	case ${FlagDir} in
393	*Y) TS_Dir=TS_YE ;;
394	*M) TS_Dir=TS_MO ;;
395	*D) TS_Dir=TS_DA ;;
396	HF) TS_Dir=TS_HF ;;
397	3H) TS_Dir=TS_HF ;;
398	INS) TS_Dir=TS_INS ;;
399	esac
400	#
401	count=0
402	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
403	#
404	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
405	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
406	#
407	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
408	if [ ! $? = 0 ] ; then
409	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
410	ActiveComp=true
411	(( count = count + 1 ))
412	fi
413	done
414	#
415	if [ ${count} -gt 0 ] ; then
416	#
417	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
418	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
419	#
420	if [ ! X${RebuildFrequency} = Xtrue ] ; then
421	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
422	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
423	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
424	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
425	if [ Xload${Patch} != X ] ; then
426	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
427	eval load${Patch}=loaded
428	fi
429	done
430	fi
431	fi
432	fi
433	fi
434	(( i=i+3 ))
435	done
436	fi
437	#
438	if [ ${ActiveComp} = true ] ; then
439	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
440	fi
441	done
442
443	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
444	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
445
446	########################################################################
447	#
448	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
449	# IF SO BRING THEM IN THE WORKING DIRECTORY
450	#
451	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
452	# - especially for IDRiS
453	# - no special protection against "command too long" !
454	# - libIGCM_sys should handle that or modify following code part
455	#
456	########################################################################
457
458	if [ ${StandAlone} != true ] ; then
459	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
460	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
461	echo \${run_PostProcessing_TimeSeriesRunning}" )
462
463	#if [ X${Running_Flag} = Xy ] ; then
464	# # Time Series Job allready running
465	# IGCM_debug_Print 1 "Time Series Job allready running exit"
466	# exit
467	#fi
468	# Now run.card know we are running
469
470
471
472
473	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
474
475
476
477
478
479	# Get information from last execution
480	#CompletedFlag=$( IGCM_sys_RshMaster \
481	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
482	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
483	fi
484
485	# Is it the first submission or not ?
486
487	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
488	# First Time Series Submission
489	FIRST_PASS=TRUE
490	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
491	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
492	else
493	# Last Completed Time Series
494	FIRST_PASS=FALSE
495	DATE_FIN_JOB_B=${CompletedFlag}
496	date=${DateBegin}_${DATE_FIN_JOB_B}
497	#
498	for comp in ${config_ListOfComponents[*]} ; do
499	#
500	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
501	#
502	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
503	#
504	unset liste_file
505	#
506	file_path=${R_SAVE}/${comp}/Analyse
507	#
508	i=0
509	file=${NULL_STR}
510	#
511	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
512	#
513	# Determine in which output can we find file
514	# IE : Analyse/TS or Analyse/DA2TS or ...
515	#
516	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
517	case ${FlagDir} in
518	*Y) TS_Dir=TS_YE ;;
519	*M) TS_Dir=TS_MO ;;
520	*D) TS_Dir=TS_DA ;;
521	HF) TS_Dir=TS_HF ;;
522	3H) TS_Dir=TS_HF ;;
523	INS) TS_Dir=TS_INS ;;
524	esac
525	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
526	#
527	IGCM_sys_TestFileArchive ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
528	if [ $? = 0 ] ; then
529	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
530	else
531	IGCM_debug_Print 1 "WARNING ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc dont exist"
532	fi
533	#
534	done
535	done
536	if [ X${file} != X${NULL_STR} ] ; then
537	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
538	IGCM_debug_Verif_Exit
539	fi
540	done
541	#
542	LEVEL=0
543	#
544	fi
545
546	########################################################################
547	#
548	# DEFINE LOOP PARAMETERS
549	#
550	########################################################################
551
552	DATE_COUNT=${DATE_FIN_JOB_B}
553	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
554
555	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
556
557	if [ ${DoJob} = true ] ; then
558	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
559	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
560	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
561	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
562	done
563	else
564	NBRE_FILE_TOT=0
565	fi
566
567	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
568	NBRE_TOUR=1
569	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
570	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
571	else
572	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
573	fi
574
575	CURRENT_LOOP=1
576	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
577	# WE HAVE 3 LEVELS
578	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
579	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
580	# - LEVEL 3 : WE ARE IN THE LAST LOOP
581
582	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
583
584	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
585	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
586	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
587
588	if [ ${FIRST_PASS} = TRUE ] ; then
589	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
590	else
591	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
592	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
593	fi
594
595	DATE_FIN=${PeriodDateEnd}
596	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
597
598	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
599	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
600
601	LEVEL=1
602	DEPOT="TRUE"
603	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
604	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
605	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
606
607	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
608	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
609	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
610	elif [ -z "${DATE_COURANTE}" ] ; then
611	DATE_COURANTE=${DATE_FIN_JOB_B}
612	else
613	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
614	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
615	fi
616
617	(( TotLength = 0 ))
618	COMPTEUR=1
619	DATE_LOOP=${DATE_COURANTE}
620
621	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
622	#
623	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
624	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
625	(( TotLength = TotLength + Length ))
626	(( COMPTEUR = COMPTEUR + 1 ))
627	#
628	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
629	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
630
631	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
632	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
633	DEPOT="FALSE"
634	LEVEL=2
635	else
636	# - LEVEL 3 : WE ARE IN THE LAST LOOP
637	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
638	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
639
640	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
641	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
642	DATE_FIN=${PeriodDateEnd}
643	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
644
645	LEVEL=3
646	DEPOT="TRUE"
647	FLAG_B="TRUE"
648	fi
649
650	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
651	for comp in ${config_ListOfComponents[*]} ; do
652	#
653	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
654	#
655	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
656	#
657	eval R_OUT_${comp}=${R_SAVE}/${comp}
658	#
659	i=0
660	#
661	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
662	#
663	# Determine in which output we can find file
664	# IE : Output/MO or Output/DA or ...
665	#
666	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
667	case ${FlagDir} in
668	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
669	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
670	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
671	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
672	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
673	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
674	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
675	IGCM_debug_Exit "Job create_ts" ;;
676	esac
677
678	# Initialize array
679	unset liste_file
680	unset liste_file_tmp
681	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
682
683	COMPTEUR=1
684	MissingFile=FALSE
685	DATE_LOOP=${DATE_COURANTE}
686
687	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
688	#
689	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
690	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
691
692	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
693	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
694
695	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
696	if [ $? = 0 ] ; then
697	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
698	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
699	else
700	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
701	MissingFile=TRUE
702	fi
703	(( COMPTEUR = COMPTEUR + 1 ))
704	#
705	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
706
707	# We skip this file type if one of them is missing.
708	if [ "${MissingFile}" = "TRUE" ] ; then
709	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
710	continue
711	fi
712
713	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
714
715	# Get selected files
716	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
717	IGCM_debug_Verif_Exit
718
719	# Apply IGCM_Patch if needed
720	if [ ! X${RebuildFrequency} = Xtrue ] ; then
721	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
722	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
723	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
724	IGCM_${Patch} ${file_t}
725	IGCM_debug_Verif_Exit
726	done
727	done
728	fi
729	fi
730
731	# WE CAN CONCATENATE FILES AT THIS POINT
732	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
733	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
734	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
735	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
736	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
737
738	liste_coord=" "
739	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
740	liste_coord=${liste_coord}${axis}","
741	done
742
743	# Time axis must be the last dimension
744	time_axis=${axis}
745
746	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
747	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
748	#
749	# We skip variables not in the first file of the list.
750	# Seems overkill to test thel all... but will do if needed
751	ncdump -hv ${var} ${liste_file_tmp[0]} > /dev/null 2>&1
752	if [ ! $? = 0 ] ; then
753	IGCM_debug_Print 1 "WARNING We skip ${var}, not present within ${liste_file_tmp[0]}"
754	continue
755	fi
756	#
757	if [ "${FLAG_B}" = "TRUE" ] ; then
758	# WE CONCATENATE WITH EXISTING FILES
759	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
760	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
761	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
762	IGCM_debug_Verif_Exit
763	#
764	IGCM_sys_Rm ${file1}
765	if [ ! "${DEPOT}" = "TRUE" ] ; then
766	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
767	# "${file1} = file_before ?
768	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
769	fi
770	else
771	# OR NOT
772	file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
773	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${file_out}
774	IGCM_debug_Verif_Exit
775	fi
776
777	#
778	if [ "${DEPOT}" = "TRUE" ] ; then
779	# WE PUT FINAL FILE ON FILE SYSTEM
780	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
781	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
782	#
783	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
784	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
785	if [ ! "X${file1}" = "X" ] ; then
786	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
787	fi
788	#
789	[ -f ${file_out} ] && IGCM_sys_Rm ${file_out}
790	#
791	fi
792	#
793	done # for var in ${liste_var} ; do
794	IGCM_sys_Rm ${liste_file_tmp[*]}
795	done # for file in ${liste_file_brut} do
796	done # for comp in ${config_ListOfComponents[*]} ; do
797	# GO AHEAD IN DEEPEST LOOP
798	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
799	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
800	# FINISH
801
802	# DODS copy
803	for comp in ${config_ListOfComponents[*]} ; do
804	for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do
805	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
806	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
807	done
808	done
809
810	export DEBUG_debug; export BigBrother ; export libIGCM; export SUBMIT_DIR; export POST_DIR; export Script_Output ;
811	export StandAlone ; export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ;
812	export RESOL_SBG
813	listVarEnv="DEBUG_debug,BigBrother,libIGCM,SUBMIT_DIR,POST_DIR,Script_Post_Output,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
814	export listVarEnv
815
816	#D-
817	# --------------------------------------------------------------------
818	#D- Test if all was right
819	# --------------------------------------------------------------------
820	IGCM_debug_Verif_Exit
821	# --------------------------------------------------------------------
822	#D- Submit MONITORING
823	# --------------------------------------------------------------------
824	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
825	# From 2D and 3D it is the last one to finish that will submit the monitoring.
826	if [ ! "X$( grep "IGCM_sys_RmRunDir" ${POST_DIR}/create_ts.${PeriodDateEnd}.[23]D.out 2> /dev/null )" = "X" ] ; then
827	Script_Post_Output=monitoring.${PeriodDateEnd}
828	IGCM_sys_QsubPost monitoring
829	fi
830	fi
831
832	#D-
833	# --------------------------------------------------------------------
834	#D- Update the run.card
835	# --------------------------------------------------------------------
836	if [ ${StandAlone} != true ] ; then
837	# Put in run.card end period of time series
838	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
839	# We have finish the job !
840	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
841	fi
842
843	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
844	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
845
846	# ------------------------------------------------------------------
847	# Finalize BigBrother to inform that the jobs end
848	# ------------------------------------------------------------------
849	IGCM_debug_BigBro_Finalize
850
851	date

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: CONFIG_DEVT/IPSLCM6.5_work_ENSEMBLES/libIGCM/AA_create_ts @ 5899

Download in other formats: