Context Navigation

source: trunk/libIGCM/AA_create_ts @ 821

Last change on this file since 821 was 817, checked in by sdipsl, 11 years ago
Skip file type processing if one of them is missing from IGCM_OUT. see #100
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 31.1 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q xlarge
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie set +x
11	#-Q- ada #!/bin/ksh
12	#-Q- ada #######################
13	#-Q- ada ## ADA IDRIS ##
14	#-Q- ada #######################
15	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
16	#-Q- ada # @ wall_clock_limit = 20:00:00
17	#-Q- ada # Nom du travail LoadLeveler
18	#-Q- ada # @ job_name = TS
19	#-Q- ada # Fichier de sortie standard du travail
20	#-Q- ada # @ output = $(job_name).$(jobid)
21	#-Q- ada # Fichier de sortie d'erreur du travail
22	#-Q- ada # @ error = $(job_name).$(jobid)
23	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
24	#-Q- ada # @ notification = error
25	#-Q- ada # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
26	#-Q- ada # @ queue
27	#-Q- sx9mercure #!/bin/ksh
28	#-Q- sx9mercure ######################
29	#-Q- sx9mercure ## SX9MERCURE CCRT ##
30	#-Q- sx9mercure ######################
31	#-Q- sx9mercure #PBS -N TS # Nom du job
32	#-Q- sx9mercure #PBS -j o # regroupement des stdout et stderr
33	#-Q- sx9mercure #PBS -S /usr/bin/ksh # shell de soumission
34	#-Q- sx9mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go
35	#-Q- sx9mercure #PBS -l elapstim_req=24:00:00 # Limite temps a 2 heures
36	#-Q- sx9mercure #PBS -q scalaire
37	#-Q- sx9mercure #PBS -r n
38	#-Q- titane #!/bin/ksh
39	#-Q- titane ######################
40	#-Q- titane ## TITANE CEA ##
41	#-Q- titane ######################
42	#-Q- titane #MSUB -r TS # Nom du job
43	#-Q- titane #MSUB -eo
44	#-Q- titane #MSUB -n 1 # Reservation du processus
45	#-Q- titane #MSUB -T 86400 # Limite de temps elapsed du job
46	#-Q- titane #MSUB -q mono
47	#-Q- titane #MSUB -E '-rn'
48	#-Q- lxiv8 ######################
49	#-Q- lxiv8 ## OBELIX LSCE ##
50	#-Q- lxiv8 ######################
51	#-Q- lxiv8 #PBS -N TS
52	#-Q- lxiv8 #PBS -m a
53	#-Q- lxiv8 #PBS -j oe
54	#-Q- lxiv8 #PBS -q medium
55	#-Q- lxiv8 #PBS -o TS.$$
56	#-Q- lxiv8 #PBS -S /bin/ksh
57	#-Q- default #!/bin/ksh
58	#-Q- default ##################
59	#-Q- default ## DEFAULT HOST ##
60	#-Q- default ##################
61
62	#**************************************************************
63	# Author: Sebastien Denvil
64	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
65	# $Revision:: $ Revision of last commit
66	# $Author:: $ Author of last commit
67	# $Date:: $ Date of last commit
68	# IPSL (2006)
69	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
70	#
71	#**************************************************************
72
73	#set -eu
74	#set -vx
75
76	date
77
78	#-Q- ada export OMP_NUM_THREADS=1
79
80	#D- Task type (computing or post-processing)
81	TaskType=post-processing
82
83	########################################################################
84
85	#D- Flag to determine if this job in a standalone mode
86	#D- Default : value from AA_job if any
87	StandAlone=${StandAlone:=true}
88
89	#D- Path to libIGCM
90	#D- Default : value from AA_job if any
91	libIGCM=${libIGCM:=::modipsl::/libIGCM}
92
93	#D- Flag to determine allready produced time series. Empty if you start from the beginning
94	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
95	CompletedFlag=${CompletedFlag:=}
96
97	#D- Flag to determine task type this script will perform.
98	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
99	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
100	TsTask=${TsTask:=2D}
101	if [ X${TsTask} = Xempty ] ; then
102	TsTask=""
103	fi
104
105	#D- Flag to determine if rebuild process has been performed asynchronously
106	#D- Possible value true or false.
107	#D- If true raw files has allready been patched by rebuild job
108	#D- If false create_ts will patch the raw files
109	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
110	RebuildFrequency=${RebuildFrequency:=true}
111
112	#D- Flag to determine job's output directory
113	#D- Default : value from libIGCM_post.ksh if any
114	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
115
116	#D- Increased verbosity (1, 2, 3)
117	#D- Default : value from AA_job if any
118	Verbosity=${Verbosity:=3}
119
120	#D- Low level debug : to bypass lib test checks and stack construction
121	#D- Default : value from AA_job if any
122	DEBUG_debug=${DEBUG_debug:=false}
123
124	#D- TEMPORARY Flag to determine atmospheric resolution
125	#D- Default : value from atmospheric driver if any
126	RESOL_ATM=ALL
127
128	#D- Flag to determine surface resolution
129	#D- Default : value from surface driver if any
130	RESOL_SRF=ALL
131
132	#D- Flag to determine surface resolution
133	#D- Default : value from surface driver if any
134	RESOL_SBG=ALL
135
136	#D- TEMPORARY Flag to determine ocean resolution
137	#D- Default : value from ocean driver if any
138	RESOL_OCE=${RESOL_OCE:=ORCA2}
139
140	#D- TEMPORARY Flag to determine ice resolution
141	#D- Default : value from ice driver if any
142	RESOL_ICE=${RESOL_ICE:=ORCA2}
143
144	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
145	#D- Default : value from ice driver if any
146	RESOL_MBG=${RESOL_MBG:=ORCA2}
147
148	########################################################################
149
150	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
151	( ${DEBUG_debug} ) && IGCM_debug_Check
152	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
153	( ${DEBUG_debug} ) && IGCM_card_Check
154	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
155	( ${DEBUG_debug} ) && IGCM_date_Check
156	#-------
157	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
158	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
159
160	########################################################################
161
162	#set -vx
163
164	# Useful cleaning function
165	MENAGE ()
166	{
167	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
168	}
169
170	########################################################################
171
172	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
173	IGCM_sys_Cd ${RUN_DIR_PATH}
174
175	# ------------------------------------------------------------------
176	# Test if all was right before proceeding further
177	# ------------------------------------------------------------------
178	IGCM_debug_Verif_Exit_Post
179
180	if [ ${StandAlone} = true ] ; then
181	CARD_DIR=${SUBMIT_DIR}
182	else
183	CARD_DIR=${RUN_DIR_PATH}
184	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
185	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
186	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
187	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
188	fi
189
190	#==================================
191	# First of all
192	#
193	# Read libIGCM compatibility version in config.card
194	# Read UserChoices section
195	# Read Ensemble section
196	# Read Post section
197	# Define all netcdf output directories
198	#==================================
199	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
200
201	#==================================
202	# Define default value to keep compatibility with previous card: means before changes due to TGCC
203	if [ X${PackDefault} = Xtrue ] ; then
204	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
205	else
206	config_Post_PackFrequency=NONE
207	fi
208
209	#==================================
210	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
211	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
212	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
213	fi
214
215	#
216	# Determine component to take care of depending on kind of task create_ts will perform
217	#
218	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
219	#
220	# Determine period for time series. Default : value from AA_job or from launch_TS if any
221	#
222	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
223
224	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
225	#
226	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
227	#
228	# Determine Dimension of the produced time series : 2D or 3D
229	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
230	Dimension=2D
231	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
232	Dimension=3D
233	else
234	Dimension=""
235	fi
236
237	# ------------------------------------------------------------------
238	#D- Test if all was right before entering the loop
239	# ------------------------------------------------------------------
240	IGCM_debug_Verif_Exit_Post
241
242	########################################################################
243	#
244	# Depending on requested time series fill following variables :
245	#
246	# - LISTE_FILE_${comp}[*]
247	# - LISTE_VARS_${file}[*]
248	# - LISTE_AXIS_${file}[*]
249	# - LISTE_PATCH_${file}[*]
250	########################################################################
251
252	#
253	# For each selected component determine which files need post-processing
254	#
255	DoJob=false
256	for comp in ${config_ListOfComponents[*]} ; do
257	#
258	ActiveComp=false
259	#
260	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
261	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
262	#
263	card=${CARD_DIR}/COMP/${compname}.card
264
265	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
266	#
267	ListFilesName=${compname}_OutputFiles_List
268	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
269	#
270	if [ X${FileName0} != X${NULL_STR} ] ; then
271	#
272	#IGCM_debug_Print 1 "Component : ${compname}"
273	#
274	# INITIALISATION
275	#
276	typeset i
277	if [ ${FlagToRead} ] ; then
278	((NbFiles=${FlagToRead}))
279	((i=${NbFiles}))
280	else
281	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
282	i=2
283	fi
284	#
285	until [ $i -gt $NbFiles ]; do
286	#
287	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
288	#
289	if [ X${flag_post} != XNONE ] ; then
290	#
291	# For each comp and each file who needs post-processing
292	# determine which variables need time series.
293	#
294	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
295	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
296	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
297	#
298	# If TimeSeriesVars list is empty we skip
299	#
300	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
301	#
302	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
303	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
304	(( i=i+3 ))
305	continue
306	#
307	fi
308	#
309	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
310	#
311	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
312	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
313	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
314	(( i=i+3 ))
315	continue
316	fi
317	#
318	# If we are not a Chunck type task, we skip files/variables handled by chunck task
319	#
320	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
321	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
322	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
323	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
324	(( i=i+3 ))
325	continue
326	fi
327	fi
328	#
329	# Now on we know we have things to do
330	#
331
332	#
333	# We need coordinates axis to properly fill produced time series
334	#
335	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
336	#
337	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
338	#
339	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
340	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
341	(( i=i+3 ))
342	continue
343	#
344	fi
345	#
346	# We need LIST of variables not allready produced (useful for standalone mode)
347	#
348	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
349	case ${FlagDir} in
350	*Y) TS_Dir=TS_YE ;;
351	*M) TS_Dir=TS_MO ;;
352	*D) TS_Dir=TS_DA ;;
353	HF) TS_Dir=TS_HF ;;
354	3H) TS_Dir=TS_HF ;;
355	INS) TS_Dir=TS_INS ;;
356	esac
357	#
358	count=0
359	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
360	#
361	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
362	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
363	#
364	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
365	if [ ! $? = 0 ] ; then
366	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
367	ActiveComp=true
368	(( count = count + 1 ))
369	fi
370	done
371	#
372	if [ ${count} -gt 0 ] ; then
373	#
374	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
375	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
376	#
377	if [ ! X${RebuildFrequency} = Xtrue ] ; then
378	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
379	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
380	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
381	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
382	if [ Xload${Patch} != X ] ; then
383	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
384	eval load${Patch}=loaded
385	fi
386	done
387	fi
388	fi
389	fi
390	fi
391	(( i=i+3 ))
392	done
393	fi
394	#
395	if [ ${ActiveComp} = true ] ; then
396	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
397	fi
398	done
399
400	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
401	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
402
403	########################################################################
404	#
405	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
406	# IF SO BRING THEM IN THE WORKING DIRECTORY
407	#
408	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
409	# - especially for IDRiS
410	# - no special protection against "command too long" !
411	# - libIGCM_sys should handle that or modify following code part
412	#
413	########################################################################
414
415	if [ ${StandAlone} != true ] ; then
416	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
417	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
418	echo \${run_PostProcessing_TimeSeriesRunning}" )
419
420	#if [ X${Running_Flag} = Xy ] ; then
421	# # Time Series Job allready running
422	# IGCM_debug_Print 1 "Time Series Job allready running exit"
423	# exit
424	#fi
425	# Now run.card know we are running
426
427
428
429
430	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
431
432
433
434
435
436	# Get information from last execution
437	#CompletedFlag=$( IGCM_sys_RshMaster \
438	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
439	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
440	fi
441
442	# Is it the first submission or not ?
443
444	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
445	# First Time Series Submission
446	FIRST_PASS=TRUE
447	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
448	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
449	else
450	# Last Completed Time Series
451	FIRST_PASS=FALSE
452	DATE_FIN_JOB_B=${CompletedFlag}
453	date=${DateBegin}_${DATE_FIN_JOB_B}
454	for comp in ${config_ListOfComponents[*]} ; do
455	#
456	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
457	#
458	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
459	#
460	#set -A liste_file
461	unset liste_file
462	#
463	file_path=${R_SAVE}/${comp}/Analyse
464	#
465	i=0
466	file=${NULL_STR}
467	#
468	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
469	#
470	# Determine in which output can we find file
471	# IE : Analyse/TS or Analyse/DA2TS or ...
472	#
473	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
474	case ${FlagDir} in
475	*Y) TS_Dir=TS_YE ;;
476	*M) TS_Dir=TS_MO ;;
477	*D) TS_Dir=TS_DA ;;
478	HF) TS_Dir=TS_HF ;;
479	3H) TS_Dir=TS_HF ;;
480	INS) TS_Dir=TS_INS ;;
481	esac
482	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
483	#
484	#set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
485	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
486	#
487	done
488	done
489	if [ X${file} != X${NULL_STR} ] ; then
490	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
491	# Even if some files are missing we tried to continue
492	# Either ListVar in compname.card are asked but not present in original history file
493	# Either a previous TS task failed
494	ExitFlag=false
495	IGCM_debug_Verif_Exit_Post
496	fi
497	done
498	#
499	LEVEL=0
500	#
501	fi
502
503	########################################################################
504	#
505	# DEFINE LOOP PARAMETERS
506	#
507	########################################################################
508
509	DATE_COUNT=${DATE_FIN_JOB_B}
510	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
511
512	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
513
514	if [ ${DoJob} = true ] ; then
515	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
516	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
517	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
518	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
519	done
520	else
521	NBRE_FILE_TOT=0
522	fi
523
524	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
525	NBRE_TOUR=1
526	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
527	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
528	else
529	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
530	fi
531
532	CURRENT_LOOP=1
533	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
534	# WE HAVE 3 LEVELS
535	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
536	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
537	# - LEVEL 3 : WE ARE IN THE LAST LOOP
538
539	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
540
541	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
542	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
543	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
544
545	if [ ${FIRST_PASS} = TRUE ] ; then
546	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
547	else
548	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
549	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
550	fi
551
552	DATE_FIN=${PeriodDateEnd}
553	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
554
555	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
556	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
557
558	LEVEL=1
559	DEPOT="TRUE"
560	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
561	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
562	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
563
564	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
565	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
566	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
567	elif [ -z "${DATE_COURANTE}" ] ; then
568	DATE_COURANTE=${DATE_FIN_JOB_B}
569	else
570	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
571	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
572	fi
573
574	(( TotLength = 0 ))
575	COMPTEUR=1
576	DATE_LOOP=${DATE_COURANTE}
577
578	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
579	#
580	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
581	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
582	(( TotLength = TotLength + Length ))
583	(( COMPTEUR = COMPTEUR + 1 ))
584	#
585	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
586	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
587
588	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
589	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
590	DEPOT="FALSE"
591	LEVEL=2
592	else
593	# - LEVEL 3 : WE ARE IN THE LAST LOOP
594	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
595	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
596
597	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
598	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
599	DATE_FIN=${PeriodDateEnd}
600	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
601
602	LEVEL=3
603	DEPOT="TRUE"
604	FLAG_B="TRUE"
605	fi
606
607	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
608	for comp in ${config_ListOfComponents[*]} ; do
609	#
610	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
611	#
612	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
613	#
614	eval R_OUT_${comp}=${R_SAVE}/${comp}
615	#
616	i=0
617	#
618	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
619	#
620	# Determine in which output we can find file
621	# IE : Output/MO or Output/DA or ...
622	#
623	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
624	case ${FlagDir} in
625	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
626	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
627	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
628	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
629	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
630	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
631	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
632	IGCM_sys_Exit "Job create_ts" ;;
633	esac
634
635	# Initialize array
636	#set -A liste_file
637	#set -A liste_file_tmp
638	unset liste_file
639	unset liste_file_tmp
640	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
641
642	COMPTEUR=1
643	MissingFile=FALSE
644	DATE_LOOP=${DATE_COURANTE}
645
646	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
647	#
648	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
649	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
650
651	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
652	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
653
654	#set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
655	#set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
656
657	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
658	if [ $? = 0 ] ; then
659	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
660	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
661	else
662	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
663	MissingFile=TRUE
664	fi
665	(( COMPTEUR = COMPTEUR + 1 ))
666	#
667	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
668
669	# We skip this file type if one of them is missing.
670	if [ "${MissingFile}" = "TRUE" ] ; then
671	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
672	continue
673	fi
674
675	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
676
677	# Get selected files
678	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
679	IGCM_debug_Verif_Exit_Post
680
681	# Apply IGCM_Patch if needed
682	if [ ! X${RebuildFrequency} = Xtrue ] ; then
683	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
684	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
685	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
686	IGCM_${Patch} ${file_t}
687	IGCM_debug_Verif_Exit_Post
688	done
689	done
690	fi
691	fi
692
693	# WE CAN CONCATENATE FILES AT THIS POINT
694	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
695	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
696	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
697	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
698	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
699
700	liste_coord=" "
701	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
702	liste_coord=${liste_coord}${axis}","
703	done
704
705	# Time axis must be the last dimension
706	time_axis=${axis}
707
708	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
709	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
710	#
711	if [ "${FLAG_B}" = "TRUE" ] ; then
712	# WE CONCATENATE WITH EXISTING FILES
713	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
714	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
715	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
716	#
717	IGCM_sys_Rm ${file1}
718	if [ ! "${DEPOT}" = "TRUE" ] ; then
719	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
720	# "${file1} = file_before ?
721	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
722	fi
723
724	ExitFlag=false
725	else
726	# OR NOT
727	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
728	fi
729
730	#
731	if [ "${DEPOT}" = "TRUE" ] ; then
732	# WE PUT FINAL FILE ON FILE SYSTEM
733	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
734	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
735	#
736	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
737	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
738	if [ ! "X${file1}" = "X" ] ; then
739	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
740	fi
741	#
742	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
743	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
744	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
745	#
746	# Compute yearly average considering calendar
747	# and set date of the mid year to 01/07 at 00:00
748	IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
749	# Extract all ancillary variables except $var, they will be appended after the cdo command
750	# Needed because cdo applies calculs on all variables
751	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
752	# This will remove the time axis because cdo rewrites severely with only considered useful variables
753	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
754	if [ $? -eq 0 ] ; then
755	# Now append all ancillary variables if previous cdo operator ok
756	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
757	fi
758
759	# Put global history attribut
760	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
761	#
762	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
763	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
764	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
765	fi
766	#
767	IGCM_sys_Rm ${file_out}
768	ExitFlag=false
769	#
770	fi
771	#
772	done # for var in ${liste_var} ; do
773	IGCM_sys_Rm ${liste_file_tmp[*]}
774	done # for file in ${liste_file_brut} do
775	done # for comp in ${config_ListOfComponents[*]} ; do
776	# GO AHEAD IN DEEPEST LOOP
777	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
778	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
779	# FINISH
780
781	# DODS copy
782	for comp in ${config_ListOfComponents[*]} ; do
783	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
784	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
785	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
786	done
787	done
788
789	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
790	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
791
792	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
793	export listVarEnv
794
795	#D-
796	# --------------------------------------------------------------------
797	#D- Test if all was right
798	# --------------------------------------------------------------------
799	IGCM_debug_Verif_Exit_Post
800	# --------------------------------------------------------------------
801	#D- Submit MONITORING
802	# --------------------------------------------------------------------
803	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
804	Script_Post_Output=monitoring.${PeriodDateEnd}
805	IGCM_sys_QsubPost monitoring
806	fi
807
808	#D-
809	# --------------------------------------------------------------------
810	#D- Update the run.card
811	# --------------------------------------------------------------------
812	if [ ${StandAlone} != true ] ; then
813	# Put in run.card end period of time series
814	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
815	# We have finish the job !
816	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
817	fi
818
819	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
820	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
821
822	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: