Context Navigation

source: trunk/libIGCM/AA_create_ts @ 918

Last change on this file since 918 was 906, checked in by sdipsl, 11 years ago
cleanup whitespace-cleanup
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 32.2 KB

Line
1	#-Q- curie ######################
2	#-Q- curie ## CURIE TGCC/CEA ##
3	#-Q- curie ######################
4	#-Q- curie #MSUB -r TS # Nom du job
5	#-Q- curie #MSUB -eo
6	#-Q- curie #MSUB -n 1 # Reservation du processus
7	#-Q- curie #MSUB -T 80000 # Limite de temps elapsed du job
8	#-Q- curie #MSUB -q xlarge
9	#-Q- curie #MSUB -Q normal
10	#-Q- curie #MSUB -A ::default_project::
11	#-Q- curie set +x
12	#-Q- ada #!/bin/ksh
13	#-Q- ada #######################
14	#-Q- ada ## ADA IDRIS ##
15	#-Q- ada #######################
16	#-Q- ada # @ job_type = serial
17	#-Q- ada # @ requirements = (Feature == "prepost")
18	#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
19	#-Q- ada # @ wall_clock_limit = 20:00:00
20	#-Q- ada # Nom du travail LoadLeveler
21	#-Q- ada # @ job_name = TS
22	#-Q- ada # Fichier de sortie standard du travail
23	#-Q- ada # @ output = $(job_name).$(jobid)
24	#-Q- ada # Fichier de sortie d'erreur du travail
25	#-Q- ada # @ error = $(job_name).$(jobid)
26	#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
27	#-Q- ada # @ notification = error
28	#-Q- ada # @ environment = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
29	#-Q- ada # @ queue
30	#-Q- sx9mercure #!/bin/ksh
31	#-Q- sx9mercure ######################
32	#-Q- sx9mercure ## SX9MERCURE CCRT ##
33	#-Q- sx9mercure ######################
34	#-Q- sx9mercure #PBS -N TS # Nom du job
35	#-Q- sx9mercure #PBS -j o # regroupement des stdout et stderr
36	#-Q- sx9mercure #PBS -S /usr/bin/ksh # shell de soumission
37	#-Q- sx9mercure #PBS -l memsz_job=1gb # Limite memoire a 1 Go
38	#-Q- sx9mercure #PBS -l elapstim_req=24:00:00 # Limite temps a 2 heures
39	#-Q- sx9mercure #PBS -q scalaire
40	#-Q- sx9mercure #PBS -r n
41	#-Q- titane #!/bin/ksh
42	#-Q- titane ######################
43	#-Q- titane ## TITANE CEA ##
44	#-Q- titane ######################
45	#-Q- titane #MSUB -r TS # Nom du job
46	#-Q- titane #MSUB -eo
47	#-Q- titane #MSUB -n 1 # Reservation du processus
48	#-Q- titane #MSUB -T 86400 # Limite de temps elapsed du job
49	#-Q- titane #MSUB -q mono
50	#-Q- titane #MSUB -E '-rn'
51	#-Q- lxiv8 ######################
52	#-Q- lxiv8 ## OBELIX LSCE ##
53	#-Q- lxiv8 ######################
54	#-Q- lxiv8 #PBS -N TS
55	#-Q- lxiv8 #PBS -m a
56	#-Q- lxiv8 #PBS -j oe
57	#-Q- lxiv8 #PBS -q medium
58	#-Q- lxiv8 #PBS -o TS.$$
59	#-Q- lxiv8 #PBS -S /bin/ksh
60	#-Q- default #!/bin/ksh
61	#-Q- default ##################
62	#-Q- default ## DEFAULT HOST ##
63	#-Q- default ##################
64
65	#**************************************************************
66	# Author: Sebastien Denvil
67	# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
68	# $Revision:: $ Revision of last commit
69	# $Author:: $ Author of last commit
70	# $Date:: $ Date of last commit
71	# IPSL (2006)
72	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
73	#
74	#**************************************************************
75
76	#set -eu
77	#set -vx
78
79	date
80
81	#-Q- ada export OMP_NUM_THREADS=1
82
83	#D- Task type (computing or post-processing)
84	TaskType=post-processing
85
86	########################################################################
87
88	#D- Flag to determine if this job in a standalone mode
89	#D- Default : value from AA_job if any
90	StandAlone=${StandAlone:=true}
91
92	#D- Path to libIGCM
93	#D- Default : value from AA_job if any
94	libIGCM=${libIGCM:=::modipsl::/libIGCM}
95
96	#D- Flag to determine allready produced time series. Empty if you start from the beginning
97	#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
98	CompletedFlag=${CompletedFlag:=}
99
100	#D- Flag to determine task type this script will perform.
101	#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
102	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
103	TsTask=${TsTask:=2D}
104	if [ X${TsTask} = Xempty ] ; then
105	TsTask=""
106	fi
107
108	#D- Flag to determine if rebuild process has been performed asynchronously
109	#D- Possible value true or false.
110	#D- If true raw files has allready been patched by rebuild job
111	#D- If false create_ts will patch the raw files
112	#D- Default : Value from AA_job if any. Usefull in StandAlone case.
113	RebuildFrequency=${RebuildFrequency:=true}
114
115	#D- Flag to determine job's output directory
116	#D- Default : value from libIGCM_post.ksh if any
117	POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
118
119	#D- Increased verbosity (1, 2, 3)
120	#D- Default : value from AA_job if any
121	Verbosity=${Verbosity:=3}
122
123	#D- Low level debug : to bypass lib test checks and stack construction
124	#D- Default : value from AA_job if any
125	DEBUG_debug=${DEBUG_debug:=false}
126
127	#D- TEMPORARY Flag to determine atmospheric resolution
128	#D- Default : value from atmospheric driver if any
129	RESOL_ATM=ALL
130
131	#D- Flag to determine surface resolution
132	#D- Default : value from surface driver if any
133	RESOL_SRF=ALL
134
135	#D- Flag to determine surface resolution
136	#D- Default : value from surface driver if any
137	RESOL_SBG=ALL
138
139	#D- TEMPORARY Flag to determine ocean resolution
140	#D- Default : value from ocean driver if any
141	RESOL_OCE=${RESOL_OCE:=ORCA2}
142
143	#D- TEMPORARY Flag to determine ice resolution
144	#D- Default : value from ice driver if any
145	RESOL_ICE=${RESOL_ICE:=ORCA2}
146
147	#D- TEMPORARY Flag to determine marine biogeochemistry resolution
148	#D- Default : value from ice driver if any
149	RESOL_MBG=${RESOL_MBG:=ORCA2}
150
151	########################################################################
152
153	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
154	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
155	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
156	#-------
157	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
158	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
159	#-------
160	( ${DEBUG_debug} ) && IGCM_debug_Check
161	( ${DEBUG_debug} ) && IGCM_card_Check
162	( ${DEBUG_debug} ) && IGCM_date_Check
163
164	########################################################################
165
166	#set -vx
167
168	# Useful cleaning function
169	MENAGE ()
170	{
171	[ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
172	}
173
174	########################################################################
175
176	IGCM_sys_MkdirWork ${RUN_DIR_PATH}
177	IGCM_sys_Cd ${RUN_DIR_PATH}
178
179	# ------------------------------------------------------------------
180	# Test if all was right before proceeding further
181	# ------------------------------------------------------------------
182	IGCM_debug_Verif_Exit_Post
183
184	if [ ${StandAlone} = true ] ; then
185	CARD_DIR=${SUBMIT_DIR}
186	else
187	CARD_DIR=${RUN_DIR_PATH}
188
189	#startTime_ms=$( getDate_ms ) #gp:added#
190	IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
191	#endTime_ms=$( getDate_ms ) #gp:added#
192	#printInfosAction IGCM_sys_Get ${SUBMIT_DIR}/config.card $startTime_ms $endTime_ms ${RUN_DIR_PATH} ${SUBMIT_DIR}/config.card #gp:added#
193
194	#startTime_ms=$( getDate_ms ) #gp:added#
195	IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card ${RUN_DIR_PATH}
196	#endTime_ms=$( getDate_ms ) #gp:added#
197	#printInfosAction IGCM_sys_Get ${SUBMIT_DIR}/run.card $startTime_ms $endTime_ms ${RUN_DIR_PATH} ${SUBMIT_DIR}/run.card #gp:added#
198
199	#startTime_ms=$( getDate_ms ) #gp:added#
200	IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP ${RUN_DIR_PATH}
201	#endTime_ms=$( getDate_ms ) #gp:added#
202	#printInfosAction IGCM_sys_Get ${SUBMIT_DIR}/COMP $startTime_ms $endTime_ms ${RUN_DIR_PATH} ${SUBMIT_DIR}/COMP #gp:added#
203
204	#startTime_ms=$( getDate_ms ) #gp:added#
205	IGCM_sys_Get_Master ${SUBMIT_DIR}/POST ${RUN_DIR_PATH}
206	#endTime_ms=$( getDate_ms ) #gp:added#
207	#printInfosAction IGCM_sys_Get ${SUBMIT_DIR}/POST $startTime_ms $endTime_ms ${RUN_DIR_PATH} ${SUBMIT_DIR}/POST #gp:added#
208	fi
209
210	#==================================
211	# First of all
212	#
213	# Read libIGCM compatibility version in config.card
214	# Read UserChoices section
215	# Read Ensemble section
216	# Read Post section
217	# Define all netcdf output directories
218	#==================================
219	IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
220
221	#==================================
222	# Define default value to keep compatibility with previous card: means before changes due to TGCC
223	if [ X${PackDefault} = Xtrue ] ; then
224	[ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
225	else
226	config_Post_PackFrequency=NONE
227	fi
228
229	#==================================
230	# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
231	if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
232	config_UserChoices_PeriodLength=${config_Post_PackFrequency}
233	fi
234
235	#
236	# Determine component to take care of depending on kind of task create_ts will perform
237	#
238	[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} \|\| IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
239	#
240	# Determine period for time series. Default : value from AA_job or from launch_TS if any
241	#
242	[ X${CompletedFlag} = X ] \|\| CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
243
244	DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
245	#
246	PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
247	#
248	# Determine Dimension of the produced time series : 2D or 3D
249	if ( [ X${TsTask} = XChunck2D ] \|\| [ X${TsTask} = X2D ] ) ; then
250	Dimension=2D
251	elif ( [ X${TsTask} = XChunck3D ] \|\| [ X${TsTask} = X3D ] ) ; then
252	Dimension=3D
253	else
254	Dimension=""
255	fi
256
257	# ------------------------------------------------------------------
258	#D- Test if all was right before entering the loop
259	# ------------------------------------------------------------------
260	IGCM_debug_Verif_Exit_Post
261
262	########################################################################
263	#
264	# Depending on requested time series fill following variables :
265	#
266	# - LISTE_FILE_${comp}[*]
267	# - LISTE_VARS_${file}[*]
268	# - LISTE_AXIS_${file}[*]
269	# - LISTE_PATCH_${file}[*]
270	########################################################################
271
272	#
273	# For each selected component determine which files need post-processing
274	#
275	DoJob=false
276	for comp in ${config_ListOfComponents[*]} ; do
277	#
278	ActiveComp=false
279	#
280	IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
281	eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
282	#
283	card=${CARD_DIR}/COMP/${compname}.card
284
285	IGCM_card_DefineArrayFromOption ${card} OutputFiles List
286	#
287	ListFilesName=${compname}_OutputFiles_List
288	eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
289	#
290	if [ X${FileName0} != X${NULL_STR} ] ; then
291	#
292	#IGCM_debug_Print 1 "Component : ${compname}"
293	#
294	# INITIALISATION
295	#
296	typeset i
297	if [ ${FlagToRead} ] ; then
298	((NbFiles=${FlagToRead}))
299	((i=${NbFiles}))
300	else
301	eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
302	i=2
303	fi
304	#
305	until [ $i -gt $NbFiles ]; do
306	#
307	eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
308	#
309	if [ X${flag_post} != XNONE ] ; then
310	#
311	# For each comp and each file who needs post-processing
312	# determine which variables need time series.
313	#
314	FILE=$( echo ${flag_post} \| awk "-FPost_" '{print $2}' )
315	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension}
316	IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
317	#
318	# If TimeSeriesVars list is empty we skip
319	#
320	if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
321	#
322	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
323	IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
324	(( i=i+3 ))
325	continue
326	#
327	fi
328	#
329	# If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
330	#
331	if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
332	IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
333	IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
334	(( i=i+3 ))
335	continue
336	fi
337	#
338	# If we are not a Chunck type task, we skip files/variables handled by chunck task
339	#
340	if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
341	if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
342	IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
343	IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
344	(( i=i+3 ))
345	continue
346	fi
347	fi
348	#
349	# Now on we know we have things to do
350	#
351
352	#
353	# We need coordinates axis to properly fill produced time series
354	#
355	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
356	#
357	if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
358	#
359	IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
360	IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
361	(( i=i+3 ))
362	continue
363	#
364	fi
365	#
366	# We need LIST of variables not allready produced (useful for standalone mode)
367	#
368	FlagDir=$( echo ${FILE} \| awk -F "_" '{print $1}' )
369	case ${FlagDir} in
370	*Y) TS_Dir=TS_YE ;;
371	*M) TS_Dir=TS_MO ;;
372	*D) TS_Dir=TS_DA ;;
373	HF) TS_Dir=TS_HF ;;
374	3H) TS_Dir=TS_HF ;;
375	INS) TS_Dir=TS_INS ;;
376	esac
377	#
378	count=0
379	for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
380	#
381	DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
382	TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
383	#
384	IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
385	if [ ! $? = 0 ] ; then
386	eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
387	ActiveComp=true
388	(( count = count + 1 ))
389	fi
390	done
391	#
392	if [ ${count} -gt 0 ] ; then
393	#
394	eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
395	eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
396	#
397	if [ ! X${RebuildFrequency} = Xtrue ] ; then
398	IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
399	eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
400	if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) != X${NULL_STR} ]; then
401	for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
402	if [ Xload${Patch} != X ] ; then
403	. ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
404	eval load${Patch}=loaded
405	fi
406	done
407	fi
408	fi
409	fi
410	fi
411	(( i=i+3 ))
412	done
413	fi
414	#
415	if [ ${ActiveComp} = true ] ; then
416	set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
417	fi
418	done
419
420	set -A config_ListOfComponents ${ActiveListOfComponents[*]}
421	[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
422
423	########################################################################
424	#
425	# IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
426	# IF SO BRING THEM IN THE WORKING DIRECTORY
427	#
428	# PS : Keep in mind that IGCM_sys_Get here is a weak link :
429	# - especially for IDRiS
430	# - no special protection against "command too long" !
431	# - libIGCM_sys should handle that or modify following code part
432	#
433	########################################################################
434
435	if [ ${StandAlone} != true ] ; then
436	Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
437	${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
438	echo \${run_PostProcessing_TimeSeriesRunning}" )
439
440	#if [ X${Running_Flag} = Xy ] ; then
441	# # Time Series Job allready running
442	# IGCM_debug_Print 1 "Time Series Job allready running exit"
443	# exit
444	#fi
445	# Now run.card know we are running
446
447
448
449
450	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
451
452
453
454
455
456	# Get information from last execution
457	#CompletedFlag=$( IGCM_sys_RshMaster \
458	# "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
459	# echo \${run_PostProcessing_TimeSeriesCompleted}" )
460	fi
461
462	# Is it the first submission or not ?
463
464	if ( [ X${CompletedFlag} = X ] \|\| [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
465	# First Time Series Submission
466	FIRST_PASS=TRUE
467	Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
468	DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
469	else
470	# Last Completed Time Series
471	FIRST_PASS=FALSE
472	DATE_FIN_JOB_B=${CompletedFlag}
473	date=${DateBegin}_${DATE_FIN_JOB_B}
474	for comp in ${config_ListOfComponents[*]} ; do
475	#
476	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
477	#
478	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
479	#
480	unset liste_file
481	#
482	file_path=${R_SAVE}/${comp}/Analyse
483	#
484	i=0
485	file=${NULL_STR}
486	#
487	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
488	#
489	# Determine in which output can we find file
490	# IE : Analyse/TS or Analyse/DA2TS or ...
491	#
492	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
493	case ${FlagDir} in
494	*Y) TS_Dir=TS_YE ;;
495	*M) TS_Dir=TS_MO ;;
496	*D) TS_Dir=TS_DA ;;
497	HF) TS_Dir=TS_HF ;;
498	3H) TS_Dir=TS_HF ;;
499	INS) TS_Dir=TS_INS ;;
500	esac
501	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
502	#
503	liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
504	#
505	done
506	done
507	if [ X${file} != X${NULL_STR} ] ; then
508	#my_size=$( size_of_tab_content liste_file[*] ) #gp:added#
509	#startTime_ms=$( getDate_ms ) #gp:added#
510	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
511	#endTime_ms=$( getDate_ms ) #gp:added#
512	#printInfos_GetList IGCM_sys_Get $my_size $startTime_ms $endTime_ms ${RUN_DIR_PATH}/${comp} ${file_path} #gp:added#
513
514	# Even if some files are missing we tried to continue
515	# Either ListVar in compname.card are asked but not present in original history file
516	# Either a previous TS task failed
517	ExitFlag=false
518	IGCM_debug_Verif_Exit_Post
519	fi
520	done
521	#
522	LEVEL=0
523	#
524	fi
525
526	########################################################################
527	#
528	# DEFINE LOOP PARAMETERS
529	#
530	########################################################################
531
532	DATE_COUNT=${DATE_FIN_JOB_B}
533	DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
534
535	( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 \|\| NBRE_FILE_TOT=1
536
537	if [ ${DoJob} = true ] ; then
538	while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
539	(( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
540	Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
541	DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
542	done
543	else
544	NBRE_FILE_TOT=0
545	fi
546
547	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
548	NBRE_TOUR=1
549	elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
550	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}
551	else
552	let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
553	fi
554
555	CURRENT_LOOP=1
556	# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
557	# WE HAVE 3 LEVELS
558	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
559	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
560	# - LEVEL 3 : WE ARE IN THE LAST LOOP
561
562	while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] ; do
563
564	if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
565	# - LEVEL 1 : JUST ONE LOOP TO COMPLETE
566	NBRE_FILE_LOOP=${NBRE_FILE_TOT}
567
568	if [ ${FIRST_PASS} = TRUE ] ; then
569	DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
570	else
571	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
572	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
573	fi
574
575	DATE_FIN=${PeriodDateEnd}
576	DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
577
578	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
579	[ -n "${LEVEL}" ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
580
581	LEVEL=1
582	DEPOT="TRUE"
583	elif [ ${CURRENT_LOOP} -ne ${NBRE_TOUR} ] ; then
584	# - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
585	NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
586
587	if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
588	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
589	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
590	elif [ -z "${DATE_COURANTE}" ] ; then
591	DATE_COURANTE=${DATE_FIN_JOB_B}
592	else
593	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
594	DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
595	fi
596
597	(( TotLength = 0 ))
598	COMPTEUR=1
599	DATE_LOOP=${DATE_COURANTE}
600
601	while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
602	#
603	Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
604	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
605	(( TotLength = TotLength + Length ))
606	(( COMPTEUR = COMPTEUR + 1 ))
607	#
608	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
609	DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
610
611	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
612	[ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" \|\| [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" \|\| FLAG_B="FALSE"
613	DEPOT="FALSE"
614	LEVEL=2
615	else
616	# - LEVEL 3 : WE ARE IN THE LAST LOOP
617	NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
618	[ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
619
620	Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
621	DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
622	DATE_FIN=${PeriodDateEnd}
623	[ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
624
625	LEVEL=3
626	DEPOT="TRUE"
627	FLAG_B="TRUE"
628	fi
629
630	# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
631	for comp in ${config_ListOfComponents[*]} ; do
632	#
633	IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
634	#
635	IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
636	#
637	eval R_OUT_${comp}=${R_SAVE}/${comp}
638	#
639	i=0
640	#
641	for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
642	#
643	# Determine in which output we can find file
644	# IE : Output/MO or Output/DA or ...
645	#
646	FlagDir=$( echo ${file} \| awk -F "_" '{print $1}' )
647	case ${FlagDir} in
648	*Y) TS_Dir=TS_YE ; FreqDir=YE ;;
649	*M) TS_Dir=TS_MO ; FreqDir=MO ;;
650	*D) TS_Dir=TS_DA ; FreqDir=DA ;;
651	HF) TS_Dir=TS_HF ; FreqDir=HF ;;
652	3H) TS_Dir=TS_HF ; FreqDir=HF ;;
653	INS) TS_Dir=TS_INS ; FreqDir=INS ;;
654	*) IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
655	IGCM_sys_Exit "Job create_ts" ;;
656	esac
657
658	# Initialize array
659	unset liste_file
660	unset liste_file_tmp
661	eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
662
663	COMPTEUR=1
664	MissingFile=FALSE
665	DATE_LOOP=${DATE_COURANTE}
666
667	while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
668	#
669	Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
670	DATE_TAB=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
671
672	Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
673	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
674
675	IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
676	if [ $? = 0 ] ; then
677	liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
678	liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
679	else
680	IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
681	MissingFile=TRUE
682	fi
683	(( COMPTEUR = COMPTEUR + 1 ))
684	#
685	done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
686
687	# We skip this file type if one of them is missing.
688	if [ "${MissingFile}" = "TRUE" ] ; then
689	IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
690	continue
691	fi
692
693	DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
694
695	# Get selected files
696	#my_size=$( size_of_tab_content liste_file[*] ) #gp:added#
697	#startTime_ms=$( getDate_ms ) #gp:added#
698	IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
699	#ndTime_ms=$( getDate_ms ) #gp:added#
700	#printInfos_GetList IGCM_sys_Get $my_size $startTime_ms $endTime_ms ${RUN_DIR_PATH}/${comp} ${file_path} #gp:added#
701
702	IGCM_debug_Verif_Exit_Post
703
704	# Apply IGCM_Patch if needed
705	if [ ! X${RebuildFrequency} = Xtrue ] ; then
706	if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) != X${NULL_STR} ]; then
707	for file_t in $( eval echo ${liste_file_tmp[*]} ); do
708	for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
709	IGCM_${Patch} ${file_t}
710	IGCM_debug_Verif_Exit_Post
711	done
712	done
713	fi
714	fi
715
716	# WE CAN CONCATENATE FILES AT THIS POINT
717	[ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
718	Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
719	DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
720	DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
721	DATE_BUILD_END=${DateBegin}_${DATE_FIN}
722
723	liste_coord=" "
724	for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
725	liste_coord=${liste_coord}${axis}","
726	done
727
728	# Time axis must be the last dimension
729	time_axis=${axis}
730
731	# SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
732	for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
733	#
734	if [ "${FLAG_B}" = "TRUE" ] ; then
735	# WE CONCATENATE WITH EXISTING FILES
736	file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
737	file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
738	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
739	#
740	IGCM_sys_Rm ${file1}
741	if [ ! "${DEPOT}" = "TRUE" ] ; then
742	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
743	# "${file1} = file_before ?
744	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
745	fi
746
747	ExitFlag=false
748	else
749	# OR NOT
750	IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
751	fi
752
753	#
754	if [ "${DEPOT}" = "TRUE" ] ; then
755	# WE PUT FINAL FILE ON FILE SYSTEM
756	[ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
757	file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
758	#
759	eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
760	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
761	if [ ! "X${file1}" = "X" ] ; then
762	[ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
763	fi
764	#
765	if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
766	file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
767	file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
768	#
769	# Compute yearly average considering calendar
770	# and set date of the mid year to 01/07 at 00:00
771	IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
772	# Extract all ancillary variables except $var, they will be appended after the cdo command
773	# Needed because cdo applies calculs on all variables
774	IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
775	# This will remove the time axis because cdo rewrites severely with only considered useful variables
776	IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
777	if [ $? -eq 0 ] ; then
778	# Now append all ancillary variables if previous cdo operator ok
779	IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
780	fi
781
782	# Put global history attribut
783	IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
784	#
785	eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
786	[ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
787	IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
788	fi
789	#
790	IGCM_sys_Rm ${file_out}
791	ExitFlag=false
792	#
793	fi
794	#
795	done # for var in ${liste_var} ; do
796	IGCM_sys_Rm ${liste_file_tmp[*]}
797	done # for file in ${liste_file_brut} do
798	done # for comp in ${config_ListOfComponents[*]} ; do
799	# GO AHEAD IN DEEPEST LOOP
800	(( CURRENT_LOOP = CURRENT_LOOP + 1 ))
801	done # while [ ${CURRENT_LOOP} -le ${NBRE_TOUR} ] do
802	# FINISH
803
804	# DODS copy
805	for comp in ${config_ListOfComponents[*]} ; do
806	for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
807	IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
808	[ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
809	done
810	done
811
812	export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
813	export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
814
815	listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
816	export listVarEnv
817
818	#D-
819	# --------------------------------------------------------------------
820	#D- Test if all was right
821	# --------------------------------------------------------------------
822	IGCM_debug_Verif_Exit_Post
823	# --------------------------------------------------------------------
824	#D- Submit MONITORING
825	# --------------------------------------------------------------------
826	if ( [ X${TsTask} = X2D ] \|\| [ X${TsTask} = X3D ] \|\| [ X${TsTask} = X ] ) ; then
827	Script_Post_Output=monitoring.${PeriodDateEnd}
828	IGCM_sys_QsubPost monitoring
829	fi
830
831	#D-
832	# --------------------------------------------------------------------
833	#D- Update the run.card
834	# --------------------------------------------------------------------
835	if [ ${StandAlone} != true ] ; then
836	# Put in run.card end period of time series
837	[ X"${DATE_FIN}" = X ] \|\| IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
838	# We have finish the job !
839	IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
840	fi
841
842	# Clean RUN_DIR_PATH (necessary for cesium and titane only)
843	IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
844
845	date

Note: See TracBrowser for help on using the repository browser.

Download in other formats: