source: trunk/libIGCM/AA_create_ts @ 835

Last change on this file since 835 was 832, checked in by sdipsl, 11 years ago
  • Trivial changes to accomodate new_libIGCM function defined here /ccc/cont003/home/dsm/p86mart/.bash_alias (ksh-->bash)

She transforms every cp and rm by IGCM_sys_Cp and IGCM_sys_Rm within IGCM_debug_Check. So we need to source sys before calling it.

  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 31.1 KB
Line 
1#-Q- curie ######################
2#-Q- curie ## CURIE   TGCC/CEA ##
3#-Q- curie ######################
4#-Q- curie #MSUB -r TS             # Nom du job               
5#-Q- curie #MSUB -eo
6#-Q- curie #MSUB -n 1              # Reservation du processus
7#-Q- curie #MSUB -T 86400          # Limite de temps elapsed du job
8#-Q- curie #MSUB -q xlarge
9#-Q- curie #MSUB -Q normal
10#-Q- curie set +x
11#-Q- ada #!/bin/ksh
12#-Q- ada #######################
13#-Q- ada ##   ADA   IDRIS  ##
14#-Q- ada #######################
15#-Q- ada # Temps Elapsed max. d'une requete hh:mm:ss
16#-Q- ada # @ wall_clock_limit = 20:00:00
17#-Q- ada # Nom du travail LoadLeveler
18#-Q- ada # @ job_name   = TS
19#-Q- ada # Fichier de sortie standard du travail
20#-Q- ada # @ output     = $(job_name).$(jobid)
21#-Q- ada # Fichier de sortie d'erreur du travail
22#-Q- ada # @ error      =  $(job_name).$(jobid)
23#-Q- ada # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
24#-Q- ada # @ notification = error
25#-Q- ada # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $MASTER ; $RebuildFrequency ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $CompletedFlag ; $TsTask ; $CompToRead ; $FlagToRead ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_MBG ; $RESOL_SRF ; $RESOL_SBG ; $MASTER
26#-Q- ada # @ queue
27#-Q- sx9mercure #!/bin/ksh
28#-Q- sx9mercure ######################
29#-Q- sx9mercure ## SX9MERCURE  CCRT ##
30#-Q- sx9mercure ######################
31#-Q- sx9mercure #PBS -N TS                   # Nom du job
32#-Q- sx9mercure #PBS -j o                    # regroupement des stdout et stderr
33#-Q- sx9mercure #PBS -S /usr/bin/ksh         # shell de soumission
34#-Q- sx9mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
35#-Q- sx9mercure #PBS -l elapstim_req=24:00:00   # Limite temps a 2 heures
36#-Q- sx9mercure #PBS -q scalaire
37#-Q- sx9mercure #PBS -r n
38#-Q- titane #!/bin/ksh
39#-Q- titane ######################
40#-Q- titane ## TITANE   CEA ##
41#-Q- titane ######################
42#-Q- titane #MSUB -r TS             # Nom du job               
43#-Q- titane #MSUB -eo
44#-Q- titane #MSUB -n 1              # Reservation du processus
45#-Q- titane #MSUB -T 86400          # Limite de temps elapsed du job
46#-Q- titane #MSUB -q mono
47#-Q- titane #MSUB -E '-rn'
48#-Q- lxiv8 ######################
49#-Q- lxiv8 ## OBELIX      LSCE ##
50#-Q- lxiv8 ######################
51#-Q- lxiv8 #PBS -N TS
52#-Q- lxiv8 #PBS -m a
53#-Q- lxiv8 #PBS -j oe
54#-Q- lxiv8 #PBS -q medium
55#-Q- lxiv8 #PBS -o TS.$$
56#-Q- lxiv8 #PBS -S /bin/ksh
57#-Q- default #!/bin/ksh
58#-Q- default ##################
59#-Q- default ## DEFAULT HOST ##
60#-Q- default ##################
61
62#**************************************************************
63# Author: Sebastien Denvil
64# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
65# $Revision::                                          $ Revision of last commit
66# $Author::                                            $ Author of last commit
67# $Date::                                              $ Date of last commit
68# IPSL (2006)
69#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
70#
71#**************************************************************
72
73#set -eu
74#set -vx
75
76date
77
78#-Q- ada export OMP_NUM_THREADS=1
79
80#D- Task type (computing or post-processing)
81TaskType=post-processing
82
83########################################################################
84
85#D- Flag to determine if this job in a standalone mode
86#D- Default : value from AA_job if any
87StandAlone=${StandAlone:=true}
88
89#D- Path to libIGCM
90#D- Default : value from AA_job if any
91libIGCM=${libIGCM:=::modipsl::/libIGCM}
92
93#D- Flag to determine allready produced time series. Empty if you start from the beginning
94#D- Default : Value from AA_job if any. Usefull in StandAlone case if you want to continue a time series
95CompletedFlag=${CompletedFlag:=}
96
97#D- Flag to determine task type this script will perform.
98#D- Possible Value : 2D, 3D, Chunck2D, Chunck3D
99#D- Default : Value from AA_job if any. Usefull in StandAlone case.
100TsTask=${TsTask:=2D}
101if [ X${TsTask} = Xempty ] ; then
102  TsTask=""
103fi
104
105#D- Flag to determine if rebuild process has been performed asynchronously
106#D- Possible value true or false.
107#D- If true raw files has allready been patched by rebuild job
108#D- If false create_ts will patch the raw files
109#D- Default : Value from AA_job if any. Usefull in StandAlone case.
110RebuildFrequency=${RebuildFrequency:=true}
111
112#D- Flag to determine job's output directory
113#D- Default : value from libIGCM_post.ksh if any
114POST_DIR=${POST_DIR:=${PBS_O_WORKDIR:=$(pwd)}}
115
116#D- Increased verbosity (1, 2, 3)
117#D- Default : value from AA_job if any
118Verbosity=${Verbosity:=3}
119
120#D- Low level debug : to bypass lib test checks and stack construction
121#D- Default : value from AA_job if any
122DEBUG_debug=${DEBUG_debug:=false}
123
124#D- TEMPORARY Flag to determine atmospheric resolution
125#D- Default : value from atmospheric driver if any
126RESOL_ATM=ALL
127
128#D- Flag to determine surface resolution
129#D- Default : value from surface driver if any
130RESOL_SRF=ALL
131
132#D- Flag to determine surface resolution
133#D- Default : value from surface driver if any
134RESOL_SBG=ALL
135
136#D- TEMPORARY Flag to determine ocean resolution
137#D- Default : value from ocean driver if any
138RESOL_OCE=${RESOL_OCE:=ORCA2}
139
140#D- TEMPORARY Flag to determine ice resolution
141#D- Default : value from ice driver if any
142RESOL_ICE=${RESOL_ICE:=ORCA2}
143
144#D- TEMPORARY Flag to determine marine biogeochemistry resolution
145#D- Default : value from ice driver if any
146RESOL_MBG=${RESOL_MBG:=ORCA2}
147
148########################################################################
149
150. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
151. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
152. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
153#-------
154. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
155. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
156#-------
157( ${DEBUG_debug} ) && IGCM_debug_Check
158( ${DEBUG_debug} ) && IGCM_card_Check
159( ${DEBUG_debug} ) && IGCM_date_Check
160
161########################################################################
162
163#set -vx
164
165# Useful cleaning function
166MENAGE () 
167{
168  [ -f $3 ] && ( IGCM_sys_RshArchive rm -f $1/$2 ;)
169}
170
171########################################################################
172
173IGCM_sys_MkdirWork ${RUN_DIR_PATH}
174IGCM_sys_Cd ${RUN_DIR_PATH}
175
176# ------------------------------------------------------------------
177# Test if all was right before proceeding further
178# ------------------------------------------------------------------
179IGCM_debug_Verif_Exit_Post
180
181if [ ${StandAlone} = true ] ; then
182  CARD_DIR=${SUBMIT_DIR}
183else
184  CARD_DIR=${RUN_DIR_PATH}
185  IGCM_sys_Get_Master ${SUBMIT_DIR}/config.card ${RUN_DIR_PATH}
186  IGCM_sys_Get_Master ${SUBMIT_DIR}/run.card    ${RUN_DIR_PATH}
187  IGCM_sys_Get_Master ${SUBMIT_DIR}/COMP        ${RUN_DIR_PATH}
188  IGCM_sys_Get_Master ${SUBMIT_DIR}/POST        ${RUN_DIR_PATH}
189fi
190
191#==================================
192# First of all
193#
194# Read libIGCM compatibility version in config.card
195# Read UserChoices section
196# Read Ensemble section
197# Read Post section
198# Define all netcdf output directories
199#==================================
200IGCM_config_CommonConfiguration ${CARD_DIR}/config.card
201
202#==================================
203# Define default value to keep compatibility with previous card: means before changes due to TGCC
204if [ X${PackDefault} = Xtrue ] ; then
205  [ X${config_Post_PackFrequency} = X ] && config_Post_PackFrequency=${config_Post_RebuildFrequency}
206else
207  config_Post_PackFrequency=NONE
208fi
209
210#==================================
211# If pack is active then PackFrequency overule the config_UserChoices_PeriodLength
212if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
213  config_UserChoices_PeriodLength=${config_Post_PackFrequency}
214fi
215
216#
217# Determine component to take care of depending on kind of task create_ts will perform
218#
219[ ${CompToRead} ] && set -A config_ListOfComponents ${CompToRead} || IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card ListOfComponents
220#
221# Determine period for time series. Default : value from AA_job or from launch_TS if any
222#
223[ X${CompletedFlag} = X ] || CompletedFlag=$( IGCM_date_ConvertFormatToGregorian ${CompletedFlag} )
224
225DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
226#
227PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
228#
229# Determine Dimension of the produced time series : 2D or 3D
230if (   [ X${TsTask} = XChunck2D ] || [ X${TsTask} = X2D ] ) ; then
231  Dimension=2D
232elif ( [ X${TsTask} = XChunck3D ] || [ X${TsTask} = X3D ] ) ; then
233  Dimension=3D
234else
235  Dimension=""
236fi
237
238# ------------------------------------------------------------------
239#D- Test if all was right before entering the loop
240# ------------------------------------------------------------------
241IGCM_debug_Verif_Exit_Post
242
243########################################################################
244#
245#      Depending on requested time series fill following variables :
246#
247#                           - LISTE_FILE_${comp}[*]
248#                           - LISTE_VARS_${file}[*]
249#                           - LISTE_AXIS_${file}[*]
250#                           - LISTE_PATCH_${file}[*]
251########################################################################
252
253#
254# For each selected component determine which files need post-processing
255#
256DoJob=false
257for comp in ${config_ListOfComponents[*]} ; do
258  #
259  ActiveComp=false
260  #
261  IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
262  eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
263  #
264  card=${CARD_DIR}/COMP/${compname}.card
265
266  IGCM_card_DefineArrayFromOption ${card} OutputFiles List
267  #
268  ListFilesName=${compname}_OutputFiles_List
269  eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
270  #
271  if [ X${FileName0} != X${NULL_STR} ] ; then
272    #
273    #IGCM_debug_Print 1 "Component      : ${compname}"
274    #
275    # INITIALISATION
276    #
277    typeset i
278    if [ ${FlagToRead} ] ; then
279      ((NbFiles=${FlagToRead}))
280      ((i=${NbFiles}))
281    else
282      eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
283      i=2
284    fi
285    #
286    until [ $i -gt $NbFiles ]; do
287      #
288      eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1
289      #
290      if [ X${flag_post} != XNONE ] ; then
291        #
292        # For each comp and each file who needs post-processing
293        # determine which variables need time series.
294        #
295        FILE=$( echo ${flag_post} | awk "-FPost_" '{print $2}' )
296        IGCM_card_DefineArrayFromOption ${card}    Post_${FILE} TimeSeriesVars${Dimension}
297        IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension}
298        #
299        # If TimeSeriesVars list is empty we skip
300        #
301        if [ $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then
302          #
303          IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
304          IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty"
305          (( i=i+3 ))
306          continue
307          #
308        fi
309        #
310        # If files/variables has been switch off by ChunckJob${Dimension}=0 we skip
311        #
312        if [ X$( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = XOFF ] ; then
313          IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
314          IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off"
315          (( i=i+3 ))
316          continue
317        fi
318        #
319          # If we are not a Chunck type task, we skip files/variables handled by chunck task
320          #
321        if ( [ ! X${TsTask} = XChunck${Dimension} ] && [ ! X${TsTask} = X ] && [ ! XChunck${Dimension} = XChunck ] ) ; then
322          if [ ! $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} ) = NONE ] ; then
323            IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}"
324            IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )"
325            (( i=i+3 ))
326            continue
327          fi
328        fi
329        #
330        # Now on we know we have things to do
331        #
332
333        #
334        #  We need coordinates axis to properly fill produced time series
335        #
336        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
337        #
338        if [ $( eval echo \${${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
339          #
340          IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
341          IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
342          (( i=i+3 ))
343          continue
344          #
345        fi
346        #
347        # We need LIST of variables not allready produced (useful for standalone mode)
348        #
349        FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' )
350        case ${FlagDir} in
351          *Y)  TS_Dir=TS_YE  ;;
352          *M)  TS_Dir=TS_MO  ;;
353          *D)  TS_Dir=TS_DA  ;;
354          HF)  TS_Dir=TS_HF  ;;
355          3H)  TS_Dir=TS_HF  ;;
356          INS) TS_Dir=TS_INS ;;
357        esac
358        #
359        count=0
360        for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do
361          #
362          DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
363          TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
364          #
365          IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
366          if [ ! $? = 0 ] ; then
367            eval set +A LISTE_VARS_${FILE} \${LISTE_VARS_${FILE}[*]} ${var}
368            ActiveComp=true
369            (( count = count + 1 ))
370          fi
371        done
372        #
373        if [ ${count} -gt 0 ] ; then
374          #
375          eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE}
376          eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]}
377          #
378          if [ ! X${RebuildFrequency} = Xtrue ] ; then
379            IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
380            eval set +A LISTE_PATCHES_${FILE} \${${compname}_Post_${FILE}_Patches[*]}
381            if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) !=  X${NULL_STR} ]; then
382              for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
383                if [ Xload${Patch} != X ] ; then
384                  . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
385                  eval load${Patch}=loaded
386                fi
387              done
388            fi
389          fi
390        fi
391      fi
392      (( i=i+3 ))
393    done
394  fi
395  #
396  if [ ${ActiveComp} = true ] ; then
397    set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
398  fi
399done
400
401set -A config_ListOfComponents ${ActiveListOfComponents[*]}
402[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
403
404########################################################################
405#
406#               IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
407#                 IF SO BRING THEM IN THE WORKING DIRECTORY
408#
409# PS : Keep in mind that IGCM_sys_Get here is a weak link :
410#      - especially for IDRiS
411#      - no special protection against "command too long" !
412#      - libIGCM_sys should handle that or modify following code part
413#
414########################################################################
415
416if [ ${StandAlone} != true ] ; then
417  Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
418  ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
419  echo \${run_PostProcessing_TimeSeriesRunning}" )
420
421  #if [ X${Running_Flag} = Xy ] ; then
422  #  # Time Series Job allready running
423  #  IGCM_debug_Print 1 "Time Series Job allready running exit"
424  #  exit
425  #fi
426  # Now run.card know we are running
427
428
429
430
431  IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
432
433
434
435
436
437  # Get information from last execution
438  #CompletedFlag=$( IGCM_sys_RshMaster \
439  # "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
440  #        echo \${run_PostProcessing_TimeSeriesCompleted}" )
441fi
442
443# Is it the first submission or not ?
444
445if ( [ X${CompletedFlag} = X ] || [ X${CompletedFlag} = X$( IGCM_date_AddDaysToGregorianDate ${DateBegin} -1 ) ] ) ; then
446  # First Time Series Submission
447  FIRST_PASS=TRUE
448  Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)
449  DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
450else
451  # Last Completed Time Series
452  FIRST_PASS=FALSE
453  DATE_FIN_JOB_B=${CompletedFlag}
454  date=${DateBegin}_${DATE_FIN_JOB_B}
455  for comp in ${config_ListOfComponents[*]} ; do
456    #
457    IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
458    #
459    IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
460    #
461    #set -A liste_file
462    unset liste_file
463    #
464    file_path=${R_SAVE}/${comp}/Analyse
465    #
466    i=0
467    file=${NULL_STR}
468    #
469    for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
470      #
471      # Determine in which output can we find file
472      # IE : Analyse/TS or Analyse/DA2TS or ...
473      #
474      FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
475      case ${FlagDir} in
476        *Y)  TS_Dir=TS_YE  ;;
477        *M)  TS_Dir=TS_MO  ;;
478        *D)  TS_Dir=TS_DA  ;;
479        HF)  TS_Dir=TS_HF  ;;
480        3H)  TS_Dir=TS_HF  ;;
481        INS) TS_Dir=TS_INS ;;
482      esac
483      for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
484        #
485        #set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
486        liste_file[${#liste_file[*]}]=${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
487        #
488      done
489    done
490    if [ X${file} != X${NULL_STR} ] ; then
491      IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
492      # Even if some files are missing we tried to continue
493      # Either ListVar in compname.card are asked but not present in original history file
494      # Either a previous TS task failed
495      ExitFlag=false
496      IGCM_debug_Verif_Exit_Post
497    fi
498  done
499  #
500  LEVEL=0
501  #
502fi
503
504########################################################################
505#
506#                          DEFINE LOOP PARAMETERS
507#
508########################################################################
509
510DATE_COUNT=${DATE_FIN_JOB_B}
511DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
512
513( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1
514
515if [ ${DoJob} = true ] ; then
516  while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
517    (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
518    Length=$( IGCM_date_DaysInNextPeriod ${DATE_COUNT} ${config_UserChoices_PeriodLength} )
519    DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
520  done
521else
522  NBRE_FILE_TOT=0
523fi
524
525if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
526  NBRE_TOUR=1
527elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
528  let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} 
529else
530  let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
531fi
532
533CURRENT_LOOP=1
534# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
535# WE HAVE 3 LEVELS
536#    - LEVEL 1 : JUST ONE LOOP TO COMPLETE
537#    - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
538#    - LEVEL 3 : WE ARE IN THE LAST LOOP
539
540while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] ; do
541
542  if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then 
543    # - LEVEL 1 : JUST ONE LOOP TO COMPLETE
544    NBRE_FILE_LOOP=${NBRE_FILE_TOT}
545
546    if [ ${FIRST_PASS} = TRUE ] ; then
547      DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
548    else
549      Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
550      DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
551    fi
552
553    DATE_FIN=${PeriodDateEnd}
554    DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
555
556    [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
557    [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE"
558
559    LEVEL=1
560    DEPOT="TRUE"       
561  elif [ ${CURRENT_LOOP} -ne  ${NBRE_TOUR} ] ; then
562    # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
563    NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
564
565    if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
566      Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN_JOB_B_LOOP} ${config_UserChoices_PeriodLength} )
567      DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
568    elif [ -z "${DATE_COURANTE}" ] ; then
569      DATE_COURANTE=${DATE_FIN_JOB_B}
570    else
571      Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
572      DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
573    fi
574
575    (( TotLength = 0 ))
576    COMPTEUR=1
577    DATE_LOOP=${DATE_COURANTE}
578
579    while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
580      #
581      Length=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
582      DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
583      (( TotLength = TotLength + Length ))
584      (( COMPTEUR = COMPTEUR + 1 ))
585      #
586    done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
587    DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
588
589    [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
590    [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE"
591    DEPOT="FALSE"
592    LEVEL=2
593  else
594    # - LEVEL 3 : WE ARE IN THE LAST LOOP
595    NBRE_FILE_LOOP=$(( ${NBRE_FILE_TOT} % ( ${UNIX_MAX_LIMIT} * ( ${CURRENT_LOOP} - 1 ) ) ))
596    [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
597
598    Length=$( IGCM_date_DaysInNextPeriod ${DATE_FIN} ${config_UserChoices_PeriodLength} )
599    DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
600    DATE_FIN=${PeriodDateEnd}
601    [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
602
603    LEVEL=3
604    DEPOT="TRUE"
605    FLAG_B="TRUE"
606  fi
607
608  # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
609  for comp in ${config_ListOfComponents[*]} ; do
610    #
611    IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
612      #
613    IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
614    #
615    eval R_OUT_${comp}=${R_SAVE}/${comp}
616    #
617    i=0
618    #
619    for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
620      #
621      # Determine in which output we can find file
622      # IE : Output/MO or Output/DA or ...
623      #
624      FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
625      case ${FlagDir} in
626        *Y)    TS_Dir=TS_YE  ; FreqDir=YE  ;;
627        *M)    TS_Dir=TS_MO  ; FreqDir=MO  ;;
628        *D)    TS_Dir=TS_DA  ; FreqDir=DA  ;;
629        HF)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
630        3H)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
631        INS)   TS_Dir=TS_INS ; FreqDir=INS ;;
632        *)     IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
633               IGCM_sys_Exit "Job create_ts" ;;
634      esac
635
636      # Initialize array
637      #set -A liste_file
638      #set -A liste_file_tmp
639      unset liste_file
640      unset liste_file_tmp
641      eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
642
643      COMPTEUR=1
644      MissingFile=FALSE
645      DATE_LOOP=${DATE_COURANTE}
646
647      while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
648        #
649        Length1=$( IGCM_date_DaysInPreviousPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} end)
650        DATE_TAB=$(  IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
651
652        Length2=$( IGCM_date_DaysInNextPeriod ${DATE_LOOP} ${config_UserChoices_PeriodLength} )
653        DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
654
655        #set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
656        #set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
657
658        IGCM_sys_TestFileArchive ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
659        if [ $? = 0 ] ; then
660          liste_file[${#liste_file[*]}]=${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
661          liste_file_tmp[${#liste_file_tmp[*]}]=${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
662        else
663          IGCM_debug_Print 1 "WARNING ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc dont exist"
664          MissingFile=TRUE
665        fi
666        (( COMPTEUR = COMPTEUR + 1 ))
667        #
668      done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
669
670      # We skip this file type if one of them is missing.
671      if [ "${MissingFile}" = "TRUE" ] ; then
672        IGCM_debug_Print 1 "WARNING We skip ${file} type processing"
673        continue
674      fi
675
676      DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
677
678      # Get selected files
679      IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
680      IGCM_debug_Verif_Exit_Post
681
682      # Apply IGCM_Patch if needed
683      if [ ! X${RebuildFrequency} = Xtrue ] ; then
684        if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) !=  X${NULL_STR} ]; then
685          for file_t in $( eval echo ${liste_file_tmp[*]} ); do         
686            for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
687              IGCM_${Patch} ${file_t}
688              IGCM_debug_Verif_Exit_Post
689            done
690          done
691        fi
692      fi
693
694      # WE CAN CONCATENATE FILES AT THIS POINT
695      [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
696      Length=$( IGCM_date_DaysInPreviousPeriod ${DATE_COURANTE} ${config_UserChoices_PeriodLength} end)
697      DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
698      DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
699      DATE_BUILD_END=${DateBegin}_${DATE_FIN}
700
701      liste_coord=" "
702      for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
703        liste_coord=${liste_coord}${axis}","
704      done
705
706      # Time axis must be the last dimension
707      time_axis=${axis}
708
709      # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
710      for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
711        #
712        if [ "${FLAG_B}" = "TRUE" ] ; then
713          # WE CONCATENATE WITH EXISTING FILES
714          file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
715          file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
716          IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
717          #
718          IGCM_sys_Rm ${file1}
719          if [ ! "${DEPOT}" = "TRUE" ] ; then
720            eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
721            # "${file1} = file_before ?
722            [ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
723          fi
724
725          ExitFlag=false
726        else
727          # OR NOT
728          IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
729        fi
730
731        #
732        if [ "${DEPOT}" = "TRUE" ] ; then
733          # WE PUT FINAL FILE ON FILE SYSTEM
734          [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
735          file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
736          #
737          eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
738          [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
739          if [ ! "X${file1}" = "X" ] ; then
740            [ ! "${file1}" = "${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file1} ${file_out}
741          fi
742          #
743          if ( [ ${FlagDir} = "1M" ] && [ -f ${file_out} ] ) ; then
744            file_before_YE=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_1Y_${var}.nc
745            file_out_YE=${config_UserChoices_JobName}_${DATE_BUILD_END}_1Y_${var}.nc
746            #
747            # Compute yearly average considering calendar
748            # and set date of the mid year to 01/07 at 00:00
749            IGCM_sys_cdo -settime,00:00 -setday,1 -setmon,7 -settunits,days -divdpy -yearsum -muldpm -selvar,${var} ${file_out} ${file_out_YE}
750            # Extract all ancillary variables except $var, they will be appended after the cdo command
751            # Needed because cdo applies calculs on all variables
752            IGCM_sys_ncks -Oh -x -v ${var} ${file_out} ${file_out_YE%%.nc}_tmp1.nc
753            # This will remove the time axis because cdo rewrites severely with only considered useful variables
754            IGCM_sys_cdo -mulc,1 ${file_out_YE%%.nc}_tmp1.nc ${file_out_YE%%.nc}_tmp2.nc 2>/dev/null
755            if [ $? -eq 0 ] ; then
756              # Now append all ancillary variables if previous cdo operator ok
757              IGCM_sys_ncks -h -A ${file_out_YE%%.nc}_tmp2.nc ${file_out_YE}
758            fi
759
760            # Put global history attribut
761            IGCM_sys_ncatted -Oh -a history,global,o,c,"build by libIGCM/create_ts" ${file_out_YE}
762            #
763            eval IGCM_sys_Put_Out ${file_out_YE} \${R_OUT_${comp}}/Analyse/TS_MO_YE/${file_out_YE}
764            [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/TS_MO_YE/ ${file_before_YE} ${file_out_YE}
765            IGCM_sys_Rm ${file_out_YE%%.nc}_tmp*.nc
766          fi
767          #
768          IGCM_sys_Rm ${file_out}
769          ExitFlag=false
770          #
771        fi
772        #
773      done     # for var in ${liste_var} ; do
774      IGCM_sys_Rm ${liste_file_tmp[*]}
775    done         # for file in ${liste_file_brut} do
776  done             # for comp in ${config_ListOfComponents[*]} ; do
777  # GO AHEAD IN DEEPEST LOOP
778  (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
779done                 # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do
780# FINISH
781
782# DODS copy
783for comp in ${config_ListOfComponents[*]} ; do
784  for TS_Dir in TS_YE TS_MO TS_MO_YE TS_DA TS_HF TS_INS ; do
785    IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
786    [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
787  done
788done
789
790export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
791export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_MBG ; export RESOL_SRF ; export RESOL_SBG
792
793listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_MBG,RESOL_SRF,RESOL_SBG"
794export listVarEnv
795
796#D-
797# --------------------------------------------------------------------
798#D- Test if all was right
799# --------------------------------------------------------------------
800IGCM_debug_Verif_Exit_Post
801# --------------------------------------------------------------------
802#D- Submit MONITORING
803# --------------------------------------------------------------------
804if ( [ X${TsTask} = X2D ] || [ X${TsTask} = X3D ] || [ X${TsTask} = X ] ) ; then
805  Script_Post_Output=monitoring.${PeriodDateEnd}
806  IGCM_sys_QsubPost monitoring
807fi
808
809#D-
810# --------------------------------------------------------------------
811#D- Update the run.card
812# --------------------------------------------------------------------
813if [ ${StandAlone} != true ] ; then
814  # Put in run.card end period of time series
815  [ X"${DATE_FIN}" = X ] || IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
816  # We have finish the job !
817  IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
818fi
819
820# Clean RUN_DIR_PATH (necessary for cesium and titane only)
821IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH}
822
823date
Note: See TracBrowser for help on using the repository browser.