source: trunk/libIGCM/AA_create_ts @ 67

Last change on this file since 67 was 67, checked in by sdipsl, 15 years ago

libIGCM is ready for ulam except concerning the monitoring

  • fix issue due to mfls (chaotic behaviour) do not use mfls anymore
  • change due to the fact that ulam do not see gaya : imply some libsys homogeneisation
  • atlas/monitoring01 needs an adaptation to ulam (because of "ls $homegaya" like command)
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Date Author Revision
File size: 26.0 KB
Line 
1#-Q- platine #!/usr/bin/ksh
2#-Q- platine ##################
3#-Q- platine ## PLATINE   CEA ##
4#-Q- platine ##################
5#-Q- platine #BSUB -J TS                     # Nom du job
6#-Q- platine #BSUB -N                        # message a la fin du job
7#-Q- platine #BSUB -n 1                      # reservation des processeurs pour le job
8#-Q- platine #BSUB -W 1:00                   # Limite temps
9#-Q- platine #BSUB -q post              # Passage en queue post
10#-Q- sx8brodie #!/bin/ksh
11#-Q- sx8brodie #######################
12#-Q- sx8brodie ## SX8BRODIE   IDRIS ##
13#-Q- sx8brodie #######################
14#-Q- sx8brodie # Temps Elapsed max. d'une requete hh:mm:ss
15#-Q- sx8brodie # @ wall_clock_limit = 10:00:00
16#-Q- sx8brodie # Nom du travail LoadLeveler
17#-Q- sx8brodie # @ job_name   = TS
18#-Q- sx8brodie # Fichier de sortie standard du travail       
19#-Q- sx8brodie # @ output     = $(job_name).$(jobid)
20#-Q- sx8brodie # Fichier de sortie d'erreur du travail
21#-Q- sx8brodie # @ error      =  $(job_name).$(jobid)
22#-Q- sx8brodie # pour recevoir un mail en cas de depassement du temps Elapsed (ou autre pb.)
23#-Q- sx8brodie # @ notification = error
24#-Q- sx8brodie # @ environment  = $DEBUG_debug ; $MODIPSL ; $libIGCM ; $libIGCM_SX ; $SUBMIT_DIR ; $REBUILD_DIR ; $RebuildFromArchive ; $POST_DIR ; $DateBegin ; $PeriodDateBegin ; $PeriodDateEnd ; $NbRebuildDir ; $StandAlone ; $RESOL_ATM ; $RESOL_OCE ; $RESOL_ICE ; $RESOL_SRF
25#-Q- sx8brodie # @ queue
26#-Q- sx8mercure #!/bin/ksh
27#-Q- sx8mercure ######################
28#-Q- sx8mercure ## SX8MERCURE   CEA ##
29#-Q- sx8mercure ######################
30#-Q- sx8mercure #PBS -N TS                   # Nom du job
31#-Q- sx8mercure #PBS -j o                    # regroupement des stdout et stderr
32#-Q- sx8mercure #PBS -S /usr/bin/ksh         # shell de soumission
33#-Q- sx8mercure #PBS -l memsz_job=1gb        # Limite memoire a 1 Go
34#-Q- sx8mercure #PBS -l cputim_job=2:00:00   # Limite temps a 2 heures
35#-Q- sx8mercure #PBS -q scalaire
36#-Q- default #!/bin/ksh
37#-Q- default ##################
38#-Q- default ## DEFAULT HOST ##
39#-Q- default ##################
40
41#set -vx
42
43date
44
45#-Q- sx8brodie export OMP_NUM_THREADS=1
46
47# $Date$
48# $Author$
49# $Revision$
50# IPSL (2006)
51#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
52
53########################################################################
54
55#D- Flag to determine if this job in a standalone mode
56#D- Default : value from AA_job if any
57StandAlone=${StandAlone:=true}
58
59#D- Flag to determine job's output directory
60#D- Default : value from libIGCM_post.ksh if any
61POST_DIR=${POST_DIR:=${PBS_O_WORKDIR}}
62
63#D- Increased verbosity (1, 2, 3)
64#D- Default : value from AA_job if any
65Verbosity=${Verbosity:=3}
66
67#D- Low level debug : to bypass lib test checks and stack construction
68#D- Default : value from AA_job if any
69DEBUG_debug=${DEBUG_debug:=false}
70
71#D- Low level debug : to bypass lib test checks and stack construction
72#D- Default : value from AA_job if any
73libIGCM=${libIGCM:=/work/cont003/p86denv/PARA_SX8_ORCA2xLMD144142/Prelude/modipsl/libIGCM}
74
75#D- TEMPORARY Flag to determine atmospheric resolution
76#D- Default : value from atmospheric driver if any
77RESOL_ATM=${RESOL_ATM:=LMD144142}
78
79#D- TEMPORARY Flag to determine ocean resolution
80#D- Default : value from ocean driver if any
81RESOL_OCE=${RESOL_OCE:=ORCA2}
82
83#D- TEMPORARY Flag to determine ice resolution
84#D- Default : value from ice driver if any
85RESOL_ICE=${RESOL_ICE:=ORCA2}
86
87#D- Flag to determine surface resolution
88#D- Default : value from surface driver if any
89RESOL_SRF=${RESOL_SRF:=LMD144142}
90
91########################################################################
92
93. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh ;
94. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh     #; IGCM_debug_Check
95. ${libIGCM}/libIGCM_card/libIGCM_card.ksh   #; IGCM_card_Check
96. ${libIGCM}/libIGCM_date/libIGCM_date.ksh   #; IGCM_date_Check
97
98########################################################################
99
100#set -vx
101
102# Useful cleaning function
103MENAGE () 
104{ 
105    [ -f $3 ] & ( rm -f $1/$2 ;) 
106}
107
108########################################################################
109
110#set -vx
111
112if [ ${DEBUG_debug} = true ] ; then
113    IGCM_sys_MkdirWork ${RUN_DIR_PATH}
114    echo "RUN_DIR_PATH ${RUN_DIR_PATH} ok."
115fi
116IGCM_sys_Cd ${RUN_DIR_PATH}
117
118if [ ${StandAlone} = true ] ; then
119    CARD_DIR=${SUBMIT_DIR}
120else
121    CARD_DIR=${RUN_DIR_PATH}/$( basename ${SUBMIT_DIR} )
122    IGCM_sys_Get_Master ${SUBMIT_DIR} ${RUN_DIR_PATH}
123fi
124
125#
126# First of all
127#
128IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card UserChoices
129typeset option
130for option in ${config_UserChoices[*]} ; do
131    IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option}
132done
133#
134echo
135IGCM_debug_Print 1 "DefineArrayFromOption  : config_UserChoices"
136IGCM_debug_PrintVariables 3 config_UserChoices_JobName
137IGCM_debug_PrintVariables 3 config_UserChoices_LongName
138IGCM_debug_PrintVariables 3 config_UserChoices_TagName
139IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType
140IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin
141IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd
142IGCM_debug_PrintVariables 3 config_UserChoices_PeriodLength
143echo
144
145#==================================
146
147R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
148#
149IGCM_card_DefineArrayFromSection       ${CARD_DIR}/config.card ListOfComponents
150#
151# Determine period for time series. Default : value from AA_job if any
152#
153DateBegin=${DateBegin:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} )}
154#
155PeriodDateEnd=${PeriodDateEnd:=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )}
156
157########################################################################
158#
159#      Depending on requested time series fill following variables :
160#
161#                           - LISTE_FILE_${comp}[*]
162#                           - LISTE_VARS_${file}[*]
163#                           - LISTE_AXIS_${file}[*]
164#                           - LISTE_PATCH_${file}[*]
165########################################################################
166
167#
168# For each selected component determine which files need post-processing
169#
170DoJob=false
171for comp in ${config_ListOfComponents[*]} ; do
172    #
173    ActiveComp=false
174    #
175    IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp}
176    eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
177    #
178    card=${CARD_DIR}/COMP/${compname}.card
179    IGCM_card_DefineArrayFromOption ${card} OutputFiles List
180    #
181    ListFilesName=${compname}_OutputFiles_List
182    eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1
183    #
184    if [ X${FileName0} != X${NULL_STR} ] ; then
185        #
186        #IGCM_debug_Print 1 "Component      : ${compname}"
187        #
188        # INITIALISATION
189        #
190        eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1
191        typeset i=0
192        #
193        until [ $i -eq $NbFiles ]; do
194            #
195            eval file_in_=\${${ListFilesName}[$i]} > /dev/null 2>&1
196            eval file_in=${file_in_}
197            let $(( i_ = i+2 ))
198            eval flag_post_=\${${ListFilesName}[$i_]} > /dev/null 2>&1
199            eval flag_post=${flag_post_}
200            #
201            if [ X${flag_post} != XNONE ] ; then
202                #
203                # Initialize ${flag_post}_${compname}_${post_freq}
204                #
205                eval export ${flag_post}_${compname}_TimeSeriesFrequency=true
206            fi
207            let $(( i=i+3 ))
208        done
209    fi
210
211    for element in $( env | grep Post_ | grep ${compname}_TimeSeriesFrequency | grep -i "=true" ) ; do
212        #
213        FILE=$( echo ${element} | awk -F "_${compname}" '{print $1}' | awk "-FPost_" '{print $2}' )
214        #
215        # For each comp and each file who needs post-processing
216        # determine which variables need time series.
217        #
218        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars
219        #
220        if [ $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars} ) = ${NULL_STR} ] ; then
221            #
222            IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}"
223            continue
224            #
225        fi
226        #
227        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} Patches
228        #
229        # So we need coordinates axis to properly fill produced time series
230        #
231        IGCM_card_DefineArrayFromOption ${card} Post_${FILE} GatherWithInternal
232        #
233        if [ $( eval echo \$\{${compname}_Post_${FILE}_GatherWithInternal} ) = ${NULL_STR} ] ; then
234            #
235            IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}"
236            IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!"
237            continue
238            #
239        fi
240        #
241        # CREATE LIST
242        #
243        FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' )
244        case ${FlagDir} in
245            *Y)  TS_Dir=TS_YE  ;;
246            *M)  TS_Dir=TS_MO  ;;
247            *D)  TS_Dir=TS_DA  ;;
248            HF)  TS_Dir=TS_HF  ;;
249            INS) TS_Dir=TS_INS ;;
250        esac
251
252        count=0
253        for var in $( eval echo \$\{${compname}_Post_${FILE}_TimeSeriesVars[*]} ) ; do
254            #
255            DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir}
256            TestedFile=${config_UserChoices_JobName}_${DateBegin}_${PeriodDateEnd}_${FlagDir}_${var}.nc
257            #
258            IGCM_sys_TestFileArchive ${DIRECTORY}/${TestedFile}
259            if [ ! $? = 0 ] ; then
260                eval set +A LISTE_VARS_${FILE} \$\{LISTE_VARS_${FILE}[*]} ${var}
261                ActiveComp=true
262                (( count = count + 1 ))
263            fi
264        done
265
266        if [ ${count} -gt 0 ] ; then
267            #
268            eval set +A LISTE_FILE_${comp} \$\{LISTE_FILE_${comp}[*]} ${FILE}
269            eval set +A LISTE_AXIS_${FILE} \$\{${compname}_Post_${FILE}_GatherWithInternal[*]}
270            eval set +A LISTE_PATCHES_${FILE} \$\{${compname}_Post_${FILE}_Patches[*]}
271            if [ X$( eval echo \${LISTE_PATCHES_${FILE}[0]} ) !=  X${NULL_STR} ]; then
272                for Patch in $( eval echo \${LISTE_PATCHES_${FILE}[*]} ); do
273                    if [ Xload${Patch} != X ] ; then
274                        . ${libIGCM}/libIGCM_post/IGCM_${Patch}.ksh
275                        eval load${Patch}=loaded
276                    fi
277                done
278            fi
279            #
280        fi
281    done
282
283    if [ ${ActiveComp} = true ] ; then
284        set +A ActiveListOfComponents ${ActiveListOfComponents[*]} ${comp}
285    fi
286
287done
288
289set -A config_ListOfComponents ${ActiveListOfComponents[*]}
290[ ! -z "${ActiveListOfComponents}" ] && DoJob=true
291
292########################################################################
293#
294#                          DEFINE DATE FUNCTIONS
295#
296########################################################################
297
298# Give the end of the next $1 period from $2 date
299function create_ts_next_date {
300    IGCM_debug_PushStack "create_ts_next_date" $@
301    typeset year month Length
302    case ${config_UserChoices_PeriodLength} in
303        *Y|*y) 
304            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
305            IGCM_date_GetYearMonth ${1} year month
306            (( Length=0 ))
307            (( i=0 ))
308            until [ $i -ge $PeriodLengthInYears ] ; do
309                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i + 1 )) ) )) 
310                (( i=i+1 ))
311            done
312            ;;
313        *M|*m) 
314            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
315            IGCM_date_GetYearMonth ${1} year month
316            (( year0 = year ))
317            if [ $(( month + 1 )) -lt 13 ] ; then
318                month0=$(( month + 1 ))
319            else
320                month0=$(( month + 1 - 12 ))
321                (( year = year0 + 1 ))
322            fi
323            (( Length=0 ))
324            (( i=0 ))
325            until [ $i -ge $PeriodLengthInMonths ] ; do
326                if [ $(( month0 + i )) -lt 13 ] ; then
327                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i )) ) ))
328                else
329                    (( year = year0 + 1 ))
330                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month0 + i - 12 )) ) ))
331                fi
332                (( i=i+1 ))
333            done
334            ;;
335        *D|*d)
336            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
337        *)
338            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
339            IGCM_debug_Verif_Exit_Post ;;
340    esac
341    echo ${Length}
342
343    IGCM_debug_PopStack "create_ts_next_date"
344}
345
346# Give the end of the actual $1 period from $2 date
347function create_ts_begin_date {
348    IGCM_debug_PushStack "create_ts_begin_date" $@
349    typeset year month Length
350    case ${config_UserChoices_PeriodLength} in
351        *Y|*y) 
352            PeriodLengthInYears=$( echo ${config_UserChoices_PeriodLength} | awk -F '[yY]' '{print $1}' )
353            IGCM_date_GetYearMonth ${1} year month
354            if [ X${2} = Xend ] ; then
355                (( year = year - PeriodLengthInYears + 1))
356            fi
357            (( Length=0 ))
358            (( i=0 ))
359            until [ $i -ge $PeriodLengthInYears ] ; do
360                (( Length = Length + $( IGCM_date_DaysInYear $(( year + i )) ) )) 
361                (( i=i+1 ))
362            done
363            ;;
364        *M|*m) 
365            PeriodLengthInMonths=$( echo ${config_UserChoices_PeriodLength} | awk -F '[mM]' '{print $1}' )
366            IGCM_date_GetYearMonth ${1} year month
367            if [ X${2} = Xend ] ; then
368                (( month = month - PeriodLengthInMonths + 1 ))
369            fi     
370            (( year0 = year ))
371            if [ $month -le 0 ] ; then
372                (( month = month + 12 ))
373                year=$( printf "%04i\n" $(( year - 1 )) )
374            fi
375            month=$( printf "%02i\n" ${month} )
376            (( Length=0 ))
377            (( i=0 ))
378            until [ $i -ge $PeriodLengthInMonths ] ; do
379                if [ $(( month + i )) -lt 13 ] ; then
380                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i )) ) ))
381                else
382                    (( year = year0 + 1 ))
383                    (( Length = Length + $( IGCM_date_DaysInMonth $year $(( month + i - 12 )) ) ))
384                fi
385                (( i=i+1 ))
386            done
387            ;;
388        *D|*d)
389            Length=$( echo ${config_UserChoices_PeriodLength} | sed -e "s/[dD]//" ) ;;
390        *)
391            IGCM_debug_Exit "create_ts " ${config_UserChoices_PeriodLength} " invalid PeriodLength : choose in 1Y, *M, *D."
392            IGCM_debug_Verif_Exit_Post ;;
393    esac
394    echo ${Length}
395
396    IGCM_debug_PopStack "create_ts_begin_date"
397}
398
399########################################################################
400#
401#               IS THERE SOME ALLREADY PRODUCED TIME SERIES ?
402#                 IF SO BRING THEM IN THE WORKING DIRECTORY
403#
404# PS : Keep in mind that IGCM_sys_Get here is a weak link :
405#      - especially for IDRiS
406#      - no special protection against "command too long" !
407#      - libIGCM_sys should handle that or modify following code part
408#
409########################################################################
410
411if [ ${StandAlone} != true ] ; then
412    Running_Flag=$( IGCM_sys_RshMaster "IGCM_card_DefineVariableFromOption \
413       ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning ; \
414       echo \${run_PostProcessing_TimeSeriesRunning}" )
415
416    if [ X${Running_Flag} = Xy ] ; then
417        # Time Series Job allready running
418        IGCM_debug_Print 1 "Time Series Job allready running exit"
419        exit
420    fi
421    # Now run.card know we are running
422    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "y"
423    # Get information from last execution
424    CompletedFlag=$( IGCM_sys_RshMaster \
425        "IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted ;\
426        echo \${run_PostProcessing_TimeSeriesCompleted}" )
427fi
428
429# Is it the first submission or not ?
430if [ X${CompletedFlag} = X ] ; then
431    # First Time Series Submission
432    FIRST_PASS=TRUE
433    Length=$( create_ts_begin_date ${DateBegin} begin)
434    DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) )
435else
436    # Last Completed Time Series
437    FIRST_PASS=FALSE
438    DATE_FIN_JOB_B=${CompletedFlag}
439    date=${DateBegin}_${DATE_FIN_JOB_B}
440    for comp in ${config_ListOfComponents[*]} ; do
441        #
442        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
443        #
444        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
445        #
446        set -A liste_file
447        #
448        file_path=${R_SAVE}/${comp}/Analyse
449        #
450        i=0
451        file=${NULL_STR}
452        #
453        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
454            #
455            # Determine in which output can we find file
456            # IE : Analyse/TS or Analyse/DA2TS or ...
457            #
458            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
459            case ${FlagDir} in
460                *Y)  TS_Dir=TS_YE  ;;
461                *M)  TS_Dir=TS_MO  ;;
462                *D)  TS_Dir=TS_DA  ;;
463                HF)  TS_Dir=TS_HF  ;;
464                INS) TS_Dir=TS_INS ;;
465            esac
466            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
467                #
468                set +A liste_file ${liste_file[*]} ${file_path}/${TS_Dir}/${config_UserChoices_JobName}_${date}_${FlagDir}_${var}.nc
469                #
470            done
471        done
472        if [ X${file} != X${NULL_STR} ] ; then
473            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
474            IGCM_debug_Verif_Exit_Post
475        fi
476    done
477    #
478    LEVEL=0
479    #
480fi
481
482########################################################################
483#
484#                          DEFINE LOOP PARAMETERS
485#
486########################################################################
487
488DATE_COUNT=${DATE_FIN_JOB_B}
489DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B}
490
491( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1
492
493if [ ${DoJob} = true ] ; then
494    while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do
495        (( NBRE_FILE_TOT = NBRE_FILE_TOT + 1 ))
496        Length=$( create_ts_next_date ${DATE_COUNT} )
497        DATE_COUNT=$( IGCM_date_AddDaysToGregorianDate ${DATE_COUNT} ${Length} )
498    done
499else
500    NBRE_FILE_TOT=0
501fi
502
503if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then
504    NBRE_TOUR=1
505elif [ ${NBRE_FILE_TOT}%${UNIX_MAX_LIMIT} -eq 0 ] ; then
506    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT} 
507else
508    let NBRE_TOUR=${NBRE_FILE_TOT}/${UNIX_MAX_LIMIT}+1
509fi
510
511CURRENT_LOOP=1
512# THIS LOOP GIVES PARAMETERS FOR THE DATE'S LOOP ie : DATE_COURANTE, DATE_FIN.
513# WE HAVE 3 LEVELS
514#    - LEVEL 1 : JUST ONE LOOP TO COMPLETE
515#    - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
516#    - LEVEL 3 : WE ARE IN THE LAST LOOP
517
518while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] ; do
519     
520    if [ ${NBRE_FILE_TOT} -le ${UNIX_MAX_LIMIT} ] ; then 
521        # - LEVEL 1 : JUST ONE LOOP TO COMPLETE
522        NBRE_FILE_LOOP=${NBRE_FILE_TOT}
523
524        if [ ${FIRST_PASS} = TRUE ] ; then
525            DATE_COURANTE=${DATE_FIN_JOB_B_LOOP}
526        else
527            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
528            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
529        fi
530
531        DATE_FIN=${PeriodDateEnd}
532        DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP}
533
534        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
535        [ -n "${LEVEL}" ] && FLAG_B="TRUE" || FLAG_B="FALSE"
536
537        LEVEL=1
538        DEPOT="TRUE"     
539    elif [ ${CURRENT_LOOP} -ne  ${NBRE_TOUR} ] ; then
540        # - LEVEL 2 : WE ARE IN THE "DEEP" LOOP
541        NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
542
543        if [ ! ${FIRST_PASS} = TRUE ] && [ ${CURRENT_LOOP} -eq 1 ] ; then
544            Length=$( create_ts_next_date ${DATE_FIN_JOB_B_LOOP} )
545            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN_JOB_B_LOOP} ${Length} )
546        elif [ -z "${DATE_COURANTE}" ] ; then
547            DATE_COURANTE=${DATE_FIN_JOB_B}
548        else
549            Length=$( create_ts_next_date ${DATE_FIN} )
550            DATE_COURANTE=$( IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
551        fi
552         
553        (( TotLength = 0 ))
554        COMPTEUR=1
555        DATE_LOOP=${DATE_COURANTE}
556
557        while [ ${COMPTEUR} -lt ${NBRE_FILE_LOOP} ] ; do
558            #
559            Length=$( create_ts_next_date ${DATE_LOOP} )
560            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length} )
561            (( TotLength = TotLength + Length ))
562            (( COMPTEUR = COMPTEUR + 1 ))
563            #
564        done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
565        DATE_FIN=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} ${TotLength} )
566
567        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
568        [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] && FLAG_B="TRUE" || [ ${CURRENT_LOOP} -gt 1 ] && FLAG_B="TRUE" || FLAG_B="FALSE"
569        DEPOT="FALSE"
570        LEVEL=2
571    else
572        # - LEVEL 3 : WE ARE IN THE LAST LOOP
573        NBRE_FILE_LOOP=$( expr ${NBRE_FILE_TOT} % $(( UNIX_MAX_LIMIT * $(( CURRENT_LOOP - 1 )) )) )
574        [ ${NBRE_FILE_LOOP} -eq 0 ] && NBRE_FILE_LOOP=${UNIX_MAX_LIMIT}
575
576        Length=$( create_ts_next_date ${DATE_FIN} )
577        DATE_COURANTE=$(IGCM_date_AddDaysToGregorianDate ${DATE_FIN} ${Length} )
578        DATE_FIN=${PeriodDateEnd}
579        [ -n "${DATE_LOOP}" ] && DATE_FIN_JOB_B_LOOP_PREC=${DATE_FIN_JOB_B_LOOP} ; DATE_FIN_JOB_B_LOOP=${DATE_LOOP}
580
581        LEVEL=3
582        DEPOT="TRUE"
583        FLAG_B="TRUE"
584    fi
585
586    # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN.
587    for comp in ${config_ListOfComponents[*]} ; do
588        #
589        IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp}
590        #
591        IGCM_sys_Cd ${RUN_DIR_PATH}/${comp}
592        #
593        eval R_OUT_${comp}=${R_SAVE}/${comp}
594        #
595        i=0
596        #
597        for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do
598            #
599            # Determine in which output can we find file
600            # IE : Output/MO or Output/DA or ...
601            #
602            FlagDir=$( echo ${file} | awk -F "_" '{print $1}' )
603            case ${FlagDir} in
604                *Y)    TS_Dir=TS_YE  ; FreqDir=YE  ;;
605                *M)    TS_Dir=TS_MO  ; FreqDir=MO  ;;
606                *D)    TS_Dir=TS_DA  ; FreqDir=DA  ;;
607                HF)    TS_Dir=TS_HF  ; FreqDir=HF  ;;
608                INS)   TS_Dir=TS_INS ; FreqDir=INS ;;
609                *) 
610                       IGCM_debug_Print 1 "Error in FlagDir=${FlagDir} for file ${file}."
611                       IGCM_sys_Exit "Job create_ts" ;;
612            esac
613
614            # Initialize array
615            set -A liste_file
616            set -A liste_file_tmp
617
618            eval file_path=\${R_OUT_${comp}}/Output/${FreqDir}/
619
620            COMPTEUR=1
621            DATE_LOOP=${DATE_COURANTE}
622
623            while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] ; do
624                #
625                Length1=$( create_ts_begin_date ${DATE_LOOP} end)
626                DATE_TAB=$(  IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} $(( 1 - Length1 )) )_${DATE_LOOP}
627
628                Length2=$( create_ts_next_date ${DATE_LOOP} end)
629                DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} ${Length2} )
630
631                set +A liste_file ${liste_file[*]} ${file_path}${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
632                set +A liste_file_tmp ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_TAB}_${file}.nc
633                (( COMPTEUR = COMPTEUR + 1 ))
634                #
635            done # while [ ${COMPTEUR} -le ${NBRE_FILE_LOOP} ] done
636            DATE_LOOP=$( IGCM_date_AddDaysToGregorianDate ${DATE_LOOP} -${Length2} )
637
638            # Get selected files
639            IGCM_sys_Get /l liste_file[*] ${RUN_DIR_PATH}/${comp}
640            eval ExitFlag=false
641            IGCM_debug_Verif_Exit_Post
642
643            # Apply IGCM_Patch if needed
644            if [ X$( eval echo \${LISTE_PATCHES_${file}[0]} ) !=  X${NULL_STR} ]; then
645                for file_t in $( eval echo ${liste_file_tmp[*]} ); do           
646                    for Patch in $( eval echo \${LISTE_PATCHES_${file}[*]} ); do
647                        IGCM_${Patch} ${file_t}
648                    done
649                done
650            fi
651
652            # WE CAN CONCATENATE FILES AT THIS POINT
653            [ "${FLAG_B}" = "TRUE" ] && DATE_BUILD_B=${DateBegin}_${DATE_FIN_JOB_B_LOOP_PREC}
654            Length=$( create_ts_begin_date ${DATE_COURANTE} end)
655            DATE_BUILD1=$( IGCM_date_AddDaysToGregorianDate ${DATE_COURANTE} -$(( Length - 1 )) )
656            DATE_BUILD=${DATE_BUILD1}_${DATE_FIN}
657            DATE_BUILD_END=${DateBegin}_${DATE_FIN}
658
659            liste_coord=" "
660            for axis in $( eval echo \${LISTE_AXIS_${file}[*]} ); do
661              liste_coord=${liste_coord}${axis}","
662            done
663
664            # Time axis must be the last dimension
665            time_axis=${axis}
666
667            # SWITCH BETWEEN MODEL TO FIT PARTICULAR CASE
668            for var in $( eval echo \${LISTE_VARS_${file}[*]} ) ; do
669                #
670                IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${liste_file_tmp[*]} ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
671                #
672#                 # Change time axis name to be CF compliant
673#                 if [ ${time_axis} != time_counter ]; then
674#                   IGCM_sys_ncrename -O -v ${time_axis},time_counter ${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
675#                 fi
676                #
677                if [ "${FLAG_B}" = "TRUE" ] ; then
678                    # WE CONCATENATE WITH EXISTING FILES
679                    file1=${config_UserChoices_JobName}_${DATE_BUILD_B}_${FlagDir}_${var}.nc
680                    file2=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
681                    file_out=${config_UserChoices_JobName}_${DATE_BUILD_END}_${FlagDir}_${var}.nc
682                    IGCM_sys_ncrcat --hst ${file1} ${file2} ${file_out}
683                    #
684                    IGCM_sys_Rm ${file1} ${file2}
685                fi
686                #
687                if [ "${DEPOT}" = "TRUE" ] ; then
688                    # WE PUT FINAL FILE ON FILE SYSTEM
689                    [ "${FLAG_B}" = "FALSE" ] && file_out=${config_UserChoices_JobName}_${DATE_BUILD}_${FlagDir}_${var}.nc
690                    #${ncatted} -a modulo,time_counter,c,c," " ${file_out}
691                    file_before=${config_UserChoices_JobName}_${DateBegin}_${DATE_FIN_JOB_B}_${FlagDir}_${var}.nc
692                    eval IGCM_sys_Put_Out ${file_out} \${R_OUT_${comp}}/Analyse/${TS_Dir}/${file_out}
693                    [ "${FLAG_B}" = "TRUE" ] && eval MENAGE \${R_OUT_${comp}}/Analyse/${TS_Dir} ${file_before} ${file_out}
694                    IGCM_sys_Rm ${file_out}
695                    #
696                fi
697                #
698                eval ExitFlag=false
699                IGCM_debug_Verif_Exit_Post
700            done     # for var in ${liste_var} ; do
701            IGCM_sys_Rm ${liste_file_tmp[*]}
702        done         # for file in ${liste_file_brut} do
703    done             # for comp in ${config_ListOfComponents[*]} ; do
704    eval ExitFlag=false
705    IGCM_debug_Verif_Exit_Post
706    # GO AHEAD IN DEEPEST LOOP
707    (( CURRENT_LOOP = CURRENT_LOOP + 1 ))
708done                 # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do
709# FINISH
710
711# DODS copy
712for comp in ${config_ListOfComponents[*]} ; do
713    for TS_Dir in TS_YE TS_MO TS_DA TS_HF TS_INS ; do
714        IGCM_sys_TestDirArchive ${R_SAVE}/${comp}/Analyse/${TS_Dir}
715        [ $? = 0 ] && IGCM_sys_Put_Dods ${comp}/Analyse/${TS_Dir}
716    done
717done
718
719# clean environment variables and export usefull one :
720if [ X$( hostname -s ) = Xrhodes ] ; then
721    LISTE_ENV=$( env | grep -iv MYLANG | grep -iv HOST | grep -iv LOGIN | grep -iv config_ | grep -iv R_ | grep -iv libIGCM | grep -iv PERIOD \
722        | grep -iv DMFDIR | grep -iv AN_| grep -iv YEARS | grep -iv ex | grep -iv RESOL_ | grep -iv PATH | grep -iv WORKDIR | grep -iv USER   \
723        | grep -iv LOGNAME | grep -iv PROFILE | grep -iv ENV | grep -iv TMPDIR | grep -iv POST_DIR | grep -iv StandAlone | grep -iv DB | awk -F= '{print $1}' )
724 
725    for variables in ${LISTE_ENV} ; do
726        unset ${variables}
727    done
728fi
729
730export POST_DIR; export SUBMIT_DIR; export libIGCM; export R_INIT; export R_BC; export StandAlone
731export RESOL_ATM; export RESOL_OCE ; export RESOL_ICE ; export RESOL_SRF
732
733listVarEnv="POST_DIR,SUBMIT_DIR,libIGCM,R_INIT,R_BC,StandAlone,RESOL_ATM,RESOL_OCE,RESOL_ICE,RESOL_SRF"
734export listVarEnv
735
736#D-
737# --------------------------------------------------------------------
738#D- Test if all was right
739# --------------------------------------------------------------------
740IGCM_debug_Verif_Exit_Post
741# --------------------------------------------------------------------
742#D- Submit MONITORING
743# --------------------------------------------------------------------
744IGCM_sys_QsubPost monitoring
745
746#D-
747# --------------------------------------------------------------------
748#D- Update the run.card
749# --------------------------------------------------------------------
750if [ ${StandAlone} != true ] ; then
751    # Put in run.card end period of time series
752    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesCompleted "${DATE_FIN}"
753    # We have finish the job !
754    IGCM_sys_RshMaster IGCM_card_WriteOption ${SUBMIT_DIR}/run.card PostProcessing TimeSeriesRunning "n"
755fi
756date
Note: See TracBrowser for help on using the repository browser.