Ignore:
Timestamp:
10/25/23 14:34:02 (8 months ago)
Author:
aclsce
Message:
  • Changed libIGCM behaviour in case of pb in pack_output.job : now, we exit from the job if there is any problem, whatever we are in PROD, DEVT, TEST mode.
  • Specification of pack frequency per file :

In order to reduce the number of inodes, it is possible to specify by file the frequency of packing. The syntax to do that is in the 4th column of OutputFiles? section of the component.card, for example as follows in lmdz.card :
[OutputFiles?]
List= (histmth.nc, ${R_OUT_ATM_O_M}/${PREFIX}_1M_histmth.nc, Post_1M_histmth, 100Y), \

(histday.nc, ${R_OUT_ATM_O_D}/${PREFIX}_1D_histday.nc, Post_1D_histday, 10Y), \

...

In this example, histmth files will be packed every 100 years and histday files will be packed every 10 years.
The pack frequency you defined in config.card is the frequency of pack by default, that means if a specific frequency of pack is specified for a file in a component.card, this file will be packed at the specific frequency whereas all other files will be packed at global pack frequency (specified in config.card) and in this case, the frequency pack (from the config.card) is the frequency the pack_output job will be launched at.
There is a constraint to use this fonctionality : the Packfrequency you defined in config.card must be greater or equal to the pack frequencies you specified for each type of file in component.card, otherwise the computing job will be stopped (with an explicit error message).
Surpack mode :
A surpack mode functionality is available through the use of pack_output.job. To enable this functionality, you have to put "surpack_mode=y" (default value is n). The way to use is similar to restart post-processing pack_output jobs, as indicated here : ​http://forge.ipsl.jussieu.fr/igcmg_doc/wiki/Doc/CheckDebug#RestartPack_output. You can either use a global pack frequency in config.card or specific pack frequency per file, as explained above.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/AA_create_ts

    r1598 r1603  
    354354          IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}" 
    355355          IGCM_debug_Print 3 "TimeSeriesVars${Dimension} for ${FILE} is empty" 
     356          (( i_ = i+1 )) 
     357          eval Testvar_=\${${ListFilesName}[$i_]} > /dev/null 2>&1 
     358          eval Testvar=${Testvar_} 
     359          if [[ ${Testvar} =~ [0-9][DMY]$ ]] ; then 
     360              (( i = i+4 )) 
     361          else 
    356362          (( i=i+3 )) 
     363          fi           
    357364          continue 
    358365          # 
    … …  
    364371          IGCM_debug_Print 3 "Nothing to do for ${FILE} according to ${card}" 
    365372          IGCM_debug_Print 3 "ChunckJob${Dimension}=OFF for ${FILE} so it has been switched off" 
     373          (( i_ = i+1 )) 
     374          eval Testvar_=\${${ListFilesName}[$i_]} > /dev/null 2>&1 
     375          eval Testvar=${Testvar_} 
     376          if [[ ${Testvar} =~ [0-9][DMY]$ ]] ; then 
     377              (( i = i+4 )) 
     378          else 
    366379          (( i=i+3 )) 
     380          fi            
    367381          continue 
    368382        fi 
    … …  
    374388            IGCM_debug_Print 3 "Nothing to do for ${FILE} TimeSeriesVars${Dimension} according to ${card}" 
    375389            IGCM_debug_Print 3 "Let another chunck task do the job $( eval echo \${${compname}_Post_${FILE}_ChunckJob${Dimension}} )" 
     390          (( i_ = i+1 )) 
     391          eval Testvar_=\${${ListFilesName}[$i_]} > /dev/null 2>&1 
     392          eval Testvar=${Testvar_} 
     393          if [[ ${Testvar} =~ [0-9][DMY]$ ]] ; then 
     394              (( i = i+4 )) 
     395          else 
    376396            (( i=i+3 )) 
     397          fi 
    377398            continue 
    378399          fi 
    … …  
    391412          IGCM_debug_Print 1 "ERROR !! We miss coordinate axis for ${FILE} according to ${card}" 
    392413          IGCM_debug_Print 1 "So no time series will be produced for ${FILE} !!" 
     414          (( i_ = i+1 )) 
     415          eval Testvar_=\${${ListFilesName}[$i_]} > /dev/null 2>&1 
     416          eval Testvar=${Testvar_} 
     417          if [[ ${Testvar} =~ [0-9][DMY]$ ]] ; then 
     418              (( i = i+4 )) 
     419          else 
    393420          (( i=i+3 )) 
     421          fi           
    394422          continue 
    395423          # 
    … …  
    426454          eval set +A LISTE_FILE_${comp} \${LISTE_FILE_${comp}[*]} ${FILE} 
    427455          eval set +A LISTE_AXIS_${FILE} \${${compname}_Post_${FILE}_GatherWithInternal[*]} 
     456          (( i_ = i+1 )) 
     457          eval Testvar_=\${${ListFilesName}[$i_]} > /dev/null 2>&1 
     458          eval Testvar=${Testvar_} 
     459          if [[ ${Testvar} =~ [0-9][DMY]$ ]] ; then 
     460              eval PeriodPack_FileType=${Testvar} 
     461          else 
     462              PeriodPack_FileType=${config_Post_PackFrequency} 
     463          fi 
     464          eval set +A LISTE_PACKFREQUENCY_${comp} \${LISTE_PACKFREQUENCY_${comp}[*]} ${PeriodPack_FileType} 
    428465          # 
    429466          if [ ! X${RebuildFrequency} = Xtrue ] ; then 
    … …  
    441478        fi 
    442479      fi 
     480      (( i_ = i+1 )) 
     481      eval Testvar_=\${${ListFilesName}[$i_]} > /dev/null 2>&1 
     482      eval Testvar=${Testvar_} 
     483      if [[ ${Testvar} =~ [0-9][DMY]$ ]] ; then 
     484          (( i = i+4 )) 
     485      else 
    443486      (( i=i+3 )) 
     487      fi           
    444488    done 
    445489  fi 
    … …  
    497541  # First Time Series Submission 
    498542  FIRST_PASS=TRUE 
    499   Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin) 
    500   DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) ) 
    501543else 
    502544  # Last Completed Time Series 
    … …  
    559601######################################################################## 
    560602 
     603( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1 
     604 
     605# THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN. 
     606for comp in ${config_ListOfComponents[*]} ; do 
     607    # 
     608    IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp} 
     609      # 
     610    IGCM_sys_Cd ${RUN_DIR_PATH}/${comp} 
     611    # 
     612    eval R_OUT_${comp}=${R_SAVE}/${comp} 
     613    # 
     614    i=0 
     615    # 
     616    for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do 
     617        config_UserChoices_PeriodLength=$( eval echo \${LISTE_PACKFREQUENCY_${comp}[$i]} ) 
     618        if [ ${FIRST_PASS} = TRUE ] ; then 
     619            Length=$( IGCM_date_DaysInPreviousPeriod ${DateBegin} ${config_UserChoices_PeriodLength} begin)  
     620            DATE_FIN_JOB_B=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( Length - 1 )) ) 
     621        fi 
    561622DATE_COUNT=${DATE_FIN_JOB_B} 
    562623DATE_FIN_JOB_B_LOOP=${DATE_FIN_JOB_B} 
    563  
    564 ( [ -n "${LEVEL}" ] && [ ${LEVEL} -eq 0 ] ) && NBRE_FILE_TOT=0 || NBRE_FILE_TOT=1 
    565  
    566624if [ ${DoJob} = true ] ; then 
    567625  while [ ${DATE_COUNT} -lt ${PeriodDateEnd} ] ; do 
    … …  
    658716 
    659717  # THEN FOR EACH FILE TYPE WE LOOP BETWEEN DATE_COURANTE AND DATE_FIN. 
    660   for comp in ${config_ListOfComponents[*]} ; do 
    661     # 
    662     IGCM_sys_Mkdir ${RUN_DIR_PATH}/${comp} 
    663       # 
    664     IGCM_sys_Cd ${RUN_DIR_PATH}/${comp} 
    665     # 
    666     eval R_OUT_${comp}=${R_SAVE}/${comp} 
    667     # 
    668     i=0 
    669     # 
    670     for file in $( eval echo \${LISTE_FILE_${comp}[*]} ); do 
    671718      # 
    672719      # Determine in which output we can find file 
    … …  
    717764      if [ "${MissingFile}" = "TRUE" ] ; then 
    718765        IGCM_debug_Print 1 "WARNING We skip ${file} type processing" 
    719         continue 
     766                continue 2 
    720767      fi 
    721768 
    … …  
    802849      done                              # for var in ${liste_var} ; do 
    803850      IGCM_sys_Rm ${liste_file_tmp[*]} 
     851            (( CURRENT_LOOP = CURRENT_LOOP + 1 )) 
     852      (( i = i + 1 )) 
    804853    done                                # for file in ${liste_file_brut} do 
    805854  done                                  # for comp in ${config_ListOfComponents[*]} ; do 
    806855  # GO AHEAD IN DEEPEST LOOP 
    807   (( CURRENT_LOOP = CURRENT_LOOP + 1 )) 
    808856done                                    # while [ ${CURRENT_LOOP} -le  ${NBRE_TOUR} ] do 
    809857# FINISH 
Note: See TracChangeset for help on using the changeset viewer.