Ignore:
Timestamp:
03/21/12 11:02:21 (12 years ago)
Author:
sdipsl
Message:
  • can extract needed restart from a tar (up to 20 minutes on titane)
    • still to do : introduce a lag when cleaning the restart buffer so that we don't look into the tar too often
    • still to do : work on the clean_month procedure
  • rebuild process submit pack_output process if needed
    • still to do : a pack_output able to submit create_ts/create_se if needed
  • bugfixes/cosmetics/cleaning
  • SX9 and titane OK.
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/AA_pack_output

    r585 r590  
    33#-Q- cesium ## CESIUM   CEA ## 
    44#-Q- cesium ###################### 
    5 #-Q- cesium #MSUB -r PACKOUTPUT    # Nom du job                 
     5#-Q- cesium #MSUB -r PACKOUTPUT    # Nom du job 
    66#-Q- cesium #MSUB -N 1              # Reservation du noeud 
    77#-Q- cesium #MSUB -n 1              # Reservation du processus 
    … …  
    2727#-Q- sx8brodie # Nom du travail LoadLeveler 
    2828#-Q- sx8brodie # @ job_name   = PACKOUTPUT 
    29 #-Q- sx8brodie # Fichier de sortie standard du travail        
     29#-Q- sx8brodie # Fichier de sortie standard du travail 
    3030#-Q- sx8brodie # @ output     = $(job_name).$(jobid) 
    3131#-Q- sx8brodie # Fichier de sortie d'erreur du travail 
    … …  
    8181#-Q- sx9mercure ## CESIUM   CEA ## 
    8282#-Q- sx9mercure ###################### 
    83 #-Q- sx9mercure #MSUB -r PACKOUTPUT    # Nom du job                 
     83#-Q- sx9mercure #MSUB -r PACKOUTPUT    # Nom du job 
    8484#-Q- sx9mercure #MSUB -N 1              # Reservation du noeud 
    8585#-Q- sx9mercure #MSUB -n 1              # Reservation du processus 
    … …  
    9191#-Q- titane ## TITANE   CEA     ## 
    9292#-Q- titane ###################### 
    93 #-Q- titane #MSUB -r PACKOUTPUT    # Nom du job                 
     93#-Q- titane #MSUB -r PACKOUTPUT    # Nom du job 
    9494#-Q- titane #MSUB -eo 
    9595#-Q- titane #MSUB -n 1              # Reservation du processus 
    … …  
    200200fi 
    201201 
    202 # 
     202#==================================================== 
    203203# First of all 
    204204# 
    205 #### 0- lecture de config.card  
    206 #### on veut les renseignements suivants :  
    207 #### JobName, PATH_SIMUL, DateBegin et DateEnd 
     205# Read config.card 
    208206 
    209207IGCM_card_DefineArrayFromSection     ${CARD_DIR}/config.card UserChoices 
    … …  
    249247# Switch to script variables meaning (try to be compatible with ipsl_pack TGCC moving procedure) 
    250248JobName=${config_UserChoices_JobName} 
    251 PATH_SIMUL=${R_BUFR} 
    252 echo $JobName $PATH_SIMUL $DateBegin $DateEnd 
     249echo $JobName $DateBegin $DateEnd 
    253250 
    254251# ------------------------------------------------------------------ 
    … …  
    268265 
    269266  IGCM_debug_PrintVariables 3 number_pack 
    270   DaysTemp=$( IGCM_date_DaysInNextPeriod ${date_begin_pack} ${PeriodPack} ) 
     267  DaysTemp=$( IGCM_date_DaysInCurrentPeriod ${date_begin_pack} ${PeriodPack} ) 
    271268  date_end_pack=$( IGCM_date_AddDaysToGregorianDate ${date_begin_pack} $(( ${DaysTemp} - 1 )) ) 
    272269 
    … …  
    283280      # Loop over the file type and pack them when in between date_begin_pack and date_end_pack 
    284281      for myType in ${fileType} ; do 
    285         grep ${myType} liste_files.${dirID}.txt > liste_files.${dirID}.${myType}.txt 
    286         nbfile=0 
    287         for file in $( gawk '{print $11}' liste_files.${dirID}.${myType}.txt ); do 
    288           extract_date_file=$( echo ${file}  | sed -e "s/.*${JobName}_[0-9]*_//" ) 
    289           date_file=$( echo ${extract_date_file} | sed 's/\([0-9]\{8\}\)_.*$/\1/g' ) 
    290           # echo pack n°${number_pack}  ${date_file} ${date_begin_pack} ${date_end_pack} 
    291           if [ ${date_file} -le ${date_end_pack} ] && [ ${date_file} -ge ${date_begin_pack} ] ; then  
    292             echo ${file} >> liste_pack_${myType}_${number_pack}.txt 
    293             ncdump -h ${file} | grep -E 'float|double' | cut -f 1 -d '(' | cut -f 2 -d ' ' >> liste_variables_${myType}_${number_pack}.txt 
    294             (( nbfile = nbfile + 1 )) 
    295           fi 
    296         done 
    297         # Select list of variables to work with 
    298         list_var=$( cat liste_variables_${myType}_${number_pack}.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d ',' ) 
    299         liste_file_tmp=$( for i in $( cat liste_pack_${myType}_${number_pack}.txt ) ; do basename $i ; done ) 
    300         # Create packed files 
    301         IGCM_debug_Print 1 "Ncrcat ongoing for ${dir} and ${myType}" 
    302         output=${JobName}_${date_begin_pack}_${date_end_pack}_${myType} 
    303         #cat liste_pack_${myType}_${number_pack}.txt | xargs ncrcat -v ${list_var} -o ${output} 
    304         if [ X${list_var} = X ] ; then 
    305           ncrcat --omp_num_threads 4 -p ${dir} ${liste_file_tmp} --output ${output} 
    306         else 
    307           ncrcat --omp_num_threads 4 -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output} 
    308         fi 
    309         # ------------------------------------------------------------------ 
     282        grep ${myType} liste_files.${dirID}.txt > liste_files.${dirID}.${myType}.txt 
     283        nbfile=0 
     284        for file in $( gawk '{print $11}' liste_files.${dirID}.${myType}.txt ); do 
     285          extract_date_file=$( echo ${file}  | sed -e "s/.*${JobName}_[0-9]*_//" ) 
     286          date_file=$( echo ${extract_date_file} | sed 's/\([0-9]\{8\}\)_.*$/\1/g' ) 
     287          # echo pack n°${number_pack}  ${date_file} ${date_begin_pack} ${date_end_pack} 
     288          if [ ${date_file} -le ${date_end_pack} ] && [ ${date_file} -ge ${date_begin_pack} ] ; then 
     289            echo ${file} >> liste_pack_${myType}_${number_pack}.txt 
     290            ncdump -h ${file} | grep -E 'float|double' | cut -f 1 -d '(' | cut -f 2 -d ' ' >> liste_variables_${myType}_${number_pack}.txt 
     291            (( nbfile = nbfile + 1 )) 
     292          fi 
     293        done 
     294        # Select list of variables to work with 
     295        list_var=$( cat liste_variables_${myType}_${number_pack}.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d ',' ) 
     296        liste_file_tmp=$( for i in $( cat liste_pack_${myType}_${number_pack}.txt ) ; do basename $i ; done ) 
     297        # Create packed files 
     298        IGCM_debug_Print 1 "Ncrcat ongoing for ${dir} and ${myType}" 
     299        output=${JobName}_${date_begin_pack}_${date_end_pack}_${myType} 
     300        #cat liste_pack_${myType}_${number_pack}.txt | xargs ncrcat -v ${list_var} -o ${output} 
     301        if [ X${list_var} = X ] ; then 
     302          IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output} 
     303        else 
     304          IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output} 
     305        fi 
     306        # ------------------------------------------------------------------ 
    310307        # Test if all was right before proceeding further 
    311308        # ------------------------------------------------------------------ 
    312         IGCM_debug_Verif_Exit_Post 
     309        IGCM_debug_Verif_Exit_Post 
    313310        # Save it 
    314         IGCM_sys_Put_Out ${output} ${R_SAVE}/$( echo $dir | sed "s:${R_BUFR}/::" )/${output} 
    315         # ------------------------------------------------------------------ 
     311        IGCM_sys_Put_Out ${output} ${R_SAVE}/$( echo $dir | sed "s:${R_BUFR}/::" )/${output} 
     312        # ------------------------------------------------------------------ 
    316313        # Test if all was right before proceeding further 
    317314        # ------------------------------------------------------------------ 
    318         IGCM_debug_Verif_Exit_Post 
     315        IGCM_debug_Verif_Exit_Post 
    319316        # Clean file produced by ncrcat 
    320         IGCM_sys_Rm ${output} 
     317        IGCM_sys_Rm ${output} 
    321318        # Clean files used by ncrcat 
    322         cat liste_pack_${myType}_${number_pack}.txt | xargs rm 
     319        cat liste_pack_${myType}_${number_pack}.txt | xargs rm 
    323320        # Save the list of files that has been pack (ncrcat) 
    324         mv liste_pack_${myType}_${date_begin_pack}_${date_end_pack}.txt ${STORE_DEBUG} 
    325         IGCM_debug_Print 1 "Ncrcat and cleaning done for ${dir} and ${myType}" 
     321        mv liste_pack_${myType}_${date_begin_pack}_${date_end_pack}.txt ${STORE_DEBUG} 
     322        IGCM_debug_Print 1 "Ncrcat and cleaning done for ${dir} and ${myType}" 
    326323      done 
    327324    done 
    … …  
    331328  date_begin_pack=$( IGCM_date_AddDaysToGregorianDate ${date_end_pack} 1 ) 
    332329done 
     330 
     331# Clean RUN_DIR_PATH (necessary for cesium and titane only) 
     332IGCM_sys_RmRunDir -Rf ${RUN_DIR_PATH} 
     333 
     334date 
Note: See TracChangeset for help on using the changeset viewer.