Changeset 2045 for TOOLS/PACK_IPSL/archive_debug.sh

Timestamp:

04/05/13 17:34:38 (11 years ago)

Author:

acosce

Message:

made some modifications to speed up archive restart and debug process

File:

• TOOLS/PACK_IPSL/archive_debug.sh (modified) (4 diffs)

TOOLS/PACK_IPSL/archive_debug.sh

@@ -95 +95 @@
 
     ls -l ${STORE_DEBUG} > ${IGCM_TMP}/${$}.new_liste_debug_files.txt
+
+
+#creation des listes en fonction des suffixes
+   if [ "x${ON_IDRIS}" == "xno" ]
+   then
+       awk '{if (NR >1) print $9}' ${IGCM_TMP}/${$}.new_liste_debug_files.txt > ${IGCM_TMP}/${$}.new_liste_debug_files.txt_tmp
+   else
+       awk '{if (NR >1) print $8}' ${IGCM_TMP}/${$}.new_liste_debug_files.txt > ${IGCM_TMP}/${$}.new_liste_debug_files.txt_tmp       
+   fi
+sort ${IGCM_TMP}/${$}.new_liste_debug_files.txt_tmp > ${IGCM_TMP}/${$}.order_new_liste_debug_files.txt
+ref=`awk -F_ '{if (NR==1) print $1}' ${IGCM_TMP}/${$}.order_new_liste_debug_files.txt`
+
+
+
+i=1
+for new_file in `awk  '{print $0}' ${IGCM_TMP}/${$}.order_new_liste_debug_files.txt`; do 
+    new_ref=$( echo ${new_file} | sed -e "s/_.*//") 
+
+    if [ ${new_ref} == ${ref} ]; then
+        echo ${new_file} >> ${IGCM_TMP}/${$}.debug_list_${i}.txt
+    else
+        ref=${new_ref}
+        (( i = $i + 1 )) 
+        echo ${new_file} >> ${IGCM_TMP}/${$}.debug_list_${i}.txt
+    fi
+done
+
+j=1
+while [[ $j -le $i  ]]; do 
+
+
 ### 2- creer les listes de fichiers par periode ####
     DEM_log -1 "Creation des listes de fichiers par periode"
@@ -100 +131 @@
     date_begin=${DateBegin}
     date_end=${DateEnd}
-
+    
     year_begin=$(echo ${date_begin} | cut -c 1-4 )
     month_begin=$(echo ${date_begin} | cut -c 5-6 )
@@ -122 +153 @@
         month_end_pack=${month_begin_pack}
         year_end_pack=`expr ${year_begin_pack} + ${period_pack}`
-
+        
         date_end_pack=${year_end_pack}${month_end_pack}01   
-        if [ "x${ON_IDRIS}" == "xno" ]
-        then
-            for file in `awk '{print $9}' ${IGCM_TMP}/${$}.new_liste_debug_files.txt`; do 
-                extract_date_file=`echo ${file}  | sed -e "s/.*\${JobName}_//"`
-                date_file=`echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g'`
-                if [[  "x${date_file}" != "x" && ! -n ${date_file//[0-9]/} ]]; then
-                    if [  ${date_file} -lt ${date_end_pack} ]
+        for file in `awk '{print $1}' ${IGCM_TMP}/${$}.debug_list_${j}.txt`; do 
+            extract_date_file=`echo ${file}  | sed -e "s/.*\${JobName}_//"`
+            date_file=`echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g'`
+            if [[  "x${date_file}" != "x" && ! -n ${date_file//[0-9]/} ]]; then
+                if [  ${date_file} -lt ${date_end_pack} ]
+                then 
+                    if [ ${date_file} -ge ${date_begin_pack} ]
                     then 
-                        if [ ${date_file} -ge ${date_begin_pack} ]
-                        then 
-                            echo ${file} >> ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt
-                        fi
-                    fi 
-                fi
-            done
-        else
-            for file in `awk '{print $8}' ${IGCM_TMP}/${$}.new_liste_debug_files.txt`; do 
-                extract_date_file=`echo ${file}  | sed -e "s/.*\${JobName}_//"`
-                date_file=`echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g'`
-        if [[  "x${date_file}" != "x" && ! -n ${date_file//[0-9]/} ]]; then
-                    if [ ${date_file} -lt ${date_end_pack} ]
-                    then 
-                        if [ ${date_file} -ge ${date_begin_pack} ]
-                        then 
-                            echo ${file} >> ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt
-                        fi
-                    fi 
-                fi
-            done  
-        fi
+                        echo ${file} >> ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt
+                        grep -v ${file} ${IGCM_TMP}/${$}.debug_list_${j}.txt > ${IGCM_TMP}/${$}.debug_list_${j}.txt.tmp
+                        mv ${IGCM_TMP}/${$}.debug_list_${j}.txt.tmp ${IGCM_TMP}/${$}.debug_list_${j}.txt
+                        
+                    fi
+                fi 
+            fi
+            if [  ${date_file} -gt ${date_end_pack} ]
+            then 
+                break
+            fi
+            
+        done
+        month_begin_pack=${month_end_pack}
+        year_begin_pack=${year_end_pack}
+
+        date_begin_pack=${year_begin_pack}${month_begin_pack}01
+
+        number_pack=`expr ${number_pack} + 1`
+    done
+
+    ((j = $j + 1 ))
+done
+
+compt_pack=1
+while [[ $compt_pack -le $number_pack ]]; do 
+
 #on archive les debug - le nom de l'archive contiendra ${yearbegin}${monthbegin}_${yearend}${monthend} 
 # end etant la date du dernier fichier contenu et non pas la premiere date suivant 
@@ -159 +195 @@
 #On classe par ordre alphabetique les fichiers contenus dans liste_pack_debug 
 #la date du dernier fichier donnera la date de fin du pack
-  if [[ -f ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt ]] ; then
-   sort ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt > ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt.tmp
-
-   first_file=$(head -n 1 ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt.tmp )
+  if [[ -f ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt ]] ; then
+   sort ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt > ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp
+
+   first_file=$(head -n 1 ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp )
    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
    DateBegin_Debug=$(echo ${extract_date_file}| sed 's%\([0-9]\{8\}\)_.*$%\1%g' )
 
-   last_file=$(tail -n 1 ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt.tmp )
+   last_file=$(tail -n 1 ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp )
    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
    DateEnd_Debug=$(   echo ${extract_date_file}| sed 's%.*\([0-9]\{8\}\)_.*$%\1%g' )
-   mv ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt.tmp ${DEBUG_TAR}/${JobName}_debug_${DateBegin_Debug}_${DateEnd_Debug}.list
+   mv ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp ${DEBUG_TAR}/${JobName}_debug_${DateBegin_Debug}_${DateEnd_Debug}.list
   fi
 
-DEM_log -1 "Passage de la commande Tar pour archiver la liste creee"
-#   tar -cf ${JobName}_${date_begin_pack}_${DateEnd_Debug}_debug.tar --dereference  --files-from ${DEBUG_TAR}/tar_debug_${date_begin_pack}_${DateEnd_Debug}.list
-#   cat  ${DEBUG_TAR}/tar_debug_${number_pack}.list | xargs rm 
 
 
 DEM_log -1 "On definit le datebegin de la  prochaine periode pack"
 #on definit la periode pack suivante
-   month_begin_pack=${month_end_pack}
-   year_begin_pack=${year_end_pack}
-
-   date_begin_pack=${year_begin_pack}${month_begin_pack}01
-
-   number_pack=`expr ${number_pack} + 1`
-
+
+   compt_pack=`expr ${compt_pack} + 1`
 done