#!/bin/bash listToProcessFile="${1}" switchtotar=0 # gpdebug : a modifier --> s'occuper des options du script orig enlarge.sh maxtries=3 #------------------------------------------------------ # export INPUT_DMF_DATA=${SCRATCHDIR}/datasLabo/dataDMNFS_simple1 export INPUT_DMF_DATA=${SCRATCHDIR}/dataDMNFS_FromTitane # export OUTPUT_STORE=${SCRATCHDIR}/PSEUDO_DMNFS_STORE export OUTPUT_STORE=${SCRATCHDIR}/dataDMNFS_FromTitane_STORE # export OUTPUT_WORK=${SCRATCHDIR}/PSEUDO_DMNFS_WORK export OUTPUT_WORK=${SCRATCHDIR}/dataDMNFS_FromTitane_WORK export IGCM_DEM=${SCRATCHDIR}/IGCM_DEM dirin=$INPUT_DMF_DATA dirout=$OUTPUT_STORE dirout_work=$OUTPUT_WORK export JOB_DIR=${LS_SUBCWD:-${PWD}} export EXE_DIR=${JOB_DIR} source ${EXE_DIR}/DEM_utilities.sh #------------------------------------------------------ dirPathOfListFile=$( dirname $listToProcessFile ) dirnameOfListFile=$( basename $dirPathOfListFile ) echo "dirnameOfListFile=${dirnameOfListFile}|" dirOK=0 if [ "$dirnameOfListFile" != "output_ncrcat" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi if [ "$dirnameOfListFile" != "output_tar" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi if [ "$dirnameOfListFile" != "restart_tar" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi if [ "$dirnameOfListFile" != "store_cp" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi if [ "$dirnameOfListFile" != "work_cp" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi if [ "$dirnameOfListFile" != "debug_tar" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi if [ "$dirnameOfListFile" != "other_tar" ] then dirOK=$(( $dirOK || 0 )) else dirOK=$(( $dirOK || 1 )) fi # echo "dirOK=$dirOK" if [ "$dirOK" == "0" ] then echo "probleme : le rep du fichier liste est mauvais 2." # gpdebug : a modifier exit 1 fi function find_varstoexclude { nbfile=0 for file in `cat $1` ; do # ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt ncdump -h ${file} | grep -Ew "byte|char|short|int|float|double" | awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}' >> tmp_$$.txt let nbfile=nbfile+1 done varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','` varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','` nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'` rm -f tmp_$$.txt return 0 } # gpdebug : pour test showPackProgress.sh ************************* export RANDOM=$$ function gives_0_or_1 { bit=-1 let "bit = RANDOM % 2" echo $bit } resultCmd= # gpdebug : ****************************** fin ******************** # rep de simu : celui contenant le rep qui contient la liste PATH_SIMU=$( dirname $dirPathOfListFile ) set=$listToProcessFile if [ "$dirnameOfListFile" == "output_ncrcat" ] then #----------------------------------- echo "#-------------------" echo "Set: $set" datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` statusfile=${set%%.list}.status logfile=${set%%.list}_${datestr}.log #----------------------------------- # Create status file if not exists if [ ! -e $statusfile ] ; then echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile fi #----------------------------------- # Switch to tar command if [ $switchtotar -eq 1 ] ; then echo "====> DELEGATE to tar command with option --switchtotar" echo "DELEGATE to tar command with option --switchtotar" >> $statusfile mv $set $PATH_SIMU/output_tar exit 0 fi #----------------------------------- # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then printf "====> " tail -1 $statusfile exit 0 # gpdebug : modif fi #----------------------------------- # nbtries = number of lines from status file (first comment line gives 1 for the first try) nbtries=`wc -l $statusfile | cut -d' ' -f1` echo "Nb of tries: $nbtries" #----------------------------------- # Set output file name from list name fileout1=$( basename $set ) fileout=${fileout1%.list} filefirst=`head -n 1 $set` diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` outputfile=$diroutputfile/${fileout} mkdir -p $diroutputfile echo "Output file to write: $outputfile" #----------------------------------- # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*') extratar=0 find_varstoexclude $set if [ $nbvars -eq 0 ] ; then varstoexclude="t_inst_.*" else # There are others variables than 't_inst_.*' so an extra tar file will be created varstoexclude="t_inst_.*,$varstoexcludefiltered" extratar=1 # Copy to output_tar directory echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile cp $set $PATH_SIMU/output_tar fi #----------------------------------- # Try 2 times before fire "FAILED" ncrcatcmd=ncrcat ncrcatoptions="--md5_digest" # ncrcatoptions="" startTime=$( getDateMilliSeconds ) # gpdebug : time resCmd=0 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir resCmd=$? meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time # resultCmd=$( gives_0_or_1 ) # resultCmd=1 # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)" echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile else # Add --md5_digest rm -f $outputfile*ncrcat.tmp startTime=$( getDateMilliSeconds ) # gpdebug : time resCmd=0 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir resCmd=$? meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time # resultCmd=$( gives_0_or_1 ) # resultCmd=1 # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)" echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile else datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` rm -f $outputfile*ncrcat.tmp echo "====> FAILED at try #$nbtries" echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile if [ $nbtries -ge $maxtries ] ; then echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile cp $set $PATH_SIMU/output_tar # gpdebug : a retablir fi fi fi fi #------------------------------------------------------ if [ "$dirnameOfListFile" == "output_tar" ] then #----------------------------------- echo "#-------------------" echo "Set: $set" datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` statusfile=${set%%.list}.status logfile=${set%%.list}_${datestr}.log #----------------------------------- # Create status file if not exists if [ ! -e $statusfile ] ; then echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile fi #----------------------------------- # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then printf "====> " tail -1 $statusfile exit 0 fi #----------------------------------- # nbtries = number of lines from status file (first comment line gives 1 for the first try) nbtries=`wc -l $statusfile | cut -d' ' -f1` # echo "Nb of tries: $nbtries" #----------------------------------- # Set output file name from list name fileout1=$( basename $set ) fileout=${fileout1%.list}.tar filefirst=`head -n 1 $set` dirinputfile=`dirname $filefirst` diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` outputfile=$diroutputfile/${fileout} mkdir -p $diroutputfile echo "Output file to write: $outputfile" #----------------------------------- # Try 1 time before fire "FAILED" # Waiting for CCRT command tarcmd=tar taroptions="--format=posix -W -cf" cd $dirinputfile set_local=$set".local" for file in $( cat $set) ; do basename $file >> $set".local" done # #----------------------------------- startTime=$( getDateMilliSeconds ) # gpdebug : time resCmd=0 $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir resCmd=$? meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time # resultCmd=$( gives_0_or_1 ) # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile else datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "====> FAILED at try #$nbtries" echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile fi rm -f $set".local" #----------------------------------- fi #------------------------------------------------------ if [ "$dirnameOfListFile" == "restart_tar" ] then #----------------------------------- echo "#-------------------" echo "Set: $set" datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` statusfile=${set%%.list}.status logfile=${set%%.list}_${datestr}.log #----------------------------------- # Create status file if not exists if [ ! -e $statusfile ] ; then echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile fi #----------------------------------- # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then printf "====> " tail -1 $statusfile exit 0 fi #----------------------------------- # nbtries = number of lines from status file (first comment line gives 1 for the first try) nbtries=`wc -l $statusfile | cut -d' ' -f1` # echo "Nb of tries: $nbtries" #----------------------------------- # Set output file name from list name dirin1=$PATH_SIMU/RESTART dirin2=${dirin1##${IGCM_DEM}} dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} diroutputfile=${OUTPUT_STORE}${dirin3} fileout1=$( basename $set ) fileout=${fileout1%.list}.tar outputfile=$diroutputfile/${fileout} mkdir -p $diroutputfile echo "Output file to write: $outputfile" #----------------------------------- # Try 1 time before fire "FAILED" # Waiting for CCRT command tarcmd=tar taroptions="--format=posix -W -cf" cd $dirin1 # ls startTime=$( getDateMilliSeconds ) # gpdebug : time resCmd=0 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir resCmd=$? meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time # resultCmd=$( gives_0_or_1 ) # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile else datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "====> FAILED at try #$nbtries" echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile fi #----------------------------------- fi #------------------------------------------------------ if [ "$dirnameOfListFile" == "debug_tar" ] then #----------------------------------- echo "#-------------------" echo "Set: $set" datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` statusfile=${set%%.list}.status logfile=${set%%.list}_${datestr}.log #----------------------------------- # Create status file if not exists if [ ! -e $statusfile ] ; then echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile fi #----------------------------------- # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then printf "====> " tail -1 $statusfile exit 0 fi #----------------------------------- # nbtries = number of lines from status file (first comment line gives 1 for the first try) nbtries=`wc -l $statusfile | cut -d' ' -f1` # echo "Nb of tries: $nbtries" #----------------------------------- # Set output file name from list name dirin1=$PATH_SIMU/DEBUG dirin2=${dirin1##${IGCM_DEM}} dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} diroutputfile=${OUTPUT_STORE}${dirin3} fileout1=$( basename $set ) fileout=${fileout1%.list}.tar outputfile=$diroutputfile/${fileout} mkdir -p $diroutputfile echo "Output file to write: $outputfile" #----------------------------------- # Try 1 time before fire "FAILED" # Waiting for CCRT command tarcmd=tar taroptions="--format=posix -W -cf" cd $dirin1 # ls resCmd=0 startTime=$( getDateMilliSeconds ) # gpdebug : time $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir resCmd=$? meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time # resultCmd=$( gives_0_or_1 ) # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile else datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "====> FAILED at try #$nbtries" echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile fi #----------------------------------- fi #------------------------------------------------------ if [ "$dirnameOfListFile" == "store_cp" ] then #----------------------------------- echo "#-------------------" echo "Set: $set" datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` statusfile=${set%%.list}.status logfile=${set%%.list}_${datestr}.log #----------------------------------- # Create status file if not exists if [ ! -e $statusfile ] ; then echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile fi #----------------------------------- # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then printf "====> " tail -1 $statusfile exit 0 fi #----------------------------------- # nbtries = number of lines from status file (first comment line gives 1 for the first try) nbtries=`wc -l $statusfile | cut -d' ' -f1` # echo "Nb of tries: $nbtries" meantime=0 resCmd=0 startTime=$( getDateMilliSeconds ) # gpdebug : time #----------------------------------- # Set output file name from first and last files from the current set for file in $( cat $set) ; do diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"` mkdir -p $diroutputfile echo "Output file to write: $diroutputfile/$( basename $file )" #----------------------------------- # Try 1 time before fire "FAILED" # Waiting for CCRT command cpcmd="cp " cpoptions="-rf" $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir resCmd=$(( $resCmd || $? )) done meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time # resultCmd=$( gives_0_or_1 ) # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile else datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "====> FAILED at try #$nbtries" echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile fi #----------------------------------- fi #------------------------------------------------------ if [ "$dirnameOfListFile" == "work_cp" ] then #----------------------------------- echo "#-------------------" echo "Set: $set" datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` statusfile=${set%%.list}.status logfile=${set%%.list}_${datestr}.log #----------------------------------- # Create status file if not exists if [ ! -e $statusfile ] ; then echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile fi #----------------------------------- # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then printf "====> " tail -1 $statusfile exit 0 fi #----------------------------------- # nbtries = number of lines from status file (first comment line gives 1 for the first try) nbtries=`wc -l $statusfile | cut -d' ' -f1` # echo "Nb of tries: $nbtries" meantime=0 resCmd=0 startTime=$( getDateMilliSeconds ) # gpdebug : time #----------------------------------- # Set output file name from first and last files from the current set for file in $( cat $set) ; do diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"` mkdir -p $diroutputfile echo "Output file to write: $diroutputfile/$( basename $file )" #----------------------------------- # Try 1 time before fire "FAILED" # Waiting for CCRT command cpcmd="cp " cpoptions="-rf" $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir resCmd=$(( $resCmd || $? )) done meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS # resultCmd=$( gives_0_or_1 ) # if [ $resultCmd -eq 0 ] ; then if [ $resCmd -eq 0 ] ; then sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile else datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` echo "====> FAILED at try #$nbtries" echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile fi #----------------------------------- fi