[2120] | 1 | #!/bin/bash |
---|
| 2 | # for a list of login we look if some file like "*JobName*" were packed (instead of "JobName*" |
---|
| 3 | # Create for each simulation a list with all these "bad" files. |
---|
| 4 | # author : Anne COZIC |
---|
| 5 | # date : 19/07/2013 |
---|
| 6 | |
---|
| 7 | STORELOG="/ccc/scratch/cont003/dsm/p86cozic/pack_ipsl_logs" |
---|
| 8 | |
---|
| 9 | for LOGIN in `awk '{print $0}' ${STORELOG}/liste_login` ; do |
---|
| 10 | |
---|
| 11 | echo "----------------- $LOGIN" |
---|
| 12 | IGCM_TMP=$SCRATCHDIR/TMP_PACK |
---|
| 13 | IGCM_DEM=$SCRATCHDIR/SAVE_PACK/${LOGIN} |
---|
| 14 | mkdir -p ${IGCM_TMP} |
---|
| 15 | rm -rf ${IGCM_TMP}/* |
---|
| 16 | |
---|
| 17 | mkdir -p $SCRATCHDIR/SAVE_PACK |
---|
| 18 | mkdir -p ${IGCM_DEM} |
---|
| 19 | rm -rf ${IGCM_DEM}/* |
---|
| 20 | |
---|
| 21 | cd ${STORELOG} |
---|
| 22 | if [ -s ${LOGIN}.Listing.txt.gz ]; then |
---|
| 23 | gunzip ${LOGIN}.Listing.txt.gz |
---|
| 24 | LISTE_DMNFS=${LOGIN}.Listing.txt |
---|
| 25 | |
---|
| 26 | else |
---|
| 27 | tar xf ${LOGIN}.ipsl_logs.tgz ${LOGIN}/IGCM_DEM/Listing.txt |
---|
| 28 | mv ${LOGIN}/IGCM_DEM/Listing.txt ${LOGIN}.Listing.txt |
---|
| 29 | rm -rf ${LOGIN}/ |
---|
| 30 | LISTE_DMNFS=${LOGIN}.Listing.txt |
---|
| 31 | fi |
---|
| 32 | |
---|
| 33 | if [ -s $LISTE_DMNFS ]; then |
---|
| 34 | |
---|
| 35 | listRestart=`grep "IGCM_OUT/" ${LISTE_DMNFS} | grep Restart/ | awk '{print $3}' | sed 's%/Restart.*%%' ` |
---|
| 36 | if [[ ${listRestart} != "" ]] ; then |
---|
| 37 | echo "${listRestart}" >> ${IGCM_TMP}/${$}.liste_Restart.txt |
---|
| 38 | fi |
---|
| 39 | |
---|
| 40 | while [[ -s ${IGCM_TMP}/${$}.liste_Restart.txt ]] |
---|
| 41 | do |
---|
| 42 | DIR=$( awk '{if (NR==1) print $0}' ${IGCM_TMP}/${$}.liste_Restart.txt ) |
---|
| 43 | |
---|
| 44 | #gestion des cas particulier tels que la creation de l'etat initial avec lmdz (JobName/ATM/Output/Restart/ au lieu de JobName/ATM/Restart/) |
---|
| 45 | PATH_SIM=$( dirname ${DIR} ) |
---|
| 46 | echo ${DIR} |
---|
| 47 | lastdir=$( echo $PATH_SIM | sed -e "s%.*/%%") |
---|
| 48 | if [ $lastdir != "ATM" ]; then |
---|
| 49 | dirname ${DIR} >> ${IGCM_TMP}/${$}.path_simul.txt |
---|
| 50 | grep -vw ${PATH_SIM} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt |
---|
| 51 | else |
---|
| 52 | dirname ${PATH_SIM} >> ${IGCM_TMP}/${$}.path_simul.txt |
---|
| 53 | PATH_SIM_TMP=$( dirname ${PATH_SIM} ) |
---|
| 54 | grep -vw ${PATH_SIM_TMP} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt |
---|
| 55 | |
---|
| 56 | fi |
---|
| 57 | |
---|
| 58 | |
---|
| 59 | |
---|
| 60 | done |
---|
| 61 | echo "----------------------------------------------fin listerestart" |
---|
| 62 | |
---|
| 63 | if [[ -s ${IGCM_TMP}/${$}.path_simul.txt ]] ; then |
---|
| 64 | # On retire les doublons pour avoir la liste des simulations a tester |
---|
| 65 | awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.path_simul.txt >> ${IGCM_TMP}/${$}.liste_simul |
---|
| 66 | awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.liste_simul >> ${IGCM_TMP}/${$}.liste_simul.tmp${$} |
---|
| 67 | mv ${IGCM_TMP}/${$}.liste_simul.tmp${$} ${IGCM_DEM}/liste_simul |
---|
| 68 | |
---|
| 69 | |
---|
| 70 | |
---|
| 71 | for PATH_SIMUL_FULL in `awk '{print $0}' ${IGCM_DEM}/liste_simul` ; do |
---|
| 72 | rm -f ${IGCM_TMP}/* |
---|
| 73 | echo "PATH_SIMUL_FULL = " $PATH_SIMUL_FULL |
---|
| 74 | |
---|
| 75 | # Etablir la liste des fichiers de restart |
---|
| 76 | grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Restart | awk '{print $2 " " $3}' | grep -e '.*/Restart/.*\..*' |grep -v "jnl" | sort >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt |
---|
| 77 | |
---|
| 78 | |
---|
| 79 | # Etablir la liste des fichiers output |
---|
| 80 | grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output | awk '{print $2 " " $3}' | grep -e '.*/Output/.*\.nc$' | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt |
---|
| 81 | |
---|
| 82 | # Lecture du nom du job |
---|
| 83 | JobName=`awk -F/ '{if (NR==1) print $(NF-3)}' ${IGCM_TMP}/${$}.liste_restart_files_config.txt` |
---|
| 84 | |
---|
| 85 | if [ $JobName = "ATM" ]; then |
---|
| 86 | JobName=`awk -F/ '{if (NR==1) print $(NF-4)}' ${IGCM_TMP}/${$}.liste_restart_files_config.txt` |
---|
| 87 | fi |
---|
| 88 | |
---|
| 89 | echo "JobName = "$JobName |
---|
| 90 | |
---|
| 91 | sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt |
---|
| 92 | cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" > ${IGCM_TMP}/${$}.info_tmp_bis.txt |
---|
| 93 | join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp |
---|
| 94 | sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp |
---|
| 95 | mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_TMP}/${$}.liste_output_files_config.txt |
---|
| 96 | |
---|
| 97 | |
---|
| 98 | rm -f ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_TMP}/${$}.output.tmp ${IGCM_TMP}/${$}.output_bis.tmp |
---|
| 99 | ## maintenant on va extraire de cette liste les fichiers qui sont du type *${JobName} et non pas ${JobName} |
---|
| 100 | sed -e "s%.*/${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt |
---|
| 101 | cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" > ${IGCM_TMP}/${$}.info_tmp_bis.txt |
---|
| 102 | |
---|
| 103 | ## la liste que l'on cherche est ${IGCM_TMP}/${$}.info_tmp_bis.txt |
---|
| 104 | if [ -s ${IGCM_TMP}/${$}.info_tmp_bis.txt ]; then |
---|
| 105 | mv ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_DEM}/${JobName}.info_bug_pack_output.txt |
---|
| 106 | fi |
---|
| 107 | done |
---|
| 108 | |
---|
| 109 | fi |
---|
| 110 | fi |
---|
| 111 | done |
---|