1 | #!/bin/bash |
---|
2 | # for a list of login we look if some file like "*JobName*" were packed (instead of "JobName*" |
---|
3 | # Create for each simulation a list with all these "bad" files. |
---|
4 | # author : Anne COZIC |
---|
5 | # date : 19/07/2013 |
---|
6 | |
---|
7 | STORELOG="/ccc/scratch/cont003/dsm/p86cozic/pack_ipsl_logs" |
---|
8 | |
---|
9 | for LOGIN in `awk '{print $0}' ${STORELOG}/liste_login` ; do |
---|
10 | |
---|
11 | echo "----------------- $LOGIN" |
---|
12 | IGCM_TMP=$SCRATCHDIR/TMP_PACK |
---|
13 | IGCM_DEM=$SCRATCHDIR/SAVE_PACK/${LOGIN} |
---|
14 | mkdir -p ${IGCM_TMP} |
---|
15 | rm -rf ${IGCM_TMP}/* |
---|
16 | |
---|
17 | mkdir -p $SCRATCHDIR/SAVE_PACK |
---|
18 | mkdir -p ${IGCM_DEM} |
---|
19 | rm -rf ${IGCM_DEM}/* |
---|
20 | |
---|
21 | cd ${STORELOG} |
---|
22 | if [ -s ${LOGIN}.Listing.txt.gz ]; then |
---|
23 | gunzip ${LOGIN}.Listing.txt.gz |
---|
24 | LISTE_DMNFS=${LOGIN}.Listing.txt |
---|
25 | |
---|
26 | else |
---|
27 | tar xf ${LOGIN}.ipsl_logs.tgz ${LOGIN}/IGCM_DEM/Listing.txt |
---|
28 | mv ${LOGIN}/IGCM_DEM/Listing.txt ${LOGIN}.Listing.txt |
---|
29 | rm -rf ${LOGIN}/ |
---|
30 | LISTE_DMNFS=${LOGIN}.Listing.txt |
---|
31 | fi |
---|
32 | |
---|
33 | if [ -s $LISTE_DMNFS ]; then |
---|
34 | |
---|
35 | listRestart=`grep "IGCM_OUT/" ${LISTE_DMNFS} | grep Restart/ | awk '{print $3}' | sed 's%/Restart.*%%' ` |
---|
36 | if [[ ${listRestart} != "" ]] ; then |
---|
37 | echo "${listRestart}" >> ${IGCM_TMP}/${$}.liste_Restart.txt |
---|
38 | fi |
---|
39 | |
---|
40 | while [[ -s ${IGCM_TMP}/${$}.liste_Restart.txt ]] |
---|
41 | do |
---|
42 | DIR=$( awk '{if (NR==1) print $0}' ${IGCM_TMP}/${$}.liste_Restart.txt ) |
---|
43 | |
---|
44 | #gestion des cas particulier tels que la creation de l'etat initial avec lmdz (JobName/ATM/Output/Restart/ au lieu de JobName/ATM/Restart/) |
---|
45 | PATH_SIM=$( dirname ${DIR} ) |
---|
46 | echo ${DIR} |
---|
47 | lastdir=$( echo $PATH_SIM | sed -e "s%.*/%%") |
---|
48 | if [ $lastdir != "ATM" ]; then |
---|
49 | dirname ${DIR} >> ${IGCM_TMP}/${$}.path_simul.txt |
---|
50 | grep -vw ${PATH_SIM} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt |
---|
51 | else |
---|
52 | dirname ${PATH_SIM} >> ${IGCM_TMP}/${$}.path_simul.txt |
---|
53 | PATH_SIM_TMP=$( dirname ${PATH_SIM} ) |
---|
54 | grep -vw ${PATH_SIM_TMP} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt |
---|
55 | |
---|
56 | fi |
---|
57 | |
---|
58 | |
---|
59 | |
---|
60 | done |
---|
61 | echo "----------------------------------------------fin listerestart" |
---|
62 | |
---|
63 | if [[ -s ${IGCM_TMP}/${$}.path_simul.txt ]] ; then |
---|
64 | # On retire les doublons pour avoir la liste des simulations a tester |
---|
65 | awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.path_simul.txt >> ${IGCM_TMP}/${$}.liste_simul |
---|
66 | awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.liste_simul >> ${IGCM_TMP}/${$}.liste_simul.tmp${$} |
---|
67 | mv ${IGCM_TMP}/${$}.liste_simul.tmp${$} ${IGCM_DEM}/liste_simul |
---|
68 | |
---|
69 | |
---|
70 | |
---|
71 | for PATH_SIMUL_FULL in `awk '{print $0}' ${IGCM_DEM}/liste_simul` ; do |
---|
72 | rm -f ${IGCM_TMP}/* |
---|
73 | echo "PATH_SIMUL_FULL = " $PATH_SIMUL_FULL |
---|
74 | |
---|
75 | # Etablir la liste des fichiers de restart |
---|
76 | grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Restart | awk '{print $2 " " $3}' | grep -e '.*/Restart/.*\..*' |grep -v "jnl" | sort >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt |
---|
77 | |
---|
78 | |
---|
79 | # Etablir la liste des fichiers output |
---|
80 | grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output | awk '{print $2 " " $3}' | grep -e '.*/Output/.*\.nc$' | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt |
---|
81 | |
---|
82 | # Lecture du nom du job |
---|
83 | JobName=`awk -F/ '{if (NR==1) print $(NF-3)}' ${IGCM_TMP}/${$}.liste_restart_files_config.txt` |
---|
84 | |
---|
85 | if [ $JobName = "ATM" ]; then |
---|
86 | JobName=`awk -F/ '{if (NR==1) print $(NF-4)}' ${IGCM_TMP}/${$}.liste_restart_files_config.txt` |
---|
87 | fi |
---|
88 | |
---|
89 | echo "JobName = "$JobName |
---|
90 | |
---|
91 | sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt |
---|
92 | cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" > ${IGCM_TMP}/${$}.info_tmp_bis.txt |
---|
93 | join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp |
---|
94 | sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp |
---|
95 | mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_TMP}/${$}.liste_output_files_config.txt |
---|
96 | |
---|
97 | |
---|
98 | rm -f ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_TMP}/${$}.output.tmp ${IGCM_TMP}/${$}.output_bis.tmp |
---|
99 | ## maintenant on va extraire de cette liste les fichiers qui sont du type *${JobName} et non pas ${JobName} |
---|
100 | sed -e "s%.*/${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt |
---|
101 | cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" > ${IGCM_TMP}/${$}.info_tmp_bis.txt |
---|
102 | |
---|
103 | ## la liste que l'on cherche est ${IGCM_TMP}/${$}.info_tmp_bis.txt |
---|
104 | if [ -s ${IGCM_TMP}/${$}.info_tmp_bis.txt ]; then |
---|
105 | mv ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_DEM}/${JobName}.info_bug_pack_output.txt |
---|
106 | fi |
---|
107 | done |
---|
108 | |
---|
109 | fi |
---|
110 | fi |
---|
111 | done |
---|