Changeset 591 for trunk/libIGCM/AA_TimeSeries_Checker
- Timestamp:
- 03/21/12 17:48:52 (12 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/libIGCM/AA_TimeSeries_Checker
r554 r591 41 41 POST_DIR=${POST_DIR:=${CARD_DIR}/OutScript} 42 42 43 # project 44 export BRIDGE_MSUB_PROJECT=gen2211 45 43 46 if [ ! -d ${CARD_DIR} ]; then 44 45 47 echo "No ${CARD_DIR}, we stop here" 48 exit 46 49 fi 47 50 … … 65 68 read ActionAnswer 66 69 67 case ${ActionAnswer} in 68 69 70 71 72 73 74 75 76 77 78 79 80 81 70 case ${ActionAnswer} in 71 oui|OUI|o|y|yes|YES) 72 echo "OK. I will submit jobs but ask permissions before removing files" 73 action=true 74 ;; 75 non|NON|n|no|NO) 76 echo "OK. I won't submit jobs and only print out files I could remove" 77 action=false 78 ;; 79 *) 80 echo "I did not recognize your answer. I will stop here." 81 echo "Please respond: oui|OUI|o|y|yes|YES" 82 echo "Or please respond: non|NON|n|no|NO" 83 exit 84 ;; 82 85 esac 83 86 87 #================================== 84 88 # First of all 89 # 90 typeset option 91 92 #================================== 93 # Read UserChoices section: 85 94 IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card UserChoices 86 typeset option 95 echo 96 IGCM_debug_Print 1 "DefineArrayFromOption : config_UserChoices" 87 97 for option in ${config_UserChoices[*]} ; do 88 98 IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card UserChoices ${option} 99 eval auxprint=\${config_UserChoices_${option}} 100 IGCM_debug_Print 3 "${option} : ${auxprint}" 89 101 done 90 91 echo 92 IGCM_debug_Print 1 "DefineVariableFromOption : config_UserChoices" 93 IGCM_debug_PrintVariables 3 config_UserChoices_JobName 94 #IGCM_debug_PrintVariables 3 config_UserChoices_SpaceName 95 #IGCM_debug_PrintVariables 3 config_UserChoices_ExperimentName 96 IGCM_debug_PrintVariables 3 config_UserChoices_CalendarType 97 IGCM_debug_PrintVariables 3 config_UserChoices_DateBegin 98 IGCM_debug_PrintVariables 3 config_UserChoices_DateEnd 102 # 103 echo 104 105 #================================== 106 # Read Post section: 107 IGCM_card_DefineArrayFromSection ${CARD_DIR}/config.card Post 108 echo 109 IGCM_debug_Print 1 "DefineArrayFromOption : config_Post" 110 111 for option in ${config_Post[*]} ; do 112 IGCM_card_DefineVariableFromOption ${CARD_DIR}/config.card Post ${option} 113 eval auxprint=\${config_Post_${option}} 114 IGCM_debug_Print 3 "${option} : ${auxprint}" 115 done 116 # 117 echo 99 118 100 119 if [ -f ${CARD_DIR}/run.card ] ; then 101 102 103 104 105 106 107 108 109 110 111 112 113 114 120 IGCM_card_DefineVariableFromOption ${CARD_DIR}/run.card Configuration OldPrefix 121 IGCM_card_DefineVariableFromOption ${CARD_DIR}/run.card Configuration PeriodState 122 IGCM_card_DefineVariableFromOption ${CARD_DIR}/run.card PostProcessing TimeSeriesCompleted 123 IGCM_debug_Print 1 "DefineVariableFromOption : run_Configuration" 124 IGCM_debug_PrintVariables 3 run_Configuration_OldPrefix 125 IGCM_debug_PrintVariables 3 run_Configuration_PeriodState 126 IGCM_debug_Print 1 "DefineVariableFromOption : run_PostProcessing" 127 IGCM_debug_PrintVariables 3 run_PostProcessing_TimeSeriesCompleted 128 if [ X${run_Configuration_PeriodState} != X"Completed" ] ; then 129 DateEnd=$( IGCM_date_ConvertFormatToGregorian $( echo ${run_Configuration_OldPrefix} | awk -F'_' '{print $2}' ) ) 130 else 131 DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} ) 132 fi 133 CompletedFlag=${run_PostProcessing_TimeSeriesCompleted} 115 134 else 116 117 135 DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} ) 136 CompletedFlag="" 118 137 fi 119 138 SavedCompletedFlag=${CompletedFlag} … … 133 152 #R_BUFR : Job output buffered directory 134 153 if ( [ ! X${config_UserChoices_SpaceName} = X ] && [ ! X${config_UserChoices_ExperimentName} = X ] ) ; then 135 136 137 138 154 FreeName=$( echo ${config_UserChoices_JobName} | sed 's/.*_//' ) 155 R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName} 156 R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName} 157 R_DODS=${config_UserChoices_TagName}/${config_UserChoices_SpaceName}/${config_UserChoices_ExperimentName}/${FreeName} 139 158 else 140 141 142 159 R_SAVE=${R_OUT}/${config_UserChoices_TagName}/${config_UserChoices_JobName} 160 R_BUFR=${R_BUF}/${config_UserChoices_TagName}/${config_UserChoices_JobName} 161 R_DODS=${config_UserChoices_TagName}/${config_UserChoices_JobName} 143 162 fi 144 163 … … 146 165 147 166 for comp in ${config_ListOfComponents[*]} ; do 148 # Debug Print 149 IGCM_debug_Print 1 ${comp} 150 # Define component 151 IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp} 152 eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1 153 eval comptagname=\${config_ListOfComponents_${comp}[1]} > /dev/null 2>&1 154 155 # Read libIGCM compatibility version in ${compname}.card 156 card=${CARD_DIR}/COMP/${compname}.card 157 158 # Read and Build Output File stuff 159 #IGCM_debug_Print 1 "DefineArrayFromOption : ${compname}_OutputFiles ${card}" 160 IGCM_card_DefineArrayFromOption ${card} OutputFiles List 161 ListFilesName=${compname}_OutputFiles_List 162 eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1 167 # Debug Print 168 IGCM_debug_Print 1 ${comp} 169 # Define component 170 IGCM_card_DefineArrayFromOption ${CARD_DIR}/config.card ListOfComponents ${comp} 171 eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1 172 eval comptagname=\${config_ListOfComponents_${comp}[1]} > /dev/null 2>&1 173 174 # Read libIGCM compatibility version in ${compname}.card 175 card=${CARD_DIR}/COMP/${compname}.card 176 177 # Read and Build Output File stuff 178 #IGCM_debug_Print 1 "DefineArrayFromOption : ${compname}_OutputFiles ${card}" 179 IGCM_card_DefineArrayFromOption ${card} OutputFiles List 180 ListFilesName=${compname}_OutputFiles_List 181 eval FileName0=\${${ListFilesName}[0]} > /dev/null 2>&1 182 # 183 if [ X${FileName0} != X${NULL_STR} ] ; then 163 184 # 164 if [ X${FileName0} != X${NULL_STR} ] ; then 185 #IGCM_debug_Print 1 "Component : ${compname}" 186 # 187 # INITIALISATION 188 # 189 eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1 190 i=2 191 # 192 until [ $i -ge $NbFiles ]; do 193 # 194 eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1 195 # 196 if [ X${flag_post} != XNONE ] ; then 165 197 # 166 #IGCM_debug_Print 1 "Component : ${compname}" 167 # 168 # INITIALISATION 169 # 170 eval NbFiles=\${#${ListFilesName}[@]} > /dev/null 2>&1 171 i=2 172 # 173 until [ $i -ge $NbFiles ]; do 174 # 175 eval flag_post=\${${ListFilesName}[$i]} > /dev/null 2>&1 176 # 177 if [ X${flag_post} != XNONE ] ; then 178 # 179 # First of all 198 # First of all 199 # 200 IGCM_card_DefineArrayFromSection ${card} ${flag_post} 201 # 202 IGCM_card_DefineArrayFromOption ${card} ${flag_post} TimeSeriesVars 203 IGCM_card_DefineArrayFromOption ${card} ${flag_post} Patches 204 if [ X"$( eval echo \${${compname}_${flag_post}_TimeSeriesVars[*]} )" = X"Option not" ] ; then 205 # New TimeSeriesVar description, with 2D, 3D and associate ChunckJob. 206 unset ListDimension 207 ListDimension[0]=2D 208 ListDimension[1]=3D 209 TimeSeries=false 210 TimeSeries2D=false 211 TimeSeries3D=false 212 chunck=false 213 iLoop=${#ListDimension[*]} 214 j=0 215 until [ $j -ge ${iLoop} ]; do 216 Dimension=${ListDimension[${j}]} 217 IGCM_card_DefineArrayFromOption ${card} ${flag_post} TimeSeriesVars${Dimension} 218 IGCM_card_DefineVariableFromOption ${card} ${flag_post} ChunckJob${Dimension} 219 # 220 # Time series WITHOUT chunk 221 # 222 if [ ! $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then 223 if [ $( eval echo \${${compname}_${flag_post}_ChunckJob${Dimension}} ) = NONE ] ; then 224 IGCM_debug_Print 2 "${Dimension} time series activated for ${flag_post}" 225 eval TimeSeries${Dimension}=true 226 chunck=false 227 fi 228 fi 229 # 230 # Time series WITH chunk 231 # 232 if [ ! $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then 233 chunck_size=$( eval echo \${${compname}_${flag_post}_ChunckJob${Dimension}} ) 234 if [ ! ${chunck_size} = NONE ] && [ ! ${chunck_size} = OFF ] ; then 235 IGCM_debug_Print 2 "${Dimension} time series activated with ${chunck_size} chunck for ${flag_post}" 236 eval TimeSeriesChunck${Dimension}=true 237 chunck=true 238 fi 239 fi 240 (( j=j+1 )) 241 # 242 # If TimeSeriesVars list is empty or OFF we skip 243 # 244 if ( [ $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] || \ 245 [ ${chunck_size} = OFF ] ) ; then 246 #IGCM_debug_Print 2 "Empty TS : ${compname}_Post_${FILE}_TimeSeriesVars${Dimension}" 247 #(( i=i+3 )) 248 continue 249 fi 250 # 251 # ICI ON TESTE QUE LES FICHIERS TS SONT LA! 252 # 253 FILE=$( echo ${flag_post} | awk "-FPost_" '{print $2}' ) 254 IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension} 255 IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension} 256 # 257 FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' ) 258 case ${FlagDir} in 259 *Y) TS_Dir=TS_YE ;; 260 *M) TS_Dir=TS_MO ;; 261 *D) TS_Dir=TS_DA ;; 262 3H|HF) TS_Dir=TS_HF ;; 263 INS) TS_Dir=TS_INS ;; 264 esac 265 # 266 # We need LIST of variables not allready produced (useful for standalone mode) 267 # 268 DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} ) 269 IGCM_date_GetYearMonth ${DateBegin} YearBegin MonthBegin 270 271 # Si on n'a pas de chunck, dans ce cas chunck_size=la durée de la simulation en années 272 YearsChunckLength=$( echo ${chunck_size} | sed -e "s/[yY]//" ) 273 [ ${chunck} = false ] && YearsChunckLength=$(( YearEnd - YearBegin + 1 )) 274 275 NbYearsChunckLoop=$(( ( YearEnd - YearBegin + 1 ) / YearsChunckLength )) 276 Reste=$(( ( YearEnd - YearBegin + 1 ) % YearsChunckLength )) 277 278 if [ ${Reste} -ne 0 ] ; then 279 NbYearsChunckLoop=$(( NbYearsChunckLoop + 1 )) 280 fi 281 282 if [ ${NbYearsChunckLoop} -eq 1 ] ; then 283 PeriodDateEnd=${DateEnd} 284 else 285 DaysInYear=$( IGCM_date_DaysInYear ${YearBegin} ) 286 PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( YearsChunckLength * DaysInYear - 1 )) ) 287 fi 288 # 289 DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir} 290 YearsChunckLoop=1 291 ChunckDebut=${DateBegin} 292 ChunckFin=${PeriodDateEnd} 293 while [ ${YearsChunckLoop} -le ${NbYearsChunckLoop} ] ; do 294 countTotal=0 295 countGood=0 296 countBad=0 297 unset RemoveList 298 unset PotentialCompletedFlag 299 for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do 300 TestedFile=${config_UserChoices_JobName}_${ChunckDebut}_${ChunckFin}_${FlagDir}_${var}.nc 180 301 # 181 IGCM_card_DefineArrayFromSection ${card} ${flag_post} 182 # 183 IGCM_card_DefineArrayFromOption ${card} ${flag_post} TimeSeriesVars 184 IGCM_card_DefineArrayFromOption ${card} ${flag_post} Patches 185 if [ X"$( eval echo \${${compname}_${flag_post}_TimeSeriesVars[*]} )" = X"Option not" ] ; then 186 # New TimeSeriesVar description, with 2D, 3D and associate ChunckJob. 187 unset ListDimension 188 ListDimension[0]=2D 189 ListDimension[1]=3D 190 TimeSeries=false 191 TimeSeries2D=false 192 TimeSeries3D=false 193 chunck=false 194 iLoop=${#ListDimension[*]} 195 j=0 196 until [ $j -ge ${iLoop} ]; do 197 Dimension=${ListDimension[${j}]} 198 IGCM_card_DefineArrayFromOption ${card} ${flag_post} TimeSeriesVars${Dimension} 199 IGCM_card_DefineVariableFromOption ${card} ${flag_post} ChunckJob${Dimension} 200 # 201 # Time series WITHOUT chunk 202 # 203 if [ ! $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then 204 if [ $( eval echo \${${compname}_${flag_post}_ChunckJob${Dimension}} ) = NONE ] ; then 205 IGCM_debug_Print 2 "${Dimension} time series activated for ${flag_post}" 206 eval TimeSeries${Dimension}=true 207 chunck=false 208 fi 209 fi 210 # 211 # Time series WITH chunk 212 # 213 if [ ! $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] ; then 214 chunck_size=$( eval echo \${${compname}_${flag_post}_ChunckJob${Dimension}} ) 215 if [ ! ${chunck_size} = NONE ] && [ ! ${chunck_size} = OFF ] ; then 216 IGCM_debug_Print 2 "${Dimension} time series activated with ${chunck_size} chunck for ${flag_post}" 217 eval TimeSeriesChunck${Dimension}=true 218 chunck=true 219 fi 220 fi 221 (( j=j+1 )) 222 # 223 # If TimeSeriesVars list is empty or OFF we skip 224 # 225 if ( [ $( eval echo \${${compname}_${flag_post}_TimeSeriesVars${Dimension}} ) = ${NULL_STR} ] || \ 226 [ ${chunck_size} = OFF ] ) ; then 227 #IGCM_debug_Print 2 "Empty TS : ${compname}_Post_${FILE}_TimeSeriesVars${Dimension}" 228 #(( i=i+3 )) 229 continue 230 fi 231 # 232 # ICI ON TESTE QUE LES FICHIERS TS SONT LA! 233 # 234 # 235 FILE=$( echo ${flag_post} | awk "-FPost_" '{print $2}' ) 236 IGCM_card_DefineArrayFromOption ${card} Post_${FILE} TimeSeriesVars${Dimension} 237 IGCM_card_DefineVariableFromOption ${card} Post_${FILE} ChunckJob${Dimension} 238 # 239 FlagDir=$( echo ${FILE} | awk -F "_" '{print $1}' ) 240 case ${FlagDir} in 241 *Y) TS_Dir=TS_YE ;; 242 *M) TS_Dir=TS_MO ;; 243 *D) TS_Dir=TS_DA ;; 244 3H|HF) TS_Dir=TS_HF ;; 245 INS) TS_Dir=TS_INS ;; 246 esac 247 # 248 # We need LIST of variables not allready produced (useful for standalone mode) 249 # 250 DateBegin=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateBegin} ) 251 IGCM_date_GetYearMonth ${DateBegin} YearBegin MonthBegin 252 253 # Si on n'a pas de chunck, dans ce cas chunck_size=la durée de la simulation en années 254 YearsChunckLength=$( echo ${chunck_size} | sed -e "s/[yY]//" ) 255 [ ${chunck} = false ] && YearsChunckLength=$(( YearEnd - YearBegin + 1 )) 256 257 NbYearsChunckLoop=$(( ( YearEnd - YearBegin + 1 ) / YearsChunckLength )) 258 Reste=$(( ( YearEnd - YearBegin + 1 ) % YearsChunckLength )) 259 260 if [ ${Reste} -ne 0 ] ; then 261 NbYearsChunckLoop=$(( NbYearsChunckLoop + 1 )) 262 fi 263 264 if [ ${NbYearsChunckLoop} -eq 1 ] ; then 265 PeriodDateEnd=${DateEnd} 266 else 267 DaysInYear=$( IGCM_date_DaysInYear ${YearBegin} ) 268 PeriodDateEnd=$( IGCM_date_AddDaysToGregorianDate ${DateBegin} $(( YearsChunckLength * DaysInYear - 1 )) ) 269 fi 270 # 271 DIRECTORY=${R_SAVE}/${comp}/Analyse/${TS_Dir} 272 YearsChunckLoop=1 273 ChunckDebut=${DateBegin} 274 ChunckFin=${PeriodDateEnd} 275 while [ ${YearsChunckLoop} -le ${NbYearsChunckLoop} ] ; do 276 countTotal=0 277 countGood=0 278 countBad=0 279 unset RemoveList 280 unset PotentialCompletedFlag 281 for var in $( eval echo \${${compname}_Post_${FILE}_TimeSeriesVars${Dimension}[*]} ) ; do 282 TestedFile=${config_UserChoices_JobName}_${ChunckDebut}_${ChunckFin}_${FlagDir}_${var}.nc 283 # 284 if [ -d ${DIRECTORY} ] ; then 285 Candidat=$( find ${DIRECTORY} -name "${config_UserChoices_JobName}_${ChunckDebut}_*_${FlagDir}_${var}.nc" ) 286 else 287 Candidat="" 288 fi 289 # 290 BestCandidat=${ChunckDebut} 291 # 292 FoundCandidat=false 293 if [ ! "X${Candidat}" = "X" ] ; then 294 for candid in ${Candidat} ; do 295 #IGCM_debug_Print 1 -e "\033[1;32m[CompletedFlag CANDIDAT MATCH FOUND]\033[m $( basename ${candid} )" 296 CandidatEnd=$( echo ${candid} | awk -F${config_UserChoices_JobName} '{print $3}' | awk -F_ '{print $3}' ) 297 if [ ${CandidatEnd} -gt ${ChunckFin} ] ; then 298 #IGCM_debug_Print 3 -e "File $( basename ${candid} ) should be suppress" 299 set +A RemoveList ${RemoveList[*]} ${candid} 300 elif [ ${CandidatEnd} -lt ${ChunckFin} ] ; then 301 #IGCM_debug_Print 3 -e "File $( basename ${candid} ) should be suppress or used as a CompletedFlag baseline" 302 # We keep the best candidat and remove the others 303 if [ ${BestCandidat} -lt ${CandidatEnd} ] ; then 304 if [ ${FoundCandidat} = true ] ; then 305 set +A RemoveList ${RemoveList[*]} ${DIRECTORY}/${config_UserChoices_JobName}_${ChunckDebut}_${BestCandidat}_${FlagDir}_${var}.nc 306 else 307 FoundCandidat=true 308 fi 309 BestCandidat=${CandidatEnd} 310 else 311 set +A RemoveList ${RemoveList[*]} ${candid} 312 fi 313 fi 314 done 315 else 316 # No CompletedFlag candidat found 317 CandidatCompletedFlag="" 318 fi 319 # 320 if [ ! -f ${DIRECTORY}/${TestedFile} ] ; then 321 (( countBad = countBad + 1 )) 322 [ ${countBad} = 1 ] && IGCM_debug_Print 3 "Missing time series from ${FILE} :" 323 IGCM_debug_Print 3 ${DIRECTORY}/${TestedFile} 324 # Then we keep a track of the Best Candidat 325 if [ ${FoundCandidat} = true ] ; then 326 if [ ${chunck} = true ] ; then 327 set +A PotentialCompletedFlag ${PotentialCompletedFlag[*]} ${BestCandidat} 328 else 329 eval set +A PotentialCompletedFlag${Dimension} \${PotentialCompletedFlag${Dimension}[*]} \${BestCandidat} 330 fi 331 fi 332 else 333 (( countGood = countGood + 1 )) 334 # Then we do not need the best candidat if any 335 [ ${FoundCandidat} = true ] && set +A RemoveList ${RemoveList[*]} ${DIRECTORY}/${config_UserChoices_JobName}_${ChunckDebut}_${BestCandidat}_${FlagDir}_${var}.nc 336 fi 337 (( countTotal = countTotal + 1 )) 338 done 339 340 SuccessRate=$(( countGood * 100 / countTotal )) 341 if [ ${SuccessRate} -ne 100 ] ; then 342 IGCM_debug_Print 2 -e "\033[1;31m${SuccessRate}% files OK.\033[m for period ${ChunckDebut}-${ChunckFin}" 343 else 344 IGCM_debug_Print 2 -e "\033[1;32m${SuccessRate}% files OK.\033[m for period ${ChunckDebut}-${ChunckFin}" 345 fi 346 echo 347 348 # Remove what's in the RemoveList 349 if [ ${RemoveList} ] ; then 350 IGCM_debug_Print 2 "File(s) that will be removed ${RemoveList[*]}" 351 if [ ${action} = true ] ; then 352 echo -e "\033[1;32mDo you want to remove them?\033[m" 353 echo -n " Your answer (y/n) : " 354 read RemoveAnswer 355 case ${RemoveAnswer} in 356 oui|OUI|o|y|yes|YES) 357 echo "OK. I will erase them" 358 IGCM_sys_Rm -f ${RemoveList[*]} 359 ;; 360 non|NON|n|no|NO) 361 echo "OK. I won't erase them" 362 ;; 363 *) 364 echo "I did not recognize your answer." 365 echo "Please respond: oui|OUI|o|y|yes|YES" 366 echo "Or please respond: non|NON|n|no|NO" 367 exit 368 ;; 369 esac 370 fi 371 fi 372 #[ ${FoundCandidat} = true ] && IGCM_debug_Print 2 "Potentials CompletedFlag: ${PotentialCompletedFlag[*]}" 373 if ( [ ${chunck} = true ] && [ ${SuccessRate} -ne 100 ] ) ; then 374 # 375 # TO REWRITE BEGIN 376 # 377 NbCandidats=${#PotentialCompletedFlag[@]} 378 countCandid=1 379 if [ ${NbCandidats} -eq 0 ] ; then 380 ChunckCompletedFlag="" 381 IGCM_debug_Print 2 -e "\033[1;31mNO CompletedFlag\033[m" 382 elif [ ${NbCandidats} -eq 1 ] ; then 383 ChunckCompletedFlag=${PotentialCompletedFlag[0]} 384 IGCM_debug_Print 2 -e "\033[1;32mCompletedFlag=${ChunckCompletedFlag}\033[m" 385 elif [ ${NbCandidats} -gt 1 ] ; then 386 k=1 387 until [ $k -ge ${NbCandidats} ]; do 388 ((k_m1=k-1)) 389 if [ ${PotentialCompletedFlag[${k_m1}]} = ${PotentialCompletedFlag[${k}]} ] ; then 390 ((countCandid=countCandid+1)) 391 fi 392 ((k=k+1)) 393 done 394 if [ ${NbCandidats} -eq ${countCandid} ] ; then 395 ChunckCompletedFlag=${PotentialCompletedFlag[0]} 396 IGCM_debug_Print 2 -e "\033[1;32mCompletedFlag=${ChunckCompletedFlag}\033[m" 397 else 398 ChunckCompletedFlag=${PotentialCompletedFlag[0]} 399 IGCM_debug_Print 2 -e "\033[1;31mToo Many Potential CompletedFlag\033[m Let start with first CompletedPeriod" 400 IGCM_debug_Print 2 -e "\033[1;31mYou will have to redo TimeSeries_Checker\033[m for other period." 401 fi 402 fi 403 # 404 # TO REWRITE END 405 # 406 if [ ${action} = true ] ; then 407 IGCM_debug_Print 2 -e "\033[1;31mSubmit ${FILE} chunck ${Dimension}\033[m period ${ChunckDebut}-${ChunckFin} and CompletedFlag=${ChunckCompletedFlag}" 408 listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask,CompToRead,FlagToRead" 409 export libIGCM=${libIGCM} 410 export SUBMIT_DIR=${CARD_DIR} 411 export POST_DIR=${POST_DIR} 412 export DateBegin=${ChunckDebut} 413 export PeriodDateEnd=${ChunckFin} 414 export CompletedFlag=${ChunckCompletedFlag} 415 export TsTask=Chunck${Dimension} 416 export CompToRead=${comp} 417 export FlagToRead=${i} 418 export NameToRead=${flag_post} 419 export listVarEnv=${listVarEnv} 420 IGCM_sys_MkdirWork ${POST_DIR} 421 IGCM_debug_Verif_Exit_Post 422 Script_Post_Output=create_ts.${PeriodDateEnd}.${TsTask}.${CompToRead}.${NameToRead} 423 IGCM_sys_QsubPost create_ts 424 echo 425 else 426 IGCM_debug_Print 2 -e "\033[1;31mI should NOW submit ${FILE} chunck ${Dimension}\033[m period ${ChunckDebut}-${ChunckFin} and CompletedFlag=${ChunckCompletedFlag}" 427 fi 428 fi 429 430 if ( [ ${chunck} = false ] && [ ${SuccessRate} -ne 100 ] ) ; then 431 eval Launch${Dimension}=true 432 fi 433 434 # Date update 435 ChunckDebut=$( IGCM_date_AddDaysToGregorianDate ${ChunckFin} 1 ) 436 437 (( YearsChunckLoop = YearsChunckLoop + 1 )) 438 439 if [ ${YearsChunckLoop} -eq ${NbYearsChunckLoop} ] ; then 440 ChunckFin=${DateEnd} 441 else 442 ChunckFin=$( IGCM_date_AddDaysToGregorianDate ${ChunckDebut} $(( YearsChunckLength * DaysInYear - 1 )) ) 443 fi 444 done 445 done 446 else 447 ListDimension[0]="" 448 TimeSeries=true 449 TimeSeries2D=false 450 TimeSeries3D=false 451 TimeSeriesChunck2D=false 452 TimeSeriesChunck3D=false 453 fi 454 fi 455 (( i=i+3 )) 456 done 457 fi 302 if [ -d ${DIRECTORY} ] ; then 303 Candidat=$( find ${DIRECTORY} -name "${config_UserChoices_JobName}_${ChunckDebut}_*_${FlagDir}_${var}.nc" ) 304 else 305 Candidat="" 306 fi 307 # 308 BestCandidat=${ChunckDebut} 309 # 310 FoundCandidat=false 311 if [ ! "X${Candidat}" = "X" ] ; then 312 for candid in ${Candidat} ; do 313 #IGCM_debug_Print 1 -e "\033[1;32m[CompletedFlag CANDIDAT MATCH FOUND]\033[m $( basename ${candid} )" 314 CandidatEnd=$( echo ${candid} | awk -F${config_UserChoices_JobName} '{print $3}' | awk -F_ '{print $3}' ) 315 if [ ${CandidatEnd} -gt ${ChunckFin} ] ; then 316 #IGCM_debug_Print 3 -e "File $( basename ${candid} ) should be suppress" 317 set +A RemoveList ${RemoveList[*]} ${candid} 318 elif [ ${CandidatEnd} -lt ${ChunckFin} ] ; then 319 #IGCM_debug_Print 3 -e "File $( basename ${candid} ) should be suppress or used as a CompletedFlag baseline" 320 # We keep the best candidat and remove the others 321 if [ ${BestCandidat} -lt ${CandidatEnd} ] ; then 322 if [ ${FoundCandidat} = true ] ; then 323 set +A RemoveList ${RemoveList[*]} ${DIRECTORY}/${config_UserChoices_JobName}_${ChunckDebut}_${BestCandidat}_${FlagDir}_${var}.nc 324 else 325 FoundCandidat=true 326 fi 327 BestCandidat=${CandidatEnd} 328 else 329 set +A RemoveList ${RemoveList[*]} ${candid} 330 fi 331 fi 332 done 333 else 334 # No CompletedFlag candidat found 335 CandidatCompletedFlag="" 336 fi 337 # 338 if [ ! -f ${DIRECTORY}/${TestedFile} ] ; then 339 (( countBad = countBad + 1 )) 340 [ ${countBad} = 1 ] && IGCM_debug_Print 3 "Missing time series from ${FILE} :" 341 IGCM_debug_Print 3 ${DIRECTORY}/${TestedFile} 342 # Then we keep a track of the Best Candidat 343 if [ ${FoundCandidat} = true ] ; then 344 if [ ${chunck} = true ] ; then 345 set +A PotentialCompletedFlag ${PotentialCompletedFlag[*]} ${BestCandidat} 346 else 347 eval set +A PotentialCompletedFlag${Dimension} \${PotentialCompletedFlag${Dimension}[*]} \${BestCandidat} 348 fi 349 fi 350 else 351 (( countGood = countGood + 1 )) 352 # Then we do not need the best candidat if any 353 [ ${FoundCandidat} = true ] && set +A RemoveList ${RemoveList[*]} ${DIRECTORY}/${config_UserChoices_JobName}_${ChunckDebut}_${BestCandidat}_${FlagDir}_${var}.nc 354 fi 355 (( countTotal = countTotal + 1 )) 356 done 357 358 SuccessRate=$(( countGood * 100 / countTotal )) 359 if [ ${SuccessRate} -ne 100 ] ; then 360 IGCM_debug_Print 2 -e "\033[1;31m${SuccessRate}% files OK.\033[m for period ${ChunckDebut}-${ChunckFin}" 361 else 362 IGCM_debug_Print 2 -e "\033[1;32m${SuccessRate}% files OK.\033[m for period ${ChunckDebut}-${ChunckFin}" 363 fi 364 echo 365 366 # Remove what's in the RemoveList 367 if [ ${RemoveList} ] ; then 368 IGCM_debug_Print 2 "File(s) that will be removed ${RemoveList[*]}" 369 if [ ${action} = true ] ; then 370 echo -e "\033[1;32mDo you want to remove them?\033[m" 371 echo -n " Your answer (y/n) : " 372 read RemoveAnswer 373 case ${RemoveAnswer} in 374 oui|OUI|o|y|yes|YES) 375 echo "OK. I will erase them" 376 IGCM_sys_Rm -f ${RemoveList[*]} 377 ;; 378 non|NON|n|no|NO) 379 echo "OK. I won't erase them" 380 ;; 381 *) 382 echo "I did not recognize your answer." 383 echo "Please respond: oui|OUI|o|y|yes|YES" 384 echo "Or please respond: non|NON|n|no|NO" 385 exit 386 ;; 387 esac 388 fi 389 fi 390 #[ ${FoundCandidat} = true ] && IGCM_debug_Print 2 "Potentials CompletedFlag: ${PotentialCompletedFlag[*]}" 391 # 392 # TO REWRITE BEGIN 393 # 394 if ( [ ${chunck} = true ] && [ ${SuccessRate} -ne 100 ] ) ; then 395 NbCandidats=${#PotentialCompletedFlag[@]} 396 countCandid=1 397 if [ ${NbCandidats} -eq 0 ] ; then 398 ChunckCompletedFlag="" 399 IGCM_debug_Print 2 -e "\033[1;31mNO CompletedFlag\033[m" 400 elif [ ${NbCandidats} -eq 1 ] ; then 401 ChunckCompletedFlag=${PotentialCompletedFlag[0]} 402 IGCM_debug_Print 2 -e "\033[1;32mCompletedFlag=${ChunckCompletedFlag}\033[m" 403 elif [ ${NbCandidats} -gt 1 ] ; then 404 k=1 405 until [ $k -ge ${NbCandidats} ]; do 406 ((k_m1=k-1)) 407 if [ ${PotentialCompletedFlag[${k_m1}]} = ${PotentialCompletedFlag[${k}]} ] ; then 408 ((countCandid=countCandid+1)) 409 fi 410 ((k=k+1)) 411 done 412 if [ ${NbCandidats} -eq ${countCandid} ] ; then 413 ChunckCompletedFlag=${PotentialCompletedFlag[0]} 414 IGCM_debug_Print 2 -e "\033[1;32mCompletedFlag=${ChunckCompletedFlag}\033[m" 415 else 416 ChunckCompletedFlag=${PotentialCompletedFlag[0]} 417 IGCM_debug_Print 2 -e "\033[1;31mToo Many Potential CompletedFlag\033[m Let start with first CompletedPeriod" 418 IGCM_debug_Print 2 -e "\033[1;31mYou will have to redo TimeSeries_Checker\033[m for other period." 419 fi 420 fi 421 # 422 # TO REWRITE END 423 # 424 if [ ${action} = true ] ; then 425 IGCM_debug_Print 2 -e "\033[1;31mSubmit ${FILE} chunck ${Dimension}\033[m period ${ChunckDebut}-${ChunckFin} and CompletedFlag=${ChunckCompletedFlag}" 426 listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask,CompToRead,FlagToRead" 427 export libIGCM=${libIGCM} 428 export SUBMIT_DIR=${CARD_DIR} 429 export POST_DIR=${POST_DIR} 430 export DateBegin=${ChunckDebut} 431 export PeriodDateEnd=${ChunckFin} 432 export CompletedFlag=${ChunckCompletedFlag} 433 export TsTask=Chunck${Dimension} 434 export CompToRead=${comp} 435 export FlagToRead=${i} 436 export NameToRead=${flag_post} 437 export listVarEnv=${listVarEnv} 438 IGCM_sys_MkdirWork ${POST_DIR} 439 IGCM_debug_Verif_Exit_Post 440 Script_Post_Output=create_ts.${PeriodDateEnd}.${TsTask}.${CompToRead}.${NameToRead} 441 IGCM_sys_QsubPost create_ts 442 echo 443 else 444 IGCM_debug_Print 2 -e "\033[1;31mI should NOW submit ${FILE} chunck ${Dimension}\033[m period ${ChunckDebut}-${ChunckFin} and CompletedFlag=${ChunckCompletedFlag}" 445 fi 446 fi 447 448 if ( [ ${chunck} = false ] && [ ${SuccessRate} -ne 100 ] ) ; then 449 eval Launch${Dimension}=true 450 fi 451 452 # Date update 453 ChunckDebut=$( IGCM_date_AddDaysToGregorianDate ${ChunckFin} 1 ) 454 455 (( YearsChunckLoop = YearsChunckLoop + 1 )) 456 457 if [ ${YearsChunckLoop} -eq ${NbYearsChunckLoop} ] ; then 458 ChunckFin=${DateEnd} 459 else 460 ChunckFin=$( IGCM_date_AddDaysToGregorianDate ${ChunckDebut} $(( YearsChunckLength * DaysInYear - 1 )) ) 461 fi 462 done 463 done 464 else 465 ListDimension[0]="" 466 TimeSeries=true 467 TimeSeries2D=false 468 TimeSeries3D=false 469 TimeSeriesChunck2D=false 470 TimeSeriesChunck3D=false 471 fi 472 fi 473 (( i=i+3 )) 474 done 475 fi 458 476 done # comp loop 459 477 … … 468 486 469 487 if [ X${Launch2D} = Xtrue ] ; then 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 488 if [ ${action} = true ] ; then 489 #IGCM_debug_Print 2 "Potentials CompletedFlag: ${PotentialCompletedFlag2D[*]}" 490 IGCM_debug_Print 2 -e "\033[1;31mSubmit 2D\033[m without chunck period ${DateBegin}-${DateEnd}" 491 listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask" 492 export libIGCM=${libIGCM} 493 export SUBMIT_DIR=${CARD_DIR} 494 export POST_DIR=${POST_DIR} 495 export DateBegin=${DateBegin} 496 export PeriodDateEnd=${DateEnd} 497 export CompletedFlag=${CompletedFlag} 498 export TsTask=2D 499 export listVarEnv=${listVarEnv} 500 IGCM_sys_MkdirWork ${POST_DIR} 501 IGCM_debug_Verif_Exit_Post 502 Script_Post_Output=create_ts.${PeriodDateEnd}.${TsTask} 503 IGCM_sys_QsubPost create_ts 504 echo 505 else 506 IGCM_debug_Print 2 -e "\033[1;31mI should NOW submit 2D without chunck\033[m period ${DateBegin}-${DateEnd}" 507 fi 490 508 fi 491 509 492 510 if [ X${Launch3D} = Xtrue ] ; then 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 511 if [ ${action} = true ] ; then 512 #IGCM_debug_Print 2 "Potentials CompletedFlag: ${PotentialCompletedFlag3D[*]}" 513 IGCM_debug_Print 2 -e "\033[1;31mSubmit 3D\033[m without chunck period ${DateBegin}-${DateEnd}" 514 listVarEnv="libIGCM,SUBMIT_DIR,POST_DIR,DateBegin,PeriodDateEnd,CompletedFlag,TsTask" 515 export libIGCM=${libIGCM} 516 export SUBMIT_DIR=${CARD_DIR} 517 export POST_DIR=${POST_DIR} 518 export DateBegin=${DateBegin} 519 export PeriodDateEnd=${DateEnd} 520 export CompletedFlag=${CompletedFlag} 521 export TsTask=3D 522 export listVarEnv=${listVarEnv} 523 IGCM_sys_MkdirWork ${POST_DIR} 524 IGCM_debug_Verif_Exit_Post 525 Script_Post_Output=create_ts.${PeriodDateEnd}.${TsTask} 526 IGCM_sys_QsubPost create_ts 527 echo 528 else 529 IGCM_debug_Print 2 -e "\033[1;31mI should NOW submit 3D without chunck\033[m period ${DateBegin}-${DateEnd}" 530 fi 513 531 fi
Note: See TracChangeset
for help on using the changeset viewer.