source: trunk/libIGCM/AA_RunChecker @ 752

Last change on this file since 752 was 752, checked in by labetoulle, 12 years ago

Replace RunChecker?.job by AA_RunChecker to benefit from the dynamic modipsl path detection
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 11.5 KB
Line 
1#!/bin/ksh
2
3# Created by: S. Labetoulle
4# Date:       May 19th 2010
5#**************************************************************
6# Author: Sonia Labetoulle
7# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
8# $Revision::                                          $ Revision of last commit
9# $Author::                                            $ Author of last commit
10# $Date::                                              $ Date of last commit
11# IPSL (2012)
12#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
13#
14#**************************************************************
15
16#======================================================================#
17
18# Display a report on how a simulation is faring.
19
20
21function SearchCatalog {
22
23  typeset num
24  unset SUBMIT_DIR
25
26  NbOcc=$( awk -v JobName=${JobName} \
27           'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
28
29  if [ ${NbOcc} -eq 0 ] ; then
30    if [ X${fg_new} == X.true. ] ; then
31      echo "${JobName} not in Catalog, we'll try to find it in ${WORK_DIR}"
32     
33      set -A FileList
34      FileList=$( find ${WORK_DIR}/ -name "Job_${JobName}" )
35      echo $FileList
36      (( NbOcc = ${#FileList[@]} ))
37    else
38      echo "${JobName} not in Catalog."
39      echo "Use option -n / --new to look for it and add it to the catalog."
40      exit
41    fi
42  fi
43
44
45  if [ ${NbOcc} -eq 0 ] ; then
46    echo "${JobName} not found."
47    exit
48  else
49    if [ ${NbOcc} -eq 1 ] ; then
50
51      if ( [ ${UserName} == $( awk -v JobName=${JobName} '$1 ~ JobName {print $2}' ${SimuCatalog} ) ] && [ ${HostName} == $( awk -v JobName=${JobName} '$1 ~ JobName {print $3}' ${SimuCatalog} ) ] ) ; then
52        JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
53        UserName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $2}' ${SimuCatalog} )
54        HostName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $3}' ${SimuCatalog} )
55        SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
56      else
57        echo "${JobName} not in Catalog."
58        exit
59      fi
60    fi
61  fi
62
63}
64
65
66function AffichResult {
67
68  fg_first=".false."
69  fg_last=".false."
70
71  while [ $# -ne 0 ] ; do
72    case ${1} in
73      -f|--first)
74        fg_first=".true."
75        shift 1 ;;
76      -l|--last)
77        fg_last=".true."
78        shift 1 ;;
79      -*)
80        echo "usage: ${0}"
81        echo "       options = -f; -l"
82        exit ;;
83      *)
84        break ;;
85    esac
86  done
87
88
89  # Define colors
90  ColEsc="\033["
91  ColNon="${ColEsc}0m"       # Return to normal
92  ColExp="${ColEsc}1m"       # Blanc - gras
93  ColFat="${ColEsc}1;31m"    # Fatal
94  ColCpl="${ColEsc}1;32m"    # Completed
95  ColAtt="${ColEsc}1;30m"    # Waiting
96  ColDef="${ColEsc}1;34m"    # Default
97  ColRbl="${ColEsc}31m"      # Rebuild
98
99
100 
101  if [ X${fg_first} == X.true. ] ; then
102    Dum=""
103    (( len = 67 - ${#JobName} ))
104    echo "|===========================================================================================================|"
105    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
106    printf "%-${len}s" ${Dum} 
107    printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} "
108    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
109    echo "|                         |             |                         |             |     Pending Rebuilds      |"
110    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
111    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
112    return
113  fi
114
115  if [ X${fg_last} == X.true. ] ; then
116    printf "|===========================================================================================================|\n"
117    date +"${DateFormat}"
118    return
119  fi
120
121  printf "| %-10s - %-10s | " \
122         $DateBegin $DateEnd 
123
124  case $PeriodState in
125    Fatal)
126      Color=${ColFat}
127      break ;;
128    Completed)
129      Color=${ColCpl}
130      break ;;
131    Waiting|OnQueue)
132      Color=${ColAtt}
133      break ;;
134    *)
135      Color=${ColDef}
136      break ;;
137  esac
138  printf "${Color}%-11s${ColNon} | " $PeriodState
139
140  printf "%-10s - %-10s | %11s | " \
141         $PeriodDateBegin $PeriodDateEnd $CumulPeriod
142
143  if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
144    printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
145           $NbRebuild $FirstRebuild $LastRebuild
146  else
147    printf "%3s : %-8s : %-8s |\n" \
148           $NbRebuild $FirstRebuild $LastRebuild
149  fi
150
151  printf "|-----------------------------------------------------------------------------------------------------------|\n"
152  printf "|                                                      Last                                                 |\n"
153  printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
154  printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
155
156  if [ X${Result[1]} == XOK  ] ; then
157    Color=${ColCpl}
158  else
159    Color=${ColFat}
160  fi
161  printf "|     ${Color}%-8s${ColNon}     |" ${LastDate[1]}
162
163  if [ X${Result[2]} == XOK  ] ; then
164    Color=${ColCpl}
165  else
166    Color=${ColFat}
167  fi
168  printf "     ${Color}%-8s${ColNon}     |" ${LastDate[2]}
169
170  if [ X${Result[3]} == XOK  ] ; then
171    Color=${ColCpl}
172  else
173    Color=${ColFat}
174  fi
175  printf "     ${Color}%-8s${ColNon}     |" ${LastDate[3]}
176
177  if [ X${Result[4]} == XOK  ] ; then
178    Color=${ColCpl}
179  else
180    Color=${ColFat}
181  fi
182  printf "     ${Color}%-8s${ColNon}     |" ${LastDate[4]}
183
184  Color=${ColExp}
185  printf "     ${Color}%-4s${ColNon}     |  ${Color}%-12s${ColNon}  |\n" \
186         ${LastDate[5]} ${LastDate[6]}
187}
188
189#======================================================================#
190
191#D- Task type (computing or post-processing)
192TaskType=post-processing
193
194libIGCM=${libIGCM:=::modipsl::/libIGCM}
195
196. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
197     ( ${DEBUG_debug} ) && IGCM_debug_Check
198. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
199     ( ${DEBUG_debug} ) && IGCM_card_Check
200. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
201     ( ${DEBUG_debug} ) && IGCM_date_Check
202#-------
203. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
204
205if ( [ $# -eq 0 ] ) ; then
206  $0 -h
207  exit
208fi
209
210# Default argument values
211UserName=$( whoami )
212HostName=$( hostname | sed -e "s/[0-9].*//" )
213
214fg_color=".true."
215fg_new=".false."
216fg_quiet=".false."
217
218# Get arguments from command line
219while [ $# -ne 0 ] ; do
220  case $1 in
221    -h|--help|-help)
222      echo "usage: $0 [-u UserName] JobName"
223      exit ;;
224    -b|--nocolor)
225      fg_color=".false."
226      shift 1 ;;
227    -n|--new)
228      fg_new=".true."
229      shift 1 ;;
230    -q|--quiet)
231      fg_quiet=".true."
232      shift 1 ;;
233    -p|--config-path)
234      ConfigPath="$2"
235      shift 2 ;;
236    -u|--user)
237      UserName="$2"
238      shift 2 ;;
239    -*)
240      $0 -h
241      exit ;;
242    *)
243      break ;;
244  esac
245done
246
247JobName=$1
248
249echo "UserName = ${UserName}"
250echo "HostName = ${HostName}"
251
252# Define useful directories
253SCRAT_DIR=$( ccc_home -s -u ${UserName} )
254STORE_DIR=$( ccc_home -A -u ${UserName} )
255CWORK_DIR=$( ccc_home -W -u ${UserName} )
256WORK_DIR=$( ccc_home -w -u ${UserName} )
257
258
259# Define the catalog in which the known simulations are stored
260SimuCatalog="$( ccc_home )/.simucatalog.dat"
261if [ ! -s ${SimuCatalog} ] ; then
262  touch ${SimuCatalog}
263fi
264
265# Date format
266DateFormat="%d/%m/%y %R:%S"
267
268# Find SUBMIT_DIR from catalog
269
270SearchCatalog
271
272# Go to SUBMIT_DIR
273if [ ! X${SUBMIT_DIR} == X ] ; then
274
275  echo "Submit:  >${SUBMIT_DIR}<"
276  cd $SUBMIT_DIR
277
278
279# Extract usefull information from run.card and config.card
280
281  RunFile="${SUBMIT_DIR}/run.card"
282  ConfFile="${SUBMIT_DIR}/config.card"
283
284  if [ -s ${RunFile} ] ; then
285    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
286    PeriodState=${run_Configuration_PeriodState}
287  else
288    PeriodState="Waiting"
289  fi
290
291  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
292    NbRun=$( ccc_mstat -f | grep -c ${JobName} )
293
294    if [ ${NbRun} -eq 0 ] ; then
295      PeriodState="Fatal"
296    fi
297  fi
298
299  IGCM_card_DefineVariableFromOption ${ConfFile} UserChoices DateBegin
300  IGCM_card_DefineVariableFromOption ${ConfFile} UserChoices DateEnd
301  IGCM_card_DefineVariableFromOption ${ConfFile} UserChoices TagName
302  IGCM_card_DefineVariableFromOption ${ConfFile} UserChoices ExperimentName
303  IGCM_card_DefineVariableFromOption ${ConfFile} UserChoices SpaceName
304  DateBegin=${config_UserChoices_DateBegin}
305  DateEnd=${config_UserChoices_DateEnd}
306  TagName=${config_UserChoices_TagName}
307  ExperimentName=${config_UserChoices_ExperimentName}
308  SpaceName=${config_UserChoices_SpaceName}
309
310  IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
311  CumulPeriod=${run_Configuration_CumulPeriod}
312
313  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
314    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
315    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
316    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
317    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
318  else
319    PeriodDateBegin="."
320    PeriodDateEnd="."
321  fi
322
323  if [ ! X${ExperimentName} == X ] ; then
324    EXP_DIR=IGCM_OUT/${TagName}/${SpaceName}/${ExperimentName}/${JobName}
325  else
326    EXP_DIR=IGCM_OUT/${TagName}/${JobName}
327  fi
328
329  DATA_DIR=${STORE_DIR}/${EXP_DIR}
330  POST_DIR=${SCRAT_DIR}/${EXP_DIR}/Out
331  CWORK_DIR=${CWORK_DIR}/${EXP_DIR}
332  REBUILD_DIR=${SCRAT_DIR}/REBUILD/${TagName}/${JobName}
333  echo "Data:    >${DATA_DIR}<"
334  echo "Rebuild: >${REBUILD_DIR}<"
335  echo "Post:    >${POST_DIR}<"
336  echo "Work:    >${CWORK_DIR}<"
337
338
339  if [ $PeriodState != "Waiting" ] ; then
340
341    # Check missing rebuilds
342
343    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
344    if [ ${#RebuildList[*]} -gt 0 ] ; then
345      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
346
347      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
348      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
349    else
350      NbRebuild="."
351
352      FirstRebuild="."
353      LastRebuild="."
354    fi
355  else
356    NbRebuild="."
357
358    FirstRebuild="."
359    LastRebuild="."
360  fi
361
362  typeset -A LastDate Result
363
364  # Check last REBUILD
365  FileType=rebuild_*
366  (( num = 1 ))
367  LastDate[$num]=$( ls ${POST_DIR}/${FileType}.*.out | tail -n 1 | awk -F"." '{ print $2 }' )
368
369  dum=$( basename $( grep ^IGCM_sys_PutBuffer_Out ${POST_DIR}/${FileType}.${LastDate[$num]}.out | tail -1 | awk '{ print $4 }' ) | awk -F"_" '{ print $3 }' )
370
371  if [[ ${dum} == ${LastDate[$num]} ]] ; then
372    Result[$num]=OK
373  else
374    Result[$num]=KO
375  fi
376
377  # Check last PACK jobs
378  for FileType in pack_output pack_restart pack_debug ; do
379    (( num = num + 1 ))
380
381    LastDate[$num]=$( ls ${POST_DIR}/${FileType}.*.out | tail -n 1 | awk -F"." '{ print $2 }' )
382
383    dum=$( basename $( grep ^IGCM_sys_Put_Out ${POST_DIR}/${FileType}.${LastDate[$num]}.out | tail -1 | awk '{ print $3 }' ) | awk -F"_" '{ print $3 }' )
384
385    if [[ ${dum} == ${LastDate[$num]} ]] ; then
386      Result[$num]=OK
387    else
388      Result[$num]=KO
389    fi
390  done
391
392
393  # Check last MONITORING jobs
394
395  #set -vx
396
397  FileType=monitoring
398  (( num = num + 1 ))
399
400  LastDate[$num]=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | awk '{ print $NF }' )
401
402  # Check last ATLAS jobs
403
404  FileType=atlas
405  (( num = num + 1 ))
406
407  LastDate[$num]=$( ls ${CWORK_DIR}/ATLAS | tail -1 )
408
409
410  # Time of last write on run.card
411  LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' )
412   
413
414  # Print results
415  AffichResult -f
416  AffichResult
417  AffichResult -l
418
419fi
420
Note: See TracBrowser for help on using the repository browser.