source: trunk/libIGCM/AA_RunChecker @ 756

Last change on this file since 756 was 756, checked in by labetoulle, 12 years ago

RunChecker? : bugfix in search procedure.
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 14.7 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sonia Labetoulle
5# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2012)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#======================================================================#
15# Display a report on how a simulation is faring.
16
17
18function ChangeUsr {
19
20  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
21                -e "s/${CurrentGrp}/${TargetGrp}/"
22
23}
24
25
26function SearchCatalog {
27
28  typeset num
29  unset SUBMIT_DIR
30
31  fg_new=false
32
33  NbOcc=$( awk -v JobName=${JobName} \
34           'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
35
36  if ( [ ${NbOcc} -eq 0 ] && ( $fg_path ) ) ; then
37    set -A FileList $( ls ${ConfigPath}/Job_* )
38    if [ X$FileList == X ] ; then
39      NbOcc=0
40    else
41      NbOcc=${#FileList[@]}
42      fg_new=true
43    fi
44  fi
45
46  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
47    SEARCH_DIR=${WORKDIR}
48    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
49      SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR})
50    fi
51    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
52
53    set -A FileList $( find ${SEARCH_DIR}/ -name "Job_${JobName}" )
54    if [ X$FileList == X ] ; then
55      NbOcc=0
56    else
57      NbOcc=${#FileList[@]}
58      fg_new=true
59    fi
60  fi
61
62  case ${NbOcc} in
63    0)
64      echo "${JobName} not found."
65      echo "You can try : *) '-s' option to automatically search your \$WORKDIR, "
66      echo "              *) '-p' option to provide the config.card path, "
67      echo "              *)  manually editing your ${SimuCatalog}"
68      exit ;;
69    1)
70      if ( ${fg_new} ) ; then
71        JobName=${JobName:=$( basename ${FileList} | awk -F"_" '{ print $2 }' )}
72        SUBMIT_DIR=$( dirname ${FileList} )
73        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
74        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
75      elif ( [ ${TargetUsr} == $( awk -v JobName=${JobName} \
76                                      '$1 ~ JobName {print $2}' \
77                                      ${SimuCatalog} ) ] \
78          && [ ${HostName}  == $( awk -v JobName=${JobName} \
79                                      '$1 ~ JobName {print $3}' \
80                                      ${SimuCatalog} ) ] ) ; then
81        JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
82        SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
83      else
84        echo "${JobName} not in Catalog."
85        exit
86      fi
87      break ;;
88    *)
89      echo "More than one job"
90      break ;;
91  esac
92
93  return
94
95}
96
97
98function AffichResult {
99
100  fg_first=false
101  fg_last=false
102  fg_job=false
103
104  while [ $# -ne 0 ] ; do
105    case ${1} in
106      -f|--first)
107        fg_first=true
108        shift 1 ;;
109      -l|--last)
110        fg_last=true
111        shift 1 ;;
112      -j|--job)
113        fg_job=true
114        shift 1 ;;
115      -*)
116        echo "usage: ${0}"
117        echo "       options = -f; -l"
118        exit ;;
119      *)
120        break ;;
121    esac
122  done
123
124
125  # Define colors
126  # =============
127  ColEsc="\033["
128  ColNon="${ColEsc}0m"       # Return to normal
129  ColExp="${ColEsc}1m"       # Blanc - gras
130  ColFat="${ColEsc}1;31m"    # Fatal
131  ColCpl="${ColEsc}1;32m"    # Completed
132  ColAtt="${ColEsc}1;30m"    # Waiting
133  ColDef="${ColEsc}1;34m"    # Default
134  ColRbl="${ColEsc}31m"      # Rebuild
135
136
137 
138  # Print header
139  # ============
140  if ( ${fg_first} ) ; then
141    Dum=""
142    (( len = 67 - ${#JobName} ))
143    echo "|===========================================================================================================|"
144    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
145    printf "%-${len}s" ${Dum} 
146    printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} "
147    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
148    echo "|                         |             |                         |             |     Pending Rebuilds      |"
149    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
150    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
151
152    printf "| %-10s - %-10s | " \
153           $DateBegin $DateEnd 
154
155    case $PeriodState in
156      Fatal)
157        Color=${ColFat}
158        break ;;
159      Completed)
160        Color=${ColCpl}
161        break ;;
162      Waiting|OnQueue)
163        Color=${ColAtt}
164        break ;;
165      *)
166        Color=${ColDef}
167        break ;;
168    esac
169    printf "${Color}%-11s${ColNon} | " $PeriodState
170
171    printf "%-10s - %-10s | %11s | " \
172           $PeriodDateBegin $PeriodDateEnd $CumulPeriod
173
174    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
175      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
176             $NbRebuild $FirstRebuild $LastRebuild
177    else
178      printf "%3s : %-8s : %-8s |\n" \
179             $NbRebuild $FirstRebuild $LastRebuild
180    fi
181
182    printf "|-----------------------------------------------------------------------------------------------------------|\n"
183    printf "|                                                      Last                                                 |\n"
184    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
185    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
186
187    return
188  fi
189
190  # Print Post-processing job status
191  # ================================
192  if ( ${fg_job} ) ; then
193    printf "|"
194
195    # Print rebuild and pack jobs
196    # ---------------------------
197    for JobType in ${JobType_list[*]} ; do
198      eval Date=\${${JobType}_Date[${ind}]}
199      eval Status=\${${JobType}_Status[${ind}]}
200
201      if [ X${Status} == XOK  ] ; then
202        Color=${ColCpl}
203      else
204        Color=${ColFat}
205      fi
206      printf "     ${Color}%-8s${ColNon}     |" ${Date}
207    done
208
209    Color=${ColExp}
210
211    # Print monitoring jobs
212    # ---------------------
213    JobType=monitoring
214    if [ $ind -eq 0 ] ; then
215      eval Date=\${${JobType}_Date}
216    else
217      Date=""
218    fi
219    printf "     ${Color}%-4s${ColNon}     |" ${Date}
220
221    # Print atlas jobs
222    # ----------------
223    JobType=atlas
224    eval Date=\${${JobType}_Date[${ind}]}
225    printf "  ${Color}%-12s${ColNon}  |" ${Date}
226
227    printf "\n"
228
229    return
230  fi
231
232  # Print footer
233  # ============
234  if ( ${fg_last} ) ; then
235    printf "|===========================================================================================================|\n"
236    date +"${DateFormat}"
237    return
238  fi
239
240}
241
242#======================================================================#
243
244#D- Task type (computing or post-processing)
245TaskType=post-processing
246typeset -i Verbosity=0
247
248CurrentUsr=$( whoami )
249CurrentGrp=$( groups $CurrentUsr | awk '{print $3}' )
250
251if ( [ $# -eq 0 ] ) ; then
252  $0 -h
253  exit
254fi
255
256# Arguments
257# =========
258# Default argument values
259# -----------------------
260TargetUsr=${CurrentUsr}
261HostName=$( hostname | sed -e "s/[0-9].*//" )
262
263fg_color=true
264fg_search=false
265fg_quiet=false
266fg_path=false
267NbHisto=10
268
269# Get arguments from command line
270# -------------------------------
271while [ $# -ne 0 ] ; do
272  case $1 in
273    -h|--help|-help)
274      echo "usage: $0 [-u user] [-n] [-q] [-j n] job_name"
275      echo "       $0 [-u user] [-n] [-q] [-j n] -p config.card_path"
276      echo ""
277      echo "options :"
278      echo "  -h : print this help and exit"
279      echo "  -u : owner of the job"
280      echo "  -q : quiet"
281      echo "  -j : print n post-processing jobs (default is 10)"
282      echo "  -s : search for a new job in \$WORKDIR and fill in "
283      echo "       the catalog before printing information"
284      echo "  -p : give the config.card path to a new simulation, "
285      echo "       instead of the job name."
286      exit ;;
287#    -b|-nocolor)
288#      fg_color=false
289#      shift 1 ;;
290    -s|-search)
291      fg_search=true
292      shift 1 ;;
293    -q|-quiet)
294      fg_quiet=true
295      shift 1 ;;
296    -p|-config-path)
297      ConfigPath="$2"
298      fg_path=true
299      shift 2 ;;
300    -j|-job-number)
301      NbHisto="$2"
302      shift 2 ;;
303    -u|-user)
304      TargetUsr="$2"
305      shift 2 ;;
306    -*)
307      $0 -h
308      exit ;;
309    *)
310      break ;;
311  esac
312done
313
314if ( ( ! $fg_path ) && [ $# -lt 1 ] ) ; then
315  $0 -h
316  exit
317fi
318
319if ( ( ${fg_path} ) && ( ${fg_search} ) ) ; then
320  echo "You cannot use -s and -p at the same time"
321  exit
322fi
323
324
325# Load libIGCM library
326# ====================
327libIGCM=${libIGCM:=::modipsl::/libIGCM}
328
329. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
330     ( ${DEBUG_debug} ) && IGCM_debug_Check
331. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
332     ( ${DEBUG_debug} ) && IGCM_card_Check
333. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
334     ( ${DEBUG_debug} ) && IGCM_date_Check
335#-------
336. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
337. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
338
339
340JobName=$1
341
342if ( ${fg_quiet} ) ; then
343  NbHisto=1
344fi
345
346echo "TargetUsr = ${TargetUsr}"
347echo "HostName = ${HostName}"
348
349TargetGrp=$( groups $TargetUsr | awk '{print $3}' )
350
351# Define the catalog in which the known simulations are stored
352SimuCatalog="$( ccc_home )/.simucatalog.dat"
353if [ ! -s ${SimuCatalog} ] ; then
354  touch ${SimuCatalog}
355fi
356
357# Date format
358DateFormat="%d/%m/%y %R:%S"
359
360# Find SUBMIT_DIR in catalog
361# ==========================
362SearchCatalog
363
364if [ ! X${SUBMIT_DIR} == X ] ; then
365
366  echo "Submit:  >${SUBMIT_DIR}<"
367  cd $SUBMIT_DIR
368
369
370  # Extract usefull information from run.card and config.card
371  # =========================================================
372
373  RunFile="${SUBMIT_DIR}/run.card"
374  ConfFile="${SUBMIT_DIR}/config.card"
375
376  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
377
378
379  if [ -s ${RunFile} ] ; then
380    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
381    PeriodState=${run_Configuration_PeriodState}
382  else
383    PeriodState="Waiting"
384  fi
385
386  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
387    NbRun=$( ccc_mstat -f | grep -c ${JobName} )
388
389    if [ ${NbRun} -eq 0 ] ; then
390      PeriodState="Fatal"
391    fi
392  fi
393
394  DateBegin=${config_UserChoices_DateBegin}
395  DateEnd=${config_UserChoices_DateEnd}
396  TagName=${config_UserChoices_TagName}
397  ExperimentName=${config_UserChoices_ExperimentName}
398  SpaceName=${config_UserChoices_SpaceName}
399
400  IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
401  CumulPeriod=${run_Configuration_CumulPeriod}
402
403  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
404    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
405    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
406    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
407    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
408  else
409    PeriodDateBegin="."
410    PeriodDateEnd="."
411  fi
412
413  DATA_DIR=${R_SAVE}
414  POST_DIR=${R_BUFR}/Out
415  CWORK_DIR=${R_FIGR}
416  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
417    REBUILD_DIR=${R_SAVE}/TMP
418    RebuildJob="rebuild_fromArchive"
419  else
420    REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
421    RebuildJob="rebuild_fromWorkdir"
422  fi
423
424  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
425    DATA_DIR=$( ChangeUsr ${DATA_DIR})
426    POST_DIR=$( ChangeUsr ${POST_DIR} )
427    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
428    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
429  fi
430
431  echo "Data:    >${DATA_DIR}<"
432  echo "Rebuild: >${REBUILD_DIR}<"
433  echo "Post:    >${POST_DIR}<"
434  echo "Work:    >${CWORK_DIR}<"
435
436  if [ $PeriodState != "Waiting" ] ; then
437
438    # Check pending rebuilds
439    # ======================
440
441    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
442    if [ ${#RebuildList[*]} -gt 0 ] ; then
443      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
444
445      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
446      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
447    else
448      NbRebuild="."
449
450      FirstRebuild="."
451      LastRebuild="."
452    fi
453  else
454    NbRebuild="."
455
456    FirstRebuild="."
457    LastRebuild="."
458  fi
459
460
461  # Check last REBUILD and PACK* jobs
462  # =================================
463  # Define input parameters
464  # -----------------------
465  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
466
467  for JobType in ${JobType_list[*]} ; do
468    typeset    name1="${JobType}_String"
469    typeset    name2="${JobType}_Field"
470    if [ X${JobType} == X${RebuildJob} ] ; then
471      eval ${name1}=IGCM_sys_PutBuffer_Out
472      eval ${name2}=4
473    else
474      eval ${name1}=IGCM_sys_Put_Out
475      eval ${name2}=3
476    fi
477  done
478
479  # Check jobs
480  # ----------
481  NbLines=0
482  for JobType in ${JobType_list[*]} ; do
483    eval String=\${${JobType}_String}
484    eval Field=\${${JobType}_Field}
485
486    set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
487
488    if [ ${#FileList[*]} -gt ${NbLines} ] ; then
489      NbLines=${#FileList[*]}
490    fi
491
492    (( ind = 0 ))
493    for FileName in ${FileList[*]} ; do
494      LastDate=$( basename ${FileName} | awk -F"." '{ print $(NF-1) }' )
495
496      dum=$( basename $( grep ^${String} ${POST_DIR}/${JobType}.${LastDate}.out | \
497               tail -1 | awk -v Field=${Field} '{ print $( Field ) }' ) | \
498               awk -F"_" '{ print $3 }' )
499
500      if [[ X${dum} == X${LastDate} ]] ; then
501        Status=OK
502      else
503        Status=KO
504      fi
505
506      eval ${JobType}_Date[$ind]=${LastDate}
507      eval ${JobType}_Status[$ind]=${Status}
508     
509      (( ind = ind + 1 ))
510    done
511  done
512
513  # Check last MONITORING jobs
514  # ==========================
515  JobType=monitoring
516  if [ -d ${CWORK_DIR}/MONITORING ] ; then
517    LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \
518                    awk '{ print $NF }' )
519    eval ${JobType}_Date=${LastDate}
520  fi
521
522  # Check last ATLAS jobs
523  # =====================
524  JobType=atlas
525  if [ -d ${CWORK_DIR}/ATLAS ] ; then
526    set -A FileList $( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )
527
528    if [ ${#FileList[*]} -gt ${NbLines}=0 ] ; then
529      NbLines=${#FileList[*]}
530    fi
531
532    (( ind = 0 ))
533    for FileName in ${FileList[*]} ; do
534      eval ${JobType}_Date[$ind]=${FileName}
535      (( ind = ind + 1 ))
536    done
537  fi
538
539
540  # Time of last write on run.card
541  # ==============================
542  LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' )
543   
544
545  # Print results
546  # =============
547  AffichResult -f
548  ind=0
549  while [ $ind -lt $NbLines ] ; do
550    AffichResult -j
551    (( ind = ind + 1 ))
552  done
553  AffichResult -l
554
555fi
556
Note: See TracBrowser for help on using the repository browser.