source: trunk/libIGCM/AA_RunChecker @ 756

#!/bin/ksh

#**************************************************************
# Author: Sonia Labetoulle
# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2012)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#======================================================================#
# Display a report on how a simulation is faring.


function ChangeUsr {

  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
                -e "s/${CurrentGrp}/${TargetGrp}/"

}


function SearchCatalog {

  typeset num
  unset SUBMIT_DIR

  fg_new=false

  NbOcc=$( awk -v JobName=${JobName} \
           'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )

  if ( [ ${NbOcc} -eq 0 ] && ( $fg_path ) ) ; then
    set -A FileList $( ls ${ConfigPath}/Job_* )
    if [ X$FileList == X ] ; then
      NbOcc=0
    else
      NbOcc=${#FileList[@]}
      fg_new=true
    fi
  fi

  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
    SEARCH_DIR=${WORKDIR}
    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
      SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR})
    fi
    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"

    set -A FileList $( find ${SEARCH_DIR}/ -name "Job_${JobName}" )
    if [ X$FileList == X ] ; then
      NbOcc=0
    else
      NbOcc=${#FileList[@]}
      fg_new=true
    fi
  fi

  case ${NbOcc} in
    0)
      echo "${JobName} not found."
      echo "You can try : *) '-s' option to automatically search your \$WORKDIR, "
      echo "              *) '-p' option to provide the config.card path, "
      echo "              *)  manually editing your ${SimuCatalog}"
      exit ;;
    1)
      if ( ${fg_new} ) ; then
        JobName=${JobName:=$( basename ${FileList} | awk -F"_" '{ print $2 }' )}
        SUBMIT_DIR=$( dirname ${FileList} )
        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
      elif ( [ ${TargetUsr} == $( awk -v JobName=${JobName} \
                                      '$1 ~ JobName {print $2}' \
                                      ${SimuCatalog} ) ] \
          && [ ${HostName}  == $( awk -v JobName=${JobName} \
                                      '$1 ~ JobName {print $3}' \
                                      ${SimuCatalog} ) ] ) ; then
        JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
        SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
      else
        echo "${JobName} not in Catalog."
        exit
      fi
      break ;;
    *)
      echo "More than one job"
      break ;;
  esac

  return

}


function AffichResult {

  fg_first=false
  fg_last=false
  fg_job=false

  while [ $# -ne 0 ] ; do
    case ${1} in
      -f|--first)
        fg_first=true
        shift 1 ;;
      -l|--last)
        fg_last=true
        shift 1 ;;
      -j|--job)
        fg_job=true
        shift 1 ;;
      -*)
        echo "usage: ${0}"
        echo "       options = -f; -l"
        exit ;;
      *)
        break ;;
    esac
  done


  # Define colors
  # =============
  ColEsc="\033["
  ColNon="${ColEsc}0m"       # Return to normal
  ColExp="${ColEsc}1m"       # Blanc - gras
  ColFat="${ColEsc}1;31m"    # Fatal
  ColCpl="${ColEsc}1;32m"    # Completed
  ColAtt="${ColEsc}1;30m"    # Waiting
  ColDef="${ColEsc}1;34m"    # Default
  ColRbl="${ColEsc}31m"      # Rebuild


  
  #Â Print header
  # ============
  if ( ${fg_first} ) ; then
    Dum=""
    (( len = 67 - ${#JobName} ))
    echo "|===========================================================================================================|"
    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
    printf "%-${len}s" ${Dum} 
    printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} "
    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
    echo "|                         |             |                         |             |     Pending Rebuilds      |"
    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"

    printf "| %-10s - %-10s | " \
           $DateBegin $DateEnd 

    case $PeriodState in
      Fatal)
        Color=${ColFat}
        break ;;
      Completed)
        Color=${ColCpl}
        break ;;
      Waiting|OnQueue)
        Color=${ColAtt}
        break ;;
      *)
        Color=${ColDef}
        break ;;
    esac
    printf "${Color}%-11s${ColNon} | " $PeriodState

    printf "%-10s - %-10s | %11s | " \
           $PeriodDateBegin $PeriodDateEnd $CumulPeriod

    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
             $NbRebuild $FirstRebuild $LastRebuild
    else
      printf "%3s : %-8s : %-8s |\n" \
             $NbRebuild $FirstRebuild $LastRebuild
    fi

    printf "|-----------------------------------------------------------------------------------------------------------|\n"
    printf "|                                                      Last                                                 |\n"
    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"

    return
  fi

  # Print Post-processing job status
  # ================================
  if ( ${fg_job} ) ; then
    printf "|"

    #Â Print rebuild and pack jobs
    #Â ---------------------------
    for JobType in ${JobType_list[*]} ; do 
      eval Date=\${${JobType}_Date[${ind}]}
      eval Status=\${${JobType}_Status[${ind}]}

      if [ X${Status} == XOK  ] ; then
        Color=${ColCpl}
      else
        Color=${ColFat}
      fi
      printf "     ${Color}%-8s${ColNon}     |" ${Date}
    done

    Color=${ColExp}

    #Â Print monitoring jobs
    #Â ---------------------
    JobType=monitoring
    if [ $ind -eq 0 ] ; then
      eval Date=\${${JobType}_Date}
    else
      Date=""
    fi
    printf "     ${Color}%-4s${ColNon}     |" ${Date}

    #Â Print atlas jobs
    #Â ----------------
    JobType=atlas
    eval Date=\${${JobType}_Date[${ind}]}
    printf "  ${Color}%-12s${ColNon}  |" ${Date}

    printf "\n"

    return
  fi

  #Â Print footer
  #Â ============
  if ( ${fg_last} ) ; then
    printf "|===========================================================================================================|\n"
    date +"${DateFormat}"
    return
  fi

}

#======================================================================#

#D- Task type (computing or post-processing)
TaskType=post-processing
typeset -i Verbosity=0

CurrentUsr=$( whoami )
CurrentGrp=$( groups $CurrentUsr | awk '{print $3}' )

if ( [ $# -eq 0 ] ) ; then
  $0 -h
  exit
fi

# Arguments
# =========
# Default argument values
#Â -----------------------
TargetUsr=${CurrentUsr}
HostName=$( hostname | sed -e "s/[0-9].*//" )

fg_color=true
fg_search=false
fg_quiet=false
fg_path=false
NbHisto=10

# Get arguments from command line
# -------------------------------
while [ $# -ne 0 ] ; do
  case $1 in
    -h|--help|-help)
      echo "usage: $0 [-u user] [-n] [-q] [-j n] job_name"
      echo "       $0 [-u user] [-n] [-q] [-j n] -p config.card_path"
      echo ""
      echo "options :"
      echo "  -h : print this help and exit"
      echo "  -u : owner of the job"
      echo "  -q : quiet"
      echo "  -j : print n post-processing jobs (default is 10)"
      echo "  -s : search for a new job in \$WORKDIR and fill in "
      echo "       the catalog before printing information"
      echo "  -p : give the config.card path to a new simulation, "
      echo "       instead of the job name."
      exit ;;
#    -b|-nocolor)
#      fg_color=false
#      shift 1 ;;
    -s|-search)
      fg_search=true
      shift 1 ;;
    -q|-quiet)
      fg_quiet=true
      shift 1 ;;
    -p|-config-path)
      ConfigPath="$2"
      fg_path=true
      shift 2 ;;
    -j|-job-number)
      NbHisto="$2"
      shift 2 ;;
    -u|-user)
      TargetUsr="$2"
      shift 2 ;;
    -*)
      $0 -h
      exit ;;
    *)
      break ;;
  esac
done

if ( ( ! $fg_path ) && [ $# -lt 1 ] ) ; then
  $0 -h
  exit
fi

if ( ( ${fg_path} ) && ( ${fg_search} ) ) ; then
  echo "You cannot use -s and -p at the same time"
  exit
fi


# Load libIGCM library
# ====================
libIGCM=${libIGCM:=::modipsl::/libIGCM}

. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
     ( ${DEBUG_debug} ) && IGCM_debug_Check
. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
     ( ${DEBUG_debug} ) && IGCM_card_Check
. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
     ( ${DEBUG_debug} ) && IGCM_date_Check
#-------
. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
. ${libIGCM}/libIGCM_config/libIGCM_config.ksh


JobName=$1

if ( ${fg_quiet} ) ; then
  NbHisto=1
fi

echo "TargetUsr = ${TargetUsr}"
echo "HostName = ${HostName}"

TargetGrp=$( groups $TargetUsr | awk '{print $3}' )

# Define the catalog in which the known simulations are stored
SimuCatalog="$( ccc_home )/.simucatalog.dat"
if [ ! -s ${SimuCatalog} ] ; then
  touch ${SimuCatalog}
fi

# Date format
DateFormat="%d/%m/%y %R:%S"

# Find SUBMIT_DIR in catalog
# ==========================
SearchCatalog

if [ ! X${SUBMIT_DIR} == X ] ; then

  echo "Submit:  >${SUBMIT_DIR}<"
  cd $SUBMIT_DIR


  # Extract usefull information from run.card and config.card
  # =========================================================

  RunFile="${SUBMIT_DIR}/run.card"
  ConfFile="${SUBMIT_DIR}/config.card"

  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card


  if [ -s ${RunFile} ] ; then
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
    PeriodState=${run_Configuration_PeriodState}
  else
    PeriodState="Waiting"
  fi

  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
    NbRun=$( ccc_mstat -f | grep -c ${JobName} )

    if [ ${NbRun} -eq 0 ] ; then
      PeriodState="Fatal"
    fi
  fi

  DateBegin=${config_UserChoices_DateBegin}
  DateEnd=${config_UserChoices_DateEnd}
  TagName=${config_UserChoices_TagName}
  ExperimentName=${config_UserChoices_ExperimentName}
  SpaceName=${config_UserChoices_SpaceName}

  IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
  CumulPeriod=${run_Configuration_CumulPeriod}

  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
  else
    PeriodDateBegin="."
    PeriodDateEnd="."
  fi

  DATA_DIR=${R_SAVE}
  POST_DIR=${R_BUFR}/Out
  CWORK_DIR=${R_FIGR}
  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
    REBUILD_DIR=${R_SAVE}/TMP
    RebuildJob="rebuild_fromArchive"
  else
    REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
    RebuildJob="rebuild_fromWorkdir"
  fi

  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
    DATA_DIR=$( ChangeUsr ${DATA_DIR})
    POST_DIR=$( ChangeUsr ${POST_DIR} )
    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
  fi

  echo "Data:    >${DATA_DIR}<"
  echo "Rebuild: >${REBUILD_DIR}<"
  echo "Post:    >${POST_DIR}<"
  echo "Work:    >${CWORK_DIR}<"

  if [ $PeriodState != "Waiting" ] ; then

    # Check pending rebuilds
    # ======================

    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
    if [ ${#RebuildList[*]} -gt 0 ] ; then
      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )

      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
    else
      NbRebuild="."

      FirstRebuild="."
      LastRebuild="."
    fi
  else
    NbRebuild="."

    FirstRebuild="."
    LastRebuild="."
  fi


  # Check last REBUILD and PACK* jobs
  # =================================
  # Define input parameters
  # -----------------------
  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"

  for JobType in ${JobType_list[*]} ; do 
    typeset    name1="${JobType}_String"
    typeset    name2="${JobType}_Field"
    if [ X${JobType} == X${RebuildJob} ] ; then
      eval ${name1}=IGCM_sys_PutBuffer_Out
      eval ${name2}=4
    else
      eval ${name1}=IGCM_sys_Put_Out
      eval ${name2}=3
    fi
  done

  #Â Check jobs
  # ----------
  NbLines=0
  for JobType in ${JobType_list[*]} ; do 
    eval String=\${${JobType}_String}
    eval Field=\${${JobType}_Field}

    set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )

    if [ ${#FileList[*]} -gt ${NbLines} ] ; then
      NbLines=${#FileList[*]}
    fi

    (( ind = 0 ))
    for FileName in ${FileList[*]} ; do
      LastDate=$( basename ${FileName} | awk -F"." '{ print $(NF-1) }' )

      dum=$( basename $( grep ^${String} ${POST_DIR}/${JobType}.${LastDate}.out | \
               tail -1 | awk -v Field=${Field} '{ print $( Field ) }' ) | \
               awk -F"_" '{ print $3 }' )

      if [[ X${dum} == X${LastDate} ]] ; then
        Status=OK
      else
        Status=KO
      fi

      eval ${JobType}_Date[$ind]=${LastDate}
      eval ${JobType}_Status[$ind]=${Status}
      
      (( ind = ind + 1 ))
    done
  done

  # Check last MONITORING jobs
  # ==========================
  JobType=monitoring
  if [ -d ${CWORK_DIR}/MONITORING ] ; then
    LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \
                    awk '{ print $NF }' )
    eval ${JobType}_Date=${LastDate}
  fi

  # Check last ATLAS jobs
  # =====================
  JobType=atlas
  if [ -d ${CWORK_DIR}/ATLAS ] ; then
    set -A FileList $( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )

    if [ ${#FileList[*]} -gt ${NbLines}=0 ] ; then
      NbLines=${#FileList[*]}
    fi

    (( ind = 0 ))
    for FileName in ${FileList[*]} ; do
      eval ${JobType}_Date[$ind]=${FileName}
      (( ind = ind + 1 ))
    done
  fi


  # Time of last write on run.card
  # ==============================
  LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' )
    

  # Print results
  # =============
  AffichResult -f
  ind=0
  while [ $ind -lt $NbLines ] ; do
    AffichResult -j
    (( ind = ind + 1 ))
  done
  AffichResult -l

fi