#!/bin/ksh #************************************************************** # Author: Sonia Labetoulle # Contact: sonia.labetoulle__at__ipsl.jussieu.fr # $Revision:: $ Revision of last commit # $Author:: $ Author of last commit # $Date:: $ Date of last commit # IPSL (2012) # This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC # #************************************************************** #======================================================================# # Display a report on how a simulation is faring. function ChangeUsr { echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \ -e "s/${CurrentGrp}/${TargetGrp}/" } function SearchCatalog { typeset num unset SUBMIT_DIR fg_new=false NbOcc=$( awk -v JobName=${JobName} \ 'BEGIN {x=0} $1 ~ JobName {++x} END {print x}' ${SimuCatalog} ) if ( [ ${NbOcc} -eq 0 ] && ( $fg_path ) ) ; then set -A FileList $( ls ${ConfigPath}/Job_* ) if [ X$FileList == X ] ; then NbOcc=0 else NbOcc=${#FileList[@]} fg_new=true fi fi if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then SEARCH_DIR=${WORKDIR} if [ ${TargetUsr} != ${CurrentUsr} ] ; then SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR}) fi echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}" set -A FileList $( find ${SEARCH_DIR}/ \ -path ${SEARCH_DIR}/IGCM_OUT -prune -o \ -name Job_${JobName} -print ) if [ X$FileList == X ] ; then NbOcc=0 else NbOcc=${#FileList[@]} fg_new=true fi fi if [ ${NbOcc} -gt 1 ] ; then echo "More than one job" ind=0 while [ ${ind} -lt ${NbOcc} ] ; do printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]} (( ind = ind + 1 )) done echo "Give your choice number or 'q' to quit : " read Choice if [ X${Choice} == Xq ] ; then exit else fg_new=true FileList=${FileList[${Choice}]} NbOcc=1 fi fi case ${NbOcc} in 0) echo "${JobName} not found." echo "You can try : *) '-s' option to automatically search your \$WORKDIR, " echo " *) '-p' option to provide the config.card path, " echo " *) manually editing your ${SimuCatalog}" exit ;; 1) if ( ${fg_new} ) ; then JobName=${JobName:=$( basename ${FileList} | awk -F"_" '{ print $2 }' )} SUBMIT_DIR=$( dirname ${FileList} ) echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog} elif ( [ ${TargetUsr} == $( awk -v JobName=${JobName} \ '$1 ~ JobName {print $2}' \ ${SimuCatalog} ) ] \ && [ ${HostName} == $( awk -v JobName=${JobName} \ '$1 ~ JobName {print $3}' \ ${SimuCatalog} ) ] ) ; then JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} ) SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} ) else echo "${JobName} not in Catalog." exit fi break ;; *) break ;; esac return } function AffichResult { fg_first=false fg_last=false fg_job=false while [ $# -ne 0 ] ; do case ${1} in -f|--first) fg_first=true shift 1 ;; -l|--last) fg_last=true shift 1 ;; -j|--job) fg_job=true shift 1 ;; -*) echo "usage: ${0}" echo " options = -f; -l" exit ;; *) break ;; esac done # Define colors # ============= ColEsc="\033[" ColNon="${ColEsc}0m" # Return to normal ColExp="${ColEsc}1m" # Blanc - gras ColFat="${ColEsc}1;31m" # Fatal ColCpl="${ColEsc}1;32m" # Completed ColAtt="${ColEsc}1;30m" # Waiting ColDef="${ColEsc}1;34m" # Default ColRbl="${ColEsc}31m" # Rebuild # Print header # ============ if ( ${fg_first} ) ; then Dum="" (( len = 67 - ${#JobName} )) echo "|===========================================================================================================|" printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName} printf "%-${len}s" ${Dum} printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} " echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|" echo "| | | | | Pending Rebuilds |" echo "| Date Begin - DateEnd | PeriodState | Current Period | CumulPeriod | Nb : from : to |" echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|" printf "| %-10s - %-10s | " \ $DateBegin $DateEnd case $PeriodState in Fatal) Color=${ColFat} break ;; Completed) Color=${ColCpl} break ;; Waiting|OnQueue) Color=${ColAtt} break ;; *) Color=${ColDef} break ;; esac printf "${Color}%-11s${ColNon} | " $PeriodState printf "%-10s - %-10s | %11s | " \ $PeriodDateBegin $PeriodDateEnd $CumulPeriod if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \ $NbRebuild $FirstRebuild $LastRebuild else printf "%3s : %-8s : %-8s |\n" \ $NbRebuild $FirstRebuild $LastRebuild fi printf "|-----------------------------------------------------------------------------------------------------------|\n" printf "| Last |\n" printf "| Rebuild | Pack_Output | Pack_Restart | Pack_Debug | Monitoring | Atlas |\n" printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n" return fi # Print Post-processing job status # ================================ if ( ${fg_job} ) ; then printf "|" # Print rebuild and pack jobs # --------------------------- for JobType in ${JobType_list[*]} ; do eval Date=\${${JobType}_Date[${ind}]} eval Status=\${${JobType}_Status[${ind}]} eval Nb=\${${JobType}_Nb[${ind}]} if [ X${Status} == XOK ] ; then Color=${ColCpl} else Color=${ColFat} fi printf " ${Color}%-8s${ColNon} : %3i |" ${Date} ${Nb} done Color=${ColExp} # Print monitoring jobs # --------------------- JobType=monitoring if [ $ind -eq 0 ] ; then eval Date=\${${JobType}_Date} else Date="" fi printf " ${Color}%-4s${ColNon} |" ${Date} # Print atlas jobs # ---------------- JobType=atlas eval Date=\${${JobType}_Date[${ind}]} printf " ${Color}%-12s${ColNon} |" ${Date} printf "\n" return fi # Print footer # ============ if ( ${fg_last} ) ; then printf "|===========================================================================================================|\n" date +"${DateFormat}" return fi } #======================================================================# #D- Task type (computing or post-processing) TaskType=post-processing typeset -i Verbosity=0 CurrentUsr=$( whoami ) CurrentGrp=$( groups $CurrentUsr | awk '{print $3}' ) if ( [ $# -eq 0 ] ) ; then $0 -h exit fi # Arguments # ========= # Default argument values # ----------------------- TargetUsr=${CurrentUsr} HostName=$( hostname | sed -e "s/[0-9].*//" ) fg_color=true fg_search=false fg_quiet=false fg_path=false NbHisto=10 # Get arguments from command line # ------------------------------- while [ $# -ne 0 ] ; do case $1 in -h|--help|-help) echo "usage: $0 [-u user] [-n] [-q] [-j n] job_name" echo " $0 [-u user] [-n] [-q] [-j n] -p config.card_path" echo "" echo "options :" echo " -h : print this help and exit" echo " -u : owner of the job" echo " -q : quiet" echo " -j : print n post-processing jobs (default is 10)" echo " -s : search for a new job in \$WORKDIR and fill in " echo " the catalog before printing information" echo " -p : give the config.card path to a new simulation, " echo " instead of the job name." exit ;; # -b|-nocolor) # fg_color=false # shift 1 ;; -s|-search) fg_search=true shift 1 ;; -q|-quiet) fg_quiet=true shift 1 ;; -p|-config-path) ConfigPath="$2" fg_path=true shift 2 ;; -j|-job-number) NbHisto="$2" shift 2 ;; -u|-user) TargetUsr="$2" shift 2 ;; -*) $0 -h exit ;; *) break ;; esac done if ( ( ! $fg_path ) && [ $# -lt 1 ] ) ; then $0 -h exit fi if ( ( ${fg_path} ) && ( ${fg_search} ) ) ; then echo "You cannot use -s and -p at the same time" exit fi # Load libIGCM library # ==================== libIGCM=${libIGCM:=::modipsl::/libIGCM} . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh ( ${DEBUG_debug} ) && IGCM_debug_Check . ${libIGCM}/libIGCM_card/libIGCM_card.ksh ( ${DEBUG_debug} ) && IGCM_card_Check . ${libIGCM}/libIGCM_date/libIGCM_date.ksh ( ${DEBUG_debug} ) && IGCM_date_Check #------- . ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh . ${libIGCM}/libIGCM_config/libIGCM_config.ksh JobName=$1 if ( ${fg_quiet} ) ; then NbHisto=1 fi echo "TargetUsr = ${TargetUsr}" echo "HostName = ${HostName}" TargetGrp=$( groups $TargetUsr | awk '{print $3}' ) # Define the catalog in which the known simulations are stored SimuCatalog="$( ccc_home )/.simucatalog.dat" if [ ! -s ${SimuCatalog} ] ; then touch ${SimuCatalog} fi # Date format DateFormat="%d/%m/%y %R:%S" # Find SUBMIT_DIR in catalog # ========================== SearchCatalog if [ ! X${SUBMIT_DIR} == X ] ; then echo "Submit: >${SUBMIT_DIR}<" cd $SUBMIT_DIR # Extract usefull information from run.card and config.card # ========================================================= RunFile="${SUBMIT_DIR}/run.card" ConfFile="${SUBMIT_DIR}/config.card" IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card if [ -s ${RunFile} ] ; then IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState PeriodState=${run_Configuration_PeriodState} else PeriodState="Waiting" fi if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then NbRun=$( ccc_mstat -f | grep -c ${JobName} ) if [ ${NbRun} -eq 0 ] ; then PeriodState="Fatal" fi fi DateBegin=${config_UserChoices_DateBegin} DateEnd=${config_UserChoices_DateEnd} TagName=${config_UserChoices_TagName} ExperimentName=${config_UserChoices_ExperimentName} SpaceName=${config_UserChoices_SpaceName} IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod CumulPeriod=${run_Configuration_CumulPeriod} if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd PeriodDateBegin=${run_Configuration_PeriodDateBegin} PeriodDateEnd=${run_Configuration_PeriodDateEnd} else PeriodDateBegin="." PeriodDateEnd="." fi DATA_DIR=${R_SAVE} POST_DIR=${R_BUFR}/Out CWORK_DIR=${R_FIGR} if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then REBUILD_DIR=${R_SAVE}/TMP RebuildJob="rebuild_fromArchive" else REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${config_UserChoices_JobName} RebuildJob="rebuild_fromWorkdir" fi if [ ${TargetUsr} != ${CurrentUsr} ] ; then DATA_DIR=$( ChangeUsr ${DATA_DIR}) POST_DIR=$( ChangeUsr ${POST_DIR} ) CWORK_DIR=$( ChangeUsr ${CWORK_DIR} ) REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} ) fi echo "Data: >${DATA_DIR}<" echo "Rebuild: >${REBUILD_DIR}<" echo "Post: >${POST_DIR}<" echo "Work: >${CWORK_DIR}<" if [ $PeriodState != "Waiting" ] ; then # Check pending rebuilds # ====================== set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort ) if [ ${#RebuildList[*]} -gt 0 ] ; then NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} ) FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ ) LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ ) else NbRebuild="." FirstRebuild="." LastRebuild="." fi else NbRebuild="." FirstRebuild="." LastRebuild="." fi # Check last REBUILD and PACK* jobs # ================================= # Define input parameters # ----------------------- set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug" for JobType in ${JobType_list[*]} ; do typeset name1="${JobType}_String" typeset name2="${JobType}_Field" if [ X${JobType} == X${RebuildJob} ] ; then eval ${name1}=IGCM_sys_PutBuffer_Out eval ${name2}=4 else eval ${name1}=IGCM_sys_Put_Out eval ${name2}=3 fi done # Check jobs # ---------- NbLines=0 for JobType in ${JobType_list[*]} ; do eval String=\${${JobType}_String} eval Field=\${${JobType}_Field} set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} ) if [ ${#FileList[*]} -gt ${NbLines} ] ; then NbLines=${#FileList[*]} fi (( ind = 0 )) for FileName in ${FileList[*]} ; do LastDate=$( basename ${FileName} | awk -F"." '{ print $(NF-1) }' ) dum=$( basename $( grep ^${String} ${POST_DIR}/${JobType}.${LastDate}.out | \ tail -1 | awk -v Field=${Field} '{ print $( Field ) }' ) | \ awk -F"_" '{ print $3 }' ) if [[ X${dum} == X${LastDate} ]] ; then Status=OK else Status=KO fi eval ${JobType}_Date[$ind]=${LastDate} eval ${JobType}_Status[$ind]=${Status} eval ${JobType}_Nb[$ind]=$( grep -c ^${String} ${POST_DIR}/${JobType}.${LastDate}.out ) (( ind = ind + 1 )) done done # Check last MONITORING jobs # ========================== JobType=monitoring if [ -d ${CWORK_DIR}/MONITORING ] ; then LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \ awk '{ print $NF }' ) eval ${JobType}_Date=${LastDate} fi # Check last ATLAS jobs # ===================== JobType=atlas if [ -d ${CWORK_DIR}/ATLAS ] ; then set -A FileList $( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} ) if [ ${#FileList[*]} -gt ${NbLines} ] ; then NbLines=${#FileList[*]} fi (( ind = 0 )) for FileName in ${FileList[*]} ; do eval ${JobType}_Date[$ind]=${FileName} (( ind = ind + 1 )) done fi # Time of last write on run.card # ============================== LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' ) # Print results # ============= AffichResult -f ind=0 while [ $ind -lt $NbLines ] ; do AffichResult -j (( ind = ind + 1 )) done AffichResult -l fi