source: trunk/libIGCM/ins_job @ 1274

Last change on this file since 1274 was 1274, checked in by mafoipsl, 9 years ago

For curie :

  • Add a new option in AA_post_jobs header on curie : "-c ::default_core::" with the number of core reserved.
  • Add 3 options to ins_job :
    • -p projet
    • -q standard/xlarge : with xlarge as default value. except for gencmip6 project
    • -c number of cores (default value is : 4 on standard node or 8 on xlarge node)
  • For batch usage, it's possible to set these options on ins_job command line. See Usage.
  • By default, 3 questions are asked interactively to the user.

With 4 or 8 cores per job we want to limit the number of simultaneous running jobs on one node
and eventually limit SCRATCH congestion.
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 14.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Jacques Belier
5# Contact:
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13#---------------------------------------------------------------------
14#- Installation of jobs according to environment
15#---------------------------------------------------------------------
16function ins_job_Usage
17{
18print - "
19ins_job installs the jobs in the directories
20which contain a file config.card
21
22ins_job must be launched on the host
23on which the job will be submitted
24
25Usage :
26  ${b_n} [-h] [-v] [-e]
27  or on curie/TGCC :
28  ${b_n} [-h] [-v] [-e] [-p project] [-q type_of_node] [-c number of cores]
29
30Options :
31  -h                  : help
32  -v                  : verbose mode
33  -e                  : turn on ensemble mode (hindcast/forecast or date restart)
34on curie only :
35  -p project          : add default project on curie
36  -q type_of_node     : add default type of nodes for postprocessing on curie standard/xlarge
37  -c number of cores  : add default number of cores for postprocessing on curie 1-16 or 1-128
38"
39}
40function ins_job_Warning
41{
42   [[ ${x_v} = 'verbose' ]] && print - "\n############### WARNING ###############";
43   [[ ${x_v} = 'verbose' ]] && print - "File ${n_f} already exists\nin directory ${j}";
44   [[ ${x_v} = 'verbose' ]] && print - "You must delete this file to update !";
45}
46
47function ins_job_Check_JobName
48{
49  verif=${JobName##[a-zA-Z]*(?([.\-])[a-zA-Z0-9])}
50
51  if [ ${#verif} -ne 0 ] ; then
52    echo "################ ERROR ################"
53    echo "${JobName} is invalid."
54    echo "- JobName can only contain alphanumeric characters, \".\" and \"-\""
55    echo "- JobName must start with a letter"
56
57    ((NbErr=NbErr+1))
58
59    Status=1
60  else
61    Status=0
62  fi
63
64  return ${Status}
65}
66
67#-
68#     dirname     and      basename
69#-
70d_n=$(dirname ${0}); b_n=$(basename ${0});
71#-
72# Retrieving and validation of the options
73#-
74x_v='silencious';
75x_e=false;
76x_p=false;
77x_q=false;
78x_c=false;
79while getopts :hvec:p:q: V ; do
80  case $V in
81  (h)  ins_job_Usage; exit 0;;
82  (v)  x_v='verbose';;
83  (e)  x_e=true;;
84  (p)  x_p=true
85       ProjectID=${OPTARG} ;;
86  (q)  x_q=true
87       ProjectNode=${OPTARG} ;;
88  (c)  x_c=true
89       ProjectCore=${OPTARG} ;;
90  (:)  echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
91       exit 2;;
92  (\?) echo ${b_n}" : -"${OPTARG}" option : not supported" 1>&2;
93       exit 2;;
94  esac
95done
96[ ${x_v} = 'silencious' ] && export DEBUG_sys=false
97shift $(($OPTIND-1));
98#-
99# Define working files
100#-
101F_MOD=$(cd ${d_n}'/..';/bin/pwd;)
102# [[ ${F_MOD##*/} != 'modipsl' ]] && \
103#  { print - "directory 'modipsl' unreachable"; exit 3; }
104W_W=${d_n}'/../libIGCM'
105[[ ! -d ${W_W} ]] && { print - "${W_W} unreachable"; exit 3; }
106libIGCM=$(cd ${W_W};/bin/pwd;)
107F_JOB=${libIGCM}'/AA_job';
108[[ ! -f ${F_JOB} ]] && { print - "${F_JOB} unreachable"; exit 3; }
109F_RCI=${libIGCM}'/run.card.init';
110[[ ! -f ${F_RCI} ]] && { print - "${F_RCI} unreachable"; exit 3; }
111#-
112# Accessing to functions (without stack)
113#-
114# No verbosity (0, 1, 2, 3)
115Verbosity=0
116# No de debug
117DEBUG_debug=false
118# Dont move libIGCM
119MirrorlibIGCM=false
120# Behave like computing job
121TaskType=computing
122# Source libIGCM
123. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
124. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
125. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
126. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
127. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
128if [ $x_e = 'true' ] ; then
129  . ${libIGCM}/libIGCM_ensemble/libIGCM_ensemble.ksh
130fi
131
132#-
133[[ ${x_v} = 'verbose' ]] && \
134 {
135  print - "";
136  print - '--- Host        : '${SYSTEM};
137  print - '--- modipsl     : '${F_MOD};
138  print - '--- libIGCM     : '${libIGCM};
139  print - '--- basic job   : '${F_JOB};
140  print - '--- basic card  : '${F_RCI};
141 }
142#-
143[[ ${x_v} = 'verbose' ]] && print - "\nInstallation of jobs for '${SYSTEM}'";
144#-
145
146NbErr=0
147
148#-
149# Define Project parameters to set up jobs header for Curie (TGCC)
150# on curie define ProjectID, ProjectNode and ProjectCore : option or answer
151#-
152
153if [ X"${SYSTEM}" == "Xcurie" ] ; then
154  if ( ! ${x_p} ) ; then
155    ProjectID=$( ccc_myproject | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | head -n 1 )
156    ProjectID=${ProjectID:="gencmip6"}
157    answer=""
158    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | sort -u ) ) :"
159    read answer
160
161    if [ "X${answer}" != "X" ] ; then
162      ProjectID=${answer}
163    fi
164  fi # if ( ! ${x_p} )
165  echo  ProjectID is ${ProjectID}
166
167  if ( ! ${x_q} ) ; then
168    #- ProjectID is known (option or answer), set ProjectNode
169    [ "${ProjectID}" = "gencmip6" ] && ProjectNode="standard" || ProjectNode="xlarge"
170    answerOK=false
171
172    while ( ! ${answerOK} ) ; do
173      answer=""
174      print - "Hit Enter or give TYPE OF NODE required for post-processing (default is \"${ProjectNode}\"), possible types of nodes are \"standard\" or \"xlarge\" : "
175      read answer
176      [ "X${answer}" == "X" ] || [ "X${answer}" == "Xstandard" ] || [ "X${answer}" == "Xxlarge" ] && answerOK=true
177    done
178
179    if [ "X${answer}" != "X" ] ; then
180      ProjectNode=${answer}
181    fi
182
183  fi # if ( ! ${x_q} )
184  echo ProjectNode is ${ProjectNode}
185
186  #- ProjectNode is known (option or answer) set ProjectCoreMax
187  [ "${ProjectNode}" = "xlarge" ] && ProjectCoreMax="128" || ProjectCoreMax="16"
188
189  if ( ! ${x_c} ) ; then
190    #- ProjectNode is known (option or answer), set ProjectCore
191    [ "${ProjectNode}" = "xlarge" ] && ProjectCore="8" || ProjectCore="4"
192    answerOK=false
193
194    while ( ! ${answerOK} ) ; do
195      answer=""
196      print - "Hit Enter or give NUMBER OF CORES required for post-processing (default is \"${ProjectCore}\"), possible numbers of cores are \"1\" to \"${ProjectCoreMax}\" : "
197      read answer
198      [ "X${answer}" == "X" ] || [ ${answer} -ge 1 -a ${answer} -le ${ProjectCoreMax} ] && answerOK=true
199    done
200
201    if [ "X${answer}" != "X" ] ; then
202      ProjectCore=${answer}
203    fi
204  fi # if ( ! ${x_c} )
205  echo ProjectCore is ${ProjectCore}
206#- ProjectCore is set (option or answer)
207# let check minimum/maximum value 1/${ProjectCoreMax}
208
209else
210  echo No option for Header regarding ProjectID, ProjectNode and ProjectCore because we are not on curie
211fi # if [ X"${SYSTEM}" == "Xcurie" ]
212
213#-
214# Define the pattern string to substitute
215#-
216W_P='#-Q- '; W_W=${W_P}${SYSTEM};
217#-
218# Extract list of 'config.card' files
219# and create jobs with AA_job
220#-
221F_CFG='config.card';
222F_CFG_ENS='ensemble.card';
223SUBMIT_DIR_ENS=$( pwd )
224for i in $( pwd )/config.card
225do
226  if [ ! -f $i ] ; then
227    echo ""
228    echo "################## WARNING ##################"
229    echo "No config.card available in current directory"
230    echo ""
231    continue
232  fi
233
234
235  j=$(cd ${i%/*};/bin/pwd;)
236  n_f=${F_RCI##*/};
237
238  if [ ! X$( echo ${j} | grep EXPERIMENTS ) = X ] ; then
239    # Do not treat config.card if it is in sub-directory of EXPERIMENTS
240    # Continue to next config.card
241    continue
242  else
243    [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
244  fi
245
246  # Find out if new structure and set .resol filename
247  if [ -d ${j}/EXPERIMENTS ] && [ -d ${j}/GENERAL ] ; then
248    # New Structure
249    [[ ${x_v} = 'verbose' ]] && echo "This is new configuration structure"
250    new_struct=yes
251    resolfile=$j/.resol
252  else
253    # Old Structure
254    new_struct=no
255    resolfile=$j/../.resol
256  fi
257
258  # Get all variables declared in section UserChoices in config.card
259  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} UserChoices
260  # Set default values
261  config_UserChoices_ExpType=""
262  RESOL_ATM_3D=this_is_a_test_string
263  RESOL=this_is_another_test_string
264  typeset option
265  for option in ${config_UserChoices[*]} ; do
266    IGCM_card_DefineVariableFromOption ${j}'/'${F_CFG} UserChoices ${option}
267  done
268
269  # Find the JobName : JobName might contain the variable RESOL_ATM_3D that will be replaced by what is in .resol file
270  if [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL_ATM_3D} ) = X ] ; then
271    TRUERESOL=$( tail -1 $resolfile | awk "-F=" '{print $2}' )
272    echo TRUERESOL = $TRUERESOL
273    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL_ATM_3D}/${TRUERESOL}/" )
274    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
275  elif [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL} ) = X ] ; then
276    TRUERESOL=$( head -1 $resolfile  )
277    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL}/${TRUERESOL}/" )
278    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
279  else
280    JobName=${config_UserChoices_JobName}
281  fi
282
283  # Check JobName validity : only alphanumerical characters, "-" and "." are authorized
284  ins_job_Check_JobName
285  RetCode=$?
286  [[ $RetCode -gt 0 ]] && continue
287
288  [[ ${x_v} = 'verbose' ]] && echo "JobName=${JobName}"
289
290  # Add specific treatment for new type of directory structure
291  if [ ${new_struct} == yes ] ; then
292
293    if [ "X${config_UserChoices_ExpType}" = X ] ; then
294      echo "\nERROR in ${j}/config.card"
295      echo "ins_job stops here"
296      echo "=> The variable ExpType must be added in config.card in section UserChoices"
297      echo "=> ExpType gives the directory for the .card configuration files for the wanted experiement. For exemple ExpType=IPSLCM5/historical"
298      exit 4
299    else
300      [[ ${x_v} = 'verbose' ]] && echo "ExpType= ${config_UserChoices_ExpType}"
301    fi
302
303    if [ -d ${j}/${JobName} ] ; then
304      echo "Directory ${j}/${JobName} exist already, continue next config.card"
305      continue
306    fi
307    echo "=> Submit directory ${JobName} will be created with cards from EXPERIMENTS/${config_UserChoices_ExpType}"
308    cp -r ${j}/EXPERIMENTS/${config_UserChoices_ExpType} ${j}/${JobName}
309    cp -r ${j}/GENERAL/* ${j}/${JobName}/.
310    cp -f ${j}/${F_CFG}  ${j}/${JobName}/.
311    if [ -f ${F_CFG_ENS} ] ; then
312      cp -f ${j}/${F_CFG_ENS}  ${j}/${JobName}/.
313      SUBMIT_DIR_ENS=${j}/${JobName}
314    fi
315    rm -f ${j}/${F_CFG}
316    rm -f ${j}/${F_CFG_ENS}
317    rm -f ${j}/${F_CFG}.bak
318    j=${j}/${JobName}
319    [[ ${x_v} = 'verbose' ]] && echo new j=$j
320  fi
321  # end specific treatment for new type directory structure
322
323  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; } || \
324   {
325    [[ ${x_v} = 'verbose' ]] && print - "\nCopying file ${F_RCI}\nin directory ${j}";
326    \cp ${F_RCI} ${j};
327   }
328
329  #==================================
330  # Read ListOfComponents section:
331  #echo
332  #IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"
333
334  IGCM_card_DefineArrayFromSection  ${j}'/'${F_CFG} ListOfComponents
335  for comp in ${config_ListOfComponents[*]} ; do
336    IGCM_card_DefineArrayFromOption  ${j}'/'${F_CFG} ListOfComponents ${comp}
337  done
338  #IGCM_debug_Print 3 ${config_ListOfComponents[*]}
339
340  #==================================
341  # Read Executable section:
342  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} Executable
343
344  # Define the execution context (MPMD, SPMD, MPI/OMP ...)
345  IGCM_config_ConfigureExecution ${j}'/'${F_CFG}
346
347  # coreNumber    : TOTAL NUMBER OF CORES
348  # mpiTasks      : TOTAL NUMBER OF MPI TASKS
349  # openMPthreads : NUMBER OF OpenMP THREADS
350
351  # File name for Job
352  n_f='Job_'${JobName};
353  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
354  [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
355  sed -e "/^${W_W} */ s///" \
356      -e "/^${W_P}/d"       \
357      -e "s%::modipsl::%${F_MOD}%" \
358      -e "s/::Jobname::/${JobName}/" \
359      -e "s/::default_project::/${ProjectID}/" \
360      ${F_JOB} > ${j}'/'${n_f}
361  chmod u+x ${j}'/'${n_f}
362
363  # update Headers so that ressources description are accurate (MPMD/SPMD/...)
364  IGCM_sys_updateHeaders ${j}'/'${n_f}
365done
366
367#-
368# Extract list of AA_* files in libIGCM
369# and create jobs (for all except AA_job)
370#-
371for i in $(find ${libIGCM} -maxdepth 1 -name "AA_*" -print)
372do
373  i_f=${i##*/};
374  [[ ${i_f} = 'AA_job' ]] && { continue; }
375  j=${i%/*}; n_f=${i_f#AA_}'.job';
376  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
377  [[ ${x_v} = 'verbose' ]] && print - "\nIn directory ${j}\n${i_f} -> ${n_f}"
378  sed -e "/^${W_W} */ s///" \
379      -e "s%::modipsl::%${F_MOD}%" \
380      -e "/^${W_P}/d"       \
381      -e "s/::default_node::/${ProjectNode}/" \
382      -e "s/::default_core::/${ProjectCore}/" \
383      -e "s/::default_project::/${ProjectID}/" \
384      ${i} > ${j}'/'${n_f}
385  chmod u+x ${j}'/'${n_f}
386done
387#-
388# set default_project in libIGCM_sys_curie.ksh too.
389#-
390if [ X"${SYSTEM}" == "Xcurie" ] ; then
391  i=${libIGCM}/libIGCM_sys/libIGCM_sys_curie.ksh
392  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
393fi
394#-
395# Limited to hindcast/forecast and date restart Ensemble for the time being
396if [ ${x_e} = 'true' ] ; then
397  #.. Read input data from ensemble.card ..
398  SUBMIT_DIR=${SUBMIT_DIR_ENS}
399  RUN_DIR="${WORKDIR}/ENSEMBLE"
400  #
401  # Copy initial things around and define variables (hindcast/forecast case)
402  IGCM_sys_Cd ${SUBMIT_DIR}
403  IGCM_ensemble_Init
404
405  if [[ ${ensemble_Ens_PARAMETRIC_active} = 'y' ]] ; then
406    echo "WARNING: Parametric Ensemble is not implemented yet..."
407  fi
408
409  if [[ ${ensemble_Ens_DATE_active} = 'y' ]] ; then
410    IGCM_sys_Cd ${SUBMIT_DIR}
411    IGCM_ensemble_DateInit
412    # As it says
413    IGCM_sys_Cd ${SUBMIT_DIR}
414    IGCM_ensemble_DatePeriodicStarts
415    # As it says
416    IGCM_sys_Cd ${SUBMIT_DIR}
417    IGCM_ensemble_DateNonPeriodicStarts
418    # Clean
419    IGCM_sys_Rm -rf ${RUN_DIR}
420  fi
421
422  if [[ ${ensemble_Ens_PERTURB_active} = 'y' ]] ; then
423    IGCM_sys_Cd ${SUBMIT_DIR}
424    IGCM_ensemble_CastInit
425    # As it says
426    IGCM_sys_Cd ${SUBMIT_DIR}
427    IGCM_ensemble_CastPeriodicStarts
428    # As it says
429    IGCM_sys_Cd ${SUBMIT_DIR}
430    IGCM_ensemble_CastNonPeriodicStarts
431    # As it says
432    IGCM_sys_Cd ${SUBMIT_DIR}
433    IGCM_ensemble_CastMemberList
434    # Done
435    IGCM_sys_Cp ${RUN_DIR}/CreatedDir.txt ${SUBMIT_DIR}
436    IGCM_sys_Cd ${SUBMIT_DIR}
437    # Clean
438    IGCM_sys_Rm -rf ${RUN_DIR}
439  fi
440fi
441#-
442[[ ${x_v} = 'verbose' ]] && print - "";
443#-
444# That's all folks
445#-
446
447if [ ${NbErr} -ne 0 ] ; then
448  echo "################ ERROR ################"
449  echo "${NbErr} invalid JobName(s) found, check the log"
450fi
451
452
453exit 0;
Note: See TracBrowser for help on using the repository browser.