Context Navigation

source: trunk/libIGCM/ins_job @ 1471

Last change on this file since 1471 was 1471, checked in by mafoipsl, 6 years ago
MAF+AC+AC : simplification of ins_job on Irene. xlarge or standard is proposed only if the project has xlarge hours.
Property svn:executable set to ``* Property svn:keywords set to `Revision Author Date`
File size: 16.6 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Jacques Belier
5	# Contact:
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13	#---------------------------------------------------------------------
14	#- Installation of jobs according to environment
15	#---------------------------------------------------------------------
16	function ins_job_Usage
17	{
18	print - "
19	ins_job installs the jobs in the directories
20	which contain a file config.card
21
22	ins_job must be launched on the host
23	on which the job will be submitted
24
25	Usage :
26	${b_n} [-h] [-v] [-e]
27	or on irene/TGCC :
28	${b_n} [-h] [-v] [-e] [-p project] [-q type_of_node] [-c number of cores]
29	or on ada /IDRIS
30	${b_n} [-h] [-v] [-e] [-m MPI environment]
31	Options :
32	-h : help
33	-v : verbose mode
34	-e : turn on ensemble mode (hindcast/forecast or date restart)
35	-f : ins_job is force to create jobs even if they already exist
36	on irene only :
37	-p project : add default project on irene
38	-q type_of_node : add default type of nodes for postprocessing on irene skylake/xlarge
39	-c number of cores : add default number of cores for postprocessing on irene 1-112
40	on ada only :
41	- m MPI environment : add default MPI environment (Intel MPI or IBM MPI)
42	"
43	}
44	function ins_job_Warning
45	{
46	[[ ${x_v} = 'verbose' ]] && print - "\n############### WARNING ###############";
47	[[ ${x_v} = 'verbose' ]] && print - "File ${n_f} already exists\nin directory ${j}";
48	[[ ${x_v} = 'verbose' ]] && print - "You must delete this file to update !";
49	}
50
51	function ins_job_Check_JobName
52	{
53	verif=${JobName##[a-zA-Z]*(?([.\-])[a-zA-Z0-9])}
54
55	if [ ${#verif} -ne 0 ] ; then
56	echo "################ ERROR ################"
57	echo "${JobName} is invalid."
58	echo "- JobName can only contain alphanumeric characters, \".\" and \"-\""
59	echo "- JobName must start with a letter"
60
61	((NbErr=NbErr+1))
62
63	Status=1
64	else
65	Status=0
66	fi
67
68	return ${Status}
69	}
70
71	#-
72	# dirname and basename
73	#-
74	d_n=$(dirname ${0}); b_n=$(basename ${0});
75	#-
76	# Retrieving and validation of the options
77	#-
78	x_v='silencious';
79	x_e=false;
80	x_f=false;
81	x_p=false;
82	x_q=false;
83	x_c=false;
84	x_m=false;
85	while getopts :hvefc:p:m:q:s: V ; do
86	case $V in
87	(h) ins_job_Usage; exit 0;;
88	(v) x_v='verbose';;
89	(e) x_e=true;;
90	(f) x_f=true;;
91	(p) x_p=true
92	ProjectID=${OPTARG} ;;
93	(q) x_q=true
94	ProjectNode=${OPTARG} ;;
95	(c) x_c=true
96	ProjectCore=${OPTARG} ;;
97	(m) x_m=true
98	MPIEnvironment=${OPTARG} ;;
99	(:) echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
100	exit 2;;
101	(\?) echo ${b_n}" : -"${OPTARG}" option : not supported" 1>&2;
102	exit 2;;
103	esac
104	done
105	[ ${x_v} = 'silencious' ] && export DEBUG_sys=false
106	shift $(($OPTIND-1));
107	#-
108	# Define working files
109	#-
110	F_MOD=$(cd ${d_n}'/..';/bin/pwd;)
111	# [[ ${F_MOD##*/} != 'modipsl' ]] && \
112	# { print - "directory 'modipsl' unreachable"; exit 3; }
113	W_W=${d_n}'/../libIGCM'
114	[[ ! -d ${W_W} ]] && { print - "${W_W} unreachable"; exit 3; }
115	libIGCM=$(cd ${W_W};/bin/pwd;)
116	F_JOB=${libIGCM}'/AA_job';
117	[[ ! -f ${F_JOB} ]] && { print - "${F_JOB} unreachable"; exit 3; }
118	F_RCI=${libIGCM}'/run.card.init';
119	[[ ! -f ${F_RCI} ]] && { print - "${F_RCI} unreachable"; exit 3; }
120	#-
121	# Accessing to functions (without stack)
122	#-
123	# No verbosity (0, 1, 2, 3)
124	Verbosity=0
125	# No de debug
126	DEBUG_debug=false
127	# Dont move libIGCM
128	MirrorlibIGCM=false
129	# Behave like computing job
130	TaskType=computing
131	# Source libIGCM
132	. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
133	. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
134	. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
135	. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
136	. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
137	if [ $x_e = 'true' ] ; then
138	. ${libIGCM}/libIGCM_ensemble/libIGCM_ensemble.ksh
139	fi
140
141	#-
142	[[ ${x_v} = 'verbose' ]] && \
143	{
144	print - "";
145	print - '--- Host : '${SYSTEM};
146	print - '--- modipsl : '${F_MOD};
147	print - '--- libIGCM : '${libIGCM};
148	print - '--- basic job : '${F_JOB};
149	print - '--- basic card : '${F_RCI};
150	}
151	#-
152	[[ ${x_v} = 'verbose' ]] && print - "\nInstallation of jobs for '${SYSTEM}'";
153	#-
154
155	NbErr=0
156
157	#-
158	# Define Project parameters to set up jobs header for Irene (TGCC)
159	# on Irene define ProjectID and ProjectCore : option or answer
160	# on Irene ProjectNode default projectnode set to xlarge since 17/10/2018
161	#-
162
163
164	if [ X"${SYSTEM}" == "Xirene" ] ; then
165
166	#- set ProjectID if required
167	if ( ! ${x_p} ) ; then
168	print - "Wait for the next question ..."
169	#- default ProjectID
170	ProjectID=$( ccc_myproject \| grep -i irene \|grep -i skylake \| gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' \| sort -u \| grep -v gencmip6 \| head -n 1 )
171	answer=""
172	print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject \| grep -i irene \| grep -i skylake \| gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' \| grep -v gencmip6 \| sort -u ) ) or other xxxcmip6 : $(for i in $(groups) ; do echo $i\|grep -v gencmip6\|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ; echo ) "
173	read answer
174
175	if [ "X${answer}" != "X" ] ; then
176	ProjectID=${answer}
177	fi
178
179	fi # if ( ! ${x_p} )
180
181	echo ProjectID is ${ProjectID} at Irene
182
183
184	# set ProjectNode if required
185	if ( ! ${x_q} ) ; then
186	#- default ProjectNode
187	ProjectNode="xlarge"
188	#- is xlarge possible for ${ProjectID} ?
189	echo $(ccc_myproject \| grep -i irene \| grep -i xlarge \| gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' \| grep -v gencmip6 \| sort -u) $(for i in $(groups) ; do echo $i\|grep -v gencmip6\|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ) \| grep ${ProjectID} >/dev/null 2>&1 \|\| ProjectNode="skylake"
190
191	if [ "X${ProjectNode}" == "Xxlarge" ] ; then
192	answerOK=false
193
194	while ( ! ${answerOK} ) ; do
195	answer=""
196	print - "Hit Enter or give TYPE OF NODE required for post-processing (default is \"${ProjectNode}\"), possible types of nodes are \"skylake\" or \"xlarge\" : "
197	read answer
198	[ "X${answer}" == "X" ] \|\| [ "X${answer}" == "Xskylake" ] \|\| [ "X${answer}" == "Xxlarge" ] && answerOK=true
199	done
200
201	if [ "X${answer}" != "X" ] ; then
202	ProjectNode=${answer}
203	fi
204
205	else
206	print - "PostProcessing will be done on ${ProjectNode}"
207	fi
208
209	fi # if ( ! ${x_q} )
210	echo ProjectNode for post-processing is ${ProjectNode} at Irene
211
212	#- ProjectNode is known (option or answer) set ProjectCoreMax
213	[ "${ProjectNode}" = "xlarge" ] && ProjectCoreMax="112" \|\| ProjectCoreMax="48"
214	#- ProjectCoreMax is 48 for standard and 112 for xlarge
215
216	if ( ! ${x_c} ) ; then
217	#- ProjectNode is known (option or answer), set ProjectCore default
218	[ "${ProjectNode}" = "xlarge" ] && ProjectCore="8" \|\| ProjectCore="4"
219
220	# let check minimum/maximum value 1/${ProjectCoreMax}
221
222	answerOK=false
223
224	while ( ! ${answerOK} ) ; do
225	answer=""
226	print - "possible numbers of cores are \"1\" to \"${ProjectCoreMax}\" for ${ProjectNode} : "
227	print - "Hit Enter or give NUMBER OF CORES required for post-processing (default is \"${ProjectCore}\")"
228	read answer
229	[ "X${answer}" == "X" ] \|\| [ ${answer} -ge 1 -a ${answer} -le ${ProjectCoreMax} ] && answerOK=true
230	done
231
232	if [ "X${answer}" != "X" ] ; then
233	ProjectCore=${answer}
234	fi
235
236	fi # if ( ! ${x_c} )
237
238	echo ProjectCore for post-processing is ${ProjectCore}
239	#- ProjectCore is set (option or answer)
240
241	elif [ X"${SYSTEM}" == "Xada" ] ; then
242	if ( ! ${x_m} ) ; then
243	MPIEnvironment=IBM
244	# Intel MPI Environment.
245	answerOK=false
246	while ( ! ${answerOK} ) ; do
247	answer=""
248	print - "Hit Enter or give MPI Environement (default is ${MPIEnvironment}), possible MPI environments are IBM (MPI IBM) and Intel (MPI Intel) :"
249	read answer
250	[ "X${answer}" == "X" ] \|\| [ "X${answer}" == "XIBM" ] \|\| [ "X${answer}" == "XIntel" ] && answerOK=true
251	done
252
253	if [ "X${answer}" != "X" ] ; then
254	MPIEnvironment=${answer}
255	fi
256	fi # if ( ! ${x_pm} )
257	echo MPIEnvironment is ${MPIEnvironment}
258
259	elif [ X"${SYSTEM}" == "Xlxiv8" ] \|\| [ X"${SYSTEM}" == "Xifort_CICLAD" ] ; then
260	# obelix, ciclad, climserv
261	echo ""
262	echo "You need to check and maybe adapt headers in the main job especially the line: "
263	echo " #PBS -l nodes=x:ppn=y "
264	echo "where x is the number of nodes, y the number of cores per node and x*y is the total number of cores for the job. "
265	echo "y must not be bigger than the maximum numer of cores per node on the machine (often 8 or 16)."
266	fi # if [ X"${SYSTEM}" == "Xcurie" ]
267
268	#-
269	# Define the pattern string to substitute
270	#-
271	W_P='#-Q- '; W_W=${W_P}${SYSTEM};
272	#-
273	# Extract list of 'config.card' files
274	# and create jobs with AA_job
275	#-
276	F_CFG='config.card';
277	F_CFG_ENS='ensemble.card';
278	SUBMIT_DIR_ENS=$( pwd )
279	for i in $( pwd )/config.card
280	do
281	if [ ! -f $i ] ; then
282	echo ""
283	echo "################## WARNING ##################"
284	echo "No config.card available in current directory"
285	echo ""
286	continue
287	fi
288
289
290	j=$(cd ${i%/*};/bin/pwd;)
291	n_f=${F_RCI##*/};
292
293	if [ ! X$( echo ${j} \| grep EXPERIMENTS ) = X ] ; then
294	# Do not treat config.card if it is in sub-directory of EXPERIMENTS
295	# Continue to next config.card
296	continue
297	else
298	[[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
299	fi
300
301	# Find out if new structure and set .resol filename
302	if [ -d ${j}/EXPERIMENTS ] && [ -d ${j}/GENERAL ] ; then
303	# New Structure
304	[[ ${x_v} = 'verbose' ]] && echo "This is new configuration structure"
305	new_struct=yes
306	resolfile=$j/.resol
307	else
308	# Old Structure
309	new_struct=no
310	resolfile=$j/../.resol
311	fi
312
313	# Get all variables declared in section UserChoices in config.card
314	IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} UserChoices
315	# Set default values
316	config_UserChoices_ExpType=""
317	RESOL_ATM_3D=this_is_a_test_string
318	RESOL=this_is_another_test_string
319	typeset option
320	for option in ${config_UserChoices[*]} ; do
321	IGCM_card_DefineVariableFromOption ${j}'/'${F_CFG} UserChoices ${option}
322	done
323
324	# Find the JobName : JobName might contain the variable RESOL_ATM_3D that will be replaced by what is in .resol file
325	if [ ! X$( echo ${config_UserChoices_JobName} \| grep ${RESOL_ATM_3D} ) = X ] ; then
326	TRUERESOL=$( tail -1 $resolfile \| awk "-F=" '{print $2}' )
327	echo TRUERESOL = $TRUERESOL
328	JobName=$( echo ${config_UserChoices_JobName} \| sed -e "s/${RESOL_ATM_3D}/${TRUERESOL}/" )
329	IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
330	elif [ ! X$( echo ${config_UserChoices_JobName} \| grep ${RESOL} ) = X ] ; then
331	TRUERESOL=$( head -1 $resolfile )
332	JobName=$( echo ${config_UserChoices_JobName} \| sed -e "s/${RESOL}/${TRUERESOL}/" )
333	IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
334	else
335	JobName=${config_UserChoices_JobName}
336	fi
337
338	# Check JobName validity : only alphanumerical characters, "-" and "." are authorized
339	ins_job_Check_JobName
340	RetCode=$?
341	[[ $RetCode -gt 0 ]] && continue
342
343	[[ ${x_v} = 'verbose' ]] && echo "JobName=${JobName}"
344
345	# Add specific treatment for new type of directory structure
346	if [ ${new_struct} == yes ] ; then
347
348	if [ "X${config_UserChoices_ExpType}" = X ] ; then
349	echo "\nERROR in ${j}/config.card"
350	echo "ins_job stops here"
351	echo "=> The variable ExpType must be added in config.card in section UserChoices"
352	echo "=> ExpType gives the directory for the .card configuration files for the wanted experiement. For exemple ExpType=IPSLCM5/historical"
353	exit 4
354	else
355	[[ ${x_v} = 'verbose' ]] && echo "ExpType= ${config_UserChoices_ExpType}"
356	fi
357
358	if [ -d ${j}/${JobName} ] ; then
359	echo "Directory ${j}/${JobName} exists already. It will not be overwritten."
360	echo "Remove the existing directory or change JobName before relaunching ins_job."
361	continue
362	fi
363	echo "=> Submit directory ${JobName} will be created with cards from EXPERIMENTS/${config_UserChoices_ExpType}"
364	cp -r ${j}/EXPERIMENTS/${config_UserChoices_ExpType} ${j}/${JobName}
365	cp -r ${j}/GENERAL/* ${j}/${JobName}/.
366	cp -f ${j}/${F_CFG} ${j}/${JobName}/.
367	if [ -f ${F_CFG_ENS} ] ; then
368	cp -f ${j}/${F_CFG_ENS} ${j}/${JobName}/.
369	SUBMIT_DIR_ENS=${j}/${JobName}
370	fi
371	rm -f ${j}/${F_CFG}
372	rm -f ${j}/${F_CFG_ENS}
373	rm -f ${j}/${F_CFG}.bak
374	j=${j}/${JobName}
375	[[ ${x_v} = 'verbose' ]] && echo new j=$j
376	fi
377	# end specific treatment for new type directory structure
378
379	[[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; } \|\| \
380	{
381	[[ ${x_v} = 'verbose' ]] && print - "\nCopying file ${F_RCI}\nin directory ${j}";
382	\cp ${F_RCI} ${j};
383	}
384
385	#==================================
386	# Read ListOfComponents section:
387	#echo
388	#IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"
389
390	IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} ListOfComponents
391	for comp in ${config_ListOfComponents[*]} ; do
392	IGCM_card_DefineArrayFromOption ${j}'/'${F_CFG} ListOfComponents ${comp}
393	done
394	#IGCM_debug_Print 3 ${config_ListOfComponents[*]}
395
396	#==================================
397	# Read Executable section:
398	IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} Executable
399
400	# Define the execution context (MPMD, SPMD, MPI/OMP ...)
401	IGCM_config_ConfigureExecution ${j}'/'${F_CFG}
402
403	# coreNumber : TOTAL NUMBER OF CORES
404	# mpiTasks : TOTAL NUMBER OF MPI TASKS
405	# openMPthreads : NUMBER OF OpenMP THREADS
406
407	# File name for Job
408	n_f='Job_'${JobName};
409	[[ ${x_f} = 'false' ]] && [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
410	[[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
411	sed -e "/^${W_W} */ s///" \
412	-e "/^${W_P}/d" \
413	-e "s%::modipsl::%${F_MOD}%" \
414	-e "s/::Jobname::/${JobName}/" \
415	-e "s/::default_project::/${ProjectID}/" \
416	${F_JOB} > ${j}'/'${n_f}
417	chmod u+x ${j}'/'${n_f}
418
419	# update Headers so that ressources description are accurate (MPMD/SPMD/...)
420	IGCM_sys_updateHeaders ${j}'/'${n_f}
421	done
422
423	#-
424	# Extract list of AA_* files in libIGCM
425	# and create jobs (for all except AA_job)
426	#-
427	for i in $(find ${libIGCM} -maxdepth 1 -name "AA_*" -print)
428	do
429	i_f=${i##*/};
430	[[ ${i_f} = 'AA_job' ]] && { continue; }
431	j=${i%/*}; n_f=${i_f#AA_}'.job';
432	[[ ${x_f} = 'false' ]] && [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
433	[[ ${x_v} = 'verbose' ]] && print - "\nIn directory ${j}\n${i_f} -> ${n_f}"
434	sed -e "/^${W_W} */ s///" \
435	-e "s%::modipsl::%${F_MOD}%" \
436	-e "/^${W_P}/d" \
437	-e "s/::default_node::/${ProjectNode}/" \
438	-e "s/::default_core::/${ProjectCore}/" \
439	-e "s/::default_project::/${ProjectID}/" \
440	-e "s/::default_post_project::/${ProjectID}/" \
441	${i} > ${j}'/'${n_f}
442	chmod u+x ${j}'/'${n_f}
443	done
444	#-
445	# set default_project in libIGCM_sys_curie.ksh too.
446	#-
447	if [ X"${SYSTEM}" == "Xcurie" ] ; then
448	i=${libIGCM}/libIGCM_sys/libIGCM_sys_curie.ksh
449	sed -i -e "s/::default_project::/${ProjectID}/" ${i}
450	fi
451	#-
452	# Limited to hindcast/forecast and date restart Ensemble for the time being
453	if [ ${x_e} = 'true' ] ; then
454	#.. Read input data from ensemble.card ..
455	SUBMIT_DIR=${SUBMIT_DIR_ENS}
456	RUN_DIR="${WORKDIR}/ENSEMBLE_TMP"
457	#
458	# Copy initial things around and define variables (hindcast/forecast case)
459	IGCM_sys_Cd ${SUBMIT_DIR}
460	IGCM_ensemble_Init
461
462	if [[ ${ensemble_Ens_PARAMETRIC_active} = 'y' ]] ; then
463	echo "WARNING: Parametric Ensemble is not implemented yet..."
464	fi
465
466	if [[ ${ensemble_Ens_DATE_active} = 'y' ]] ; then
467	IGCM_sys_Cd ${SUBMIT_DIR}
468	IGCM_ensemble_DateInit
469	# As it says
470	IGCM_sys_Cd ${SUBMIT_DIR}
471	IGCM_ensemble_DatePeriodicStarts
472	# As it says
473	IGCM_sys_Cd ${SUBMIT_DIR}
474	IGCM_ensemble_DateNonPeriodicStarts
475	# Clean
476	IGCM_sys_Rm -rf ${RUN_DIR}
477	fi
478
479	if [[ ${ensemble_Ens_PERTURB_active} = 'y' ]] ; then
480	IGCM_sys_Cd ${SUBMIT_DIR}
481	IGCM_ensemble_CastInit
482	# As it says
483	IGCM_sys_Cd ${SUBMIT_DIR}
484	IGCM_ensemble_CastPeriodicStarts
485	# As it says
486	IGCM_sys_Cd ${SUBMIT_DIR}
487	IGCM_ensemble_CastNonPeriodicStarts
488	# As it says
489	IGCM_sys_Cd ${SUBMIT_DIR}
490	IGCM_ensemble_CastMemberList
491	# Done
492	#IGCM_sys_Cp ${RUN_DIR}/CreatedDir.txt ${SUBMIT_DIR}
493	IGCM_sys_Cd ${SUBMIT_DIR}
494	# Clean
495	IGCM_sys_Rm -rf ${RUN_DIR}
496	fi
497	fi
498	#-
499	[[ ${x_v} = 'verbose' ]] && print - "";
500	#-
501	# That's all folks
502	#-
503
504	if [ ${NbErr} -ne 0 ] ; then
505	echo "################ ERROR ################"
506	echo "${NbErr} invalid JobName(s) found, check the log"
507	fi
508
509
510	exit 0;

Note: See TracBrowser for help on using the repository browser.

Download in other formats: