| 1 | #!/bin/ksh |
|---|
| 2 | #set -x |
|---|
| 3 | |
|---|
| 4 | host=`uname -n` |
|---|
| 5 | user=`whoami` |
|---|
| 6 | |
|---|
| 7 | ## - User's choice of computing architecture |
|---|
| 8 | arch=kraken_intel_impi_openmp # kraken_intel_impi_openmp, belenos |
|---|
| 9 | |
|---|
| 10 | ## - Define paths |
|---|
| 11 | srcdir=`pwd` |
|---|
| 12 | oasisdir=$srcdir/OASIS |
|---|
| 13 | xiosdir=$srcdir/XIOS |
|---|
| 14 | esmfdir=$srcdir/ESMF |
|---|
| 15 | casename=`basename $srcdir` |
|---|
| 16 | |
|---|
| 17 | ## - Define case |
|---|
| 18 | if [ $# -eq 0 ] ; then |
|---|
| 19 | echo "By default, i.e. without arguments, the source grid is bggd," |
|---|
| 20 | echo "the target grid is nogt and the remapping is 1st order conservative with SCRIP;" |
|---|
| 21 | echo "1 node, 1 MPI task per node and 1 OpenMP thread per MPI task are used for the run," |
|---|
| 22 | echo "and no suffixe is used in the rundir name." |
|---|
| 23 | SGRID=bggd ; TGRID=nogt ; remap=conserv1st ; n_p_t=1_1_1 ; nnode=1 ; mpiprocs=1 ; threads=1 ; library=SCRP ; ext="" |
|---|
| 24 | elif [ $# -ne 6 ] ; then |
|---|
| 25 | echo "If you don't want to run the default case without arguments, " |
|---|
| 26 | echo "you must run the script with 6 arguments i.e. './run_testinterp.sh src tgt remap nnodes_nprocs_nthreads library ext'" |
|---|
| 27 | echo "where 'src' is the source grid, 'tgt' the target grid and 'remap' the remapping," |
|---|
| 28 | echo "'nnodes' the total number of nodes for the run, 'nprocs' the number of MPI tasks per node" |
|---|
| 29 | echo "'nthreads' the number of OpenMP threads per MPI task" |
|---|
| 30 | echo "'library' is the regridder used (either SCRP, ESMF or XIOS)" |
|---|
| 31 | echo "'ext' is suffixe is used in the rundir name." |
|---|
| 32 | exit |
|---|
| 33 | else |
|---|
| 34 | SGRID=$1 ; TGRID=$2 ; remap=$3 ; n_p_t=$4 ; library=$5 ; ext=$6 |
|---|
| 35 | nnode=`echo $n_p_t | awk -F _ '{print $1}'` |
|---|
| 36 | mpiprocs=`echo $n_p_t | awk -F _ '{print $2}'` |
|---|
| 37 | threads=`echo $n_p_t | awk -F _ '{print $3}'` |
|---|
| 38 | fi |
|---|
| 39 | nproces=`echo $(($nnode*$mpiprocs))` |
|---|
| 40 | |
|---|
| 41 | ## - Check grids |
|---|
| 42 | ## bggd is an atmosphere structured (LR) grid ; sse7 is an atmosphere gaussian reduced (D) grid |
|---|
| 43 | ## icos/icoh is an atmosphere unstructured (U) grid ; nogt/torc is an ocean structured (LR) grid |
|---|
| 44 | if [ ${SGRID} != "bggd" ] && [ ${SGRID} != "sse7" ] && [ ${SGRID} != "icos" ] && [ ${SGRID} != "icoh" ] && [ ${SGRID} != "nogt" ] && [ ${SGRID} != "torc" ]; then |
|---|
| 45 | echo "Source grid must be either bggd, sse7, icos, icoh, nogt or torc" |
|---|
| 46 | exit |
|---|
| 47 | fi |
|---|
| 48 | if [ ${TGRID} != "bggd" ] && [ ${TGRID} != "sse7" ] && [ ${TGRID} != "icos" ] && [ ${TGRID} != "icoh" ] && [ ${TGRID} != "nogt" ] && [ ${TGRID} != "torc" ]; then |
|---|
| 49 | echo "Target grid must be either bggd, sse7, icos, icoh, nogt, torc" |
|---|
| 50 | exit |
|---|
| 51 | fi |
|---|
| 52 | if [ ${SGRID} == "bggd" ] && [ ${SGRID} == "sse7" ] && [ ${SGRID} == "icos" ] && [ ${SGRID} == "icoh" ]; then |
|---|
| 53 | if [ ${TGRID} != "nogt" ] && [ ${TGRID} != "torc" ]; then |
|---|
| 54 | echo "You have to match an atmospheric grid (bggd, sse7, icos or icoh) with an ocean grid (nogt, torc)" |
|---|
| 55 | exit |
|---|
| 56 | fi |
|---|
| 57 | fi |
|---|
| 58 | if [ ${TGRID} == "bggd" ] && [ ${TGRID} == "sse7" ] && [ ${TGRID} == "icos" ] && [ ${TGRID} == "icoh" ]; then |
|---|
| 59 | if [ ${SGRID} != "nogt" ] && [ ${SGRID} != "torc" ]; then |
|---|
| 60 | echo "You have to match an an ocean grid (nogt, torc) with an atmospheric grid (bggd, sse7, icos or icoh)" |
|---|
| 61 | exit |
|---|
| 62 | fi |
|---|
| 63 | fi |
|---|
| 64 | |
|---|
| 65 | ## - Check remap |
|---|
| 66 | ## distwgt (nearest-neighbour), bili (bilinear), bicu (bicubic), conserv1st or conserv2nd (1st or 2nd order conservative remapping) |
|---|
| 67 | if [ ${remap} != "distwgt" ] && [ ${remap} != "bili" ] && [ ${remap} != "bicu" ] && [ ${remap} != "conserv1st" ] && [ ${remap} != "conserv2nd" ]; then |
|---|
| 68 | echo "Remapping must be either distwgt, bili, bicu, conserv1st, conserv2nd" |
|---|
| 69 | exit |
|---|
| 70 | fi |
|---|
| 71 | |
|---|
| 72 | ## - Check library |
|---|
| 73 | if [ ${library} != "SCRP" ] && [ ${library} != "ESMF" ] && [ ${library} != "XIOS" ]; then |
|---|
| 74 | echo "Remapping library must be either SCRP (for SCRIP), ESMF or XIOS" |
|---|
| 75 | exit |
|---|
| 76 | fi |
|---|
| 77 | |
|---|
| 78 | ## - Check source grid type and remapping for SCRP (no conserv2nd for sse7 ; no bili, bicu, conserv2nd for icos) |
|---|
| 79 | if [ ${library} == "SCRP" ]; then |
|---|
| 80 | if [ ${SGRID} == "sse7" ]; then |
|---|
| 81 | if [ ${remap} == "conserv2nd" ]; then |
|---|
| 82 | echo "Impossible to perform conserv2nd remapping from gaussian reduced grid sse7" |
|---|
| 83 | exit |
|---|
| 84 | fi |
|---|
| 85 | elif [ ${SGRID} == "icos" ] || [ ${SGRID} == "icoh" ]; then |
|---|
| 86 | if [ ${remap} == "conserv2nd" ] || [ ${remap} == "bicu" ] || [ ${remap} == "bili" ]; then |
|---|
| 87 | echo "Impossible to perform ${remap} remapping from unstructured grid icos" |
|---|
| 88 | exit |
|---|
| 89 | fi |
|---|
| 90 | fi |
|---|
| 91 | fi |
|---|
| 92 | |
|---|
| 93 | ## - Only 1st and 2nd order conservative remapping for XIOS |
|---|
| 94 | if [ ${library} == "XIOS" ]; then |
|---|
| 95 | if [ ${remap} == "conserv1st" ]; then |
|---|
| 96 | xiosmethod=CONSERV_FRACAREA |
|---|
| 97 | order=1 |
|---|
| 98 | elif [ ${remap} == "conserv2nd" ]; then |
|---|
| 99 | xiosmethod=CONS2ND_FRACAREA |
|---|
| 100 | order=2 |
|---|
| 101 | else |
|---|
| 102 | echo "XIOS does not support ${remap} remapping " |
|---|
| 103 | exit |
|---|
| 104 | fi |
|---|
| 105 | fi |
|---|
| 106 | ## - Source grid characteristics |
|---|
| 107 | if [ ${SGRID} == bggd ]; then |
|---|
| 108 | STYPE=LR ; SRCP=P ; SRCPN=0 |
|---|
| 109 | elif [ ${SGRID} == sse7 ]; then |
|---|
| 110 | STYPE=D ; SRCP=P ; SRCPN=0 |
|---|
| 111 | elif [ ${SGRID} == icos ] || [ ${SGRID} == icoh ]; then |
|---|
| 112 | STYPE=U ; SRCP=P ; SRCPN=0 |
|---|
| 113 | elif [ ${SGRID} == nogt ] || [ ${SGRID} == torc ]; then |
|---|
| 114 | STYPE=LR ; SRCP=P ; SRCPN=2 |
|---|
| 115 | fi |
|---|
| 116 | ## - Target grid characteristics |
|---|
| 117 | if [ ${TGRID} == bggd ]; then |
|---|
| 118 | TTYPE=LR ; TGTP=P |
|---|
| 119 | elif [ ${TGRID} == sse7 ]; then |
|---|
| 120 | TTYPE=D ; TGTP=P |
|---|
| 121 | elif [ ${TGRID} == icos ] || [ ${TGRID} == icoh ]; then |
|---|
| 122 | TTYPE=U ; TGTP=P |
|---|
| 123 | elif [ ${TGRID} == nogt ] || [ ${TGRID} == torc ]; then |
|---|
| 124 | TTYPE=LR ; TGTP=P |
|---|
| 125 | fi |
|---|
| 126 | |
|---|
| 127 | ## - rundir definition |
|---|
| 128 | rundir=$srcdir/RUNDIR_${library}_${ext}/${casename}_${SGRID}_${TGRID}_${remap}_${nnode}_${mpiprocs}_${threads}_${library}_${ext} |
|---|
| 129 | \rm -fr $rundir/* ; mkdir -p $rundir |
|---|
| 130 | |
|---|
| 131 | ## - Create namcouple |
|---|
| 132 | ./namcouple_create.sh ${SGRID} ${TGRID} ${remap} ${n_p_t} ${library} ${ext} |
|---|
| 133 | |
|---|
| 134 | ## - Name of the executables |
|---|
| 135 | exe1=model1 |
|---|
| 136 | ## |
|---|
| 137 | echo '' |
|---|
| 138 | echo '**************************************************************************************************************' |
|---|
| 139 | echo '*** '$casename' : '$run |
|---|
| 140 | echo '' |
|---|
| 141 | echo "Running test_interpolation on $nnode nodes with $mpiprocs MPI tasks per node and $threads threads per MPI task" |
|---|
| 142 | echo '**************************************************************************************************************' |
|---|
| 143 | echo 'Source grid :' $SGRID |
|---|
| 144 | echo 'Target grid :' $TGRID |
|---|
| 145 | echo 'Rundir :' $rundir |
|---|
| 146 | echo 'Architecture :' $arch |
|---|
| 147 | echo 'Host : '$host |
|---|
| 148 | echo 'User : '$user |
|---|
| 149 | echo 'Grids : '$SGRID'-->'$TGRID |
|---|
| 150 | echo 'Remap : '$remap |
|---|
| 151 | echo 'Remapping library: '$library |
|---|
| 152 | echo '' |
|---|
| 153 | echo $exe1' runs on '$nproces 'processes' |
|---|
| 154 | echo '' |
|---|
| 155 | |
|---|
| 156 | ### - Define mask name which depends on ocean grid |
|---|
| 157 | if [ ${SGRID} == "nogt" ] || [ ${TGRID} == "nogt" ]; then |
|---|
| 158 | maskname=$oasisdir/${library}_masks/masks_nogt_${library}.nc |
|---|
| 159 | elif [ ${SGRID} == "torc" ] || [ ${TGRID} == "torc" ]; then |
|---|
| 160 | maskname=$oasisdir/${library}_masks/masks_torc_${library}.nc |
|---|
| 161 | fi |
|---|
| 162 | |
|---|
| 163 | ### |
|---|
| 164 | if [ ${library} == "ESMF" ]; then |
|---|
| 165 | ## With ESMF, nogt should be transformed to an unstructured grid |
|---|
| 166 | OasisGridsToESMF="OasisGridsToESMF.py" |
|---|
| 167 | if [ ${SGRID} == "nogt" ]; then |
|---|
| 168 | if [ ${remap} == "conserv1st" ] || [ ${remap} == "conserv2nd" ]; then |
|---|
| 169 | OasisGridsToESMF="OasisGridsToESMF_nogtunstruct.py" |
|---|
| 170 | fi |
|---|
| 171 | fi |
|---|
| 172 | cp -f $esmfdir/$OasisGridsToESMF $rundir/. |
|---|
| 173 | cp -f $esmfdir/ESMFWeightsToOasis.sh $rundir/. |
|---|
| 174 | ### Define regridding options |
|---|
| 175 | case $remap in |
|---|
| 176 | bili) meth_esmfname="bilinear" ; options="--extrap_method neareststod --src_loc center --dst_loc center" ;; |
|---|
| 177 | bicu) meth_esmfname="patch" ; options="--extrap_method neareststod --src_loc center --dst_loc center" ;; |
|---|
| 178 | distwgt) meth_esmfname="neareststod" ; options="--extrap_method neareststod --src_loc center --dst_loc center" ;; |
|---|
| 179 | conserv1st) meth_esmfname="conserve" ; options="--ignore_unmapped --norm_type fracarea" ;; |
|---|
| 180 | conserv2nd) meth_esmfname="conserve2nd" ; options="--ignore_unmapped --norm_type fracarea" ;; |
|---|
| 181 | *) echo "Method $remap unknown in ESMF." |
|---|
| 182 | exit ;; |
|---|
| 183 | esac |
|---|
| 184 | ## |
|---|
| 185 | elif [ ${library} == "XIOS" ]; then |
|---|
| 186 | exexios=oasis_testcase.exe |
|---|
| 187 | cat <<EOF > param.def |
|---|
| 188 | ¶ms_run |
|---|
| 189 | nb_proc_toy=$nproces |
|---|
| 190 | / |
|---|
| 191 | EOF |
|---|
| 192 | cp -f param.def $rundir/param.def |
|---|
| 193 | cat <<EOF > sed.sh |
|---|
| 194 | #!/bin/ksh |
|---|
| 195 | sed -e 's:SGRID:$SGRID:' -e 's:TGRID:$TGRID:' -e 's:GRIDS:$rundir/grids.nc:' -e 's:MASKS:$rundir/masks.nc:' $xiosdir/iodef.xml_template > ${rundir}/iodef.xml |
|---|
| 196 | sed -e 's:ORDER:$order:' $xiosdir/context_toy.xml_template > ${rundir}/context_toy.xml |
|---|
| 197 | EOF |
|---|
| 198 | chmod u+x sed.sh |
|---|
| 199 | ./sed.sh |
|---|
| 200 | cp -f $xiosdir/$exexios $rundir/$exexios |
|---|
| 201 | # |
|---|
| 202 | fi |
|---|
| 203 | |
|---|
| 204 | ### - Link everything needed into rundir |
|---|
| 205 | ln -sf $oasisdir/grids.nc $rundir/grids.nc |
|---|
| 206 | ln -sf ${maskname} $rundir/masks.nc |
|---|
| 207 | ln -sf $srcdir/src/$exe1 $rundir/. |
|---|
| 208 | |
|---|
| 209 | ## - Create name_grids.dat, that will be read by the models, from namcouple informations |
|---|
| 210 | cat <<EOF >> $rundir/name_grids.dat |
|---|
| 211 | \$grid_source_characteristics |
|---|
| 212 | cl_grd_src='$SGRID' |
|---|
| 213 | cl_remap='$remap' |
|---|
| 214 | cl_type_src='$STYPE' |
|---|
| 215 | cl_period_src='$SRCP' |
|---|
| 216 | il_overlap_src=$SRCPN |
|---|
| 217 | \$end |
|---|
| 218 | \$grid_target_characteristics |
|---|
| 219 | cl_grd_tgt='$TGRID' |
|---|
| 220 | cl_type_tgt='$TTYPE' |
|---|
| 221 | \$end |
|---|
| 222 | \$remapper |
|---|
| 223 | cl_library='$library' |
|---|
| 224 | \$end |
|---|
| 225 | EOF |
|---|
| 226 | # |
|---|
| 227 | cd $rundir |
|---|
| 228 | |
|---|
| 229 | ###################################################################### |
|---|
| 230 | ## - Creation of batch job scripts |
|---|
| 231 | |
|---|
| 232 | ###--------------------------------------------------------------------- |
|---|
| 233 | ### KRAKEN_INTEL_IMPI_OPENMP |
|---|
| 234 | ###--------------------------------------------------------------------- |
|---|
| 235 | if [ ${arch} == kraken_intel_impi_openmp ]; then |
|---|
| 236 | timreq=00:30:00 |
|---|
| 237 | if [ ${library} == "SCRP" ]; then |
|---|
| 238 | cat <<EOF > $rundir/run_$casename.$arch |
|---|
| 239 | #!/bin/bash -l |
|---|
| 240 | #Partition |
|---|
| 241 | #SBATCH --partition prod |
|---|
| 242 | # Nom du job |
|---|
| 243 | #SBATCH --job-name ${n_p_t} |
|---|
| 244 | # Time limit for the job |
|---|
| 245 | #SBATCH --time=$timreq |
|---|
| 246 | #SBATCH --output=$rundir/$casename.o |
|---|
| 247 | #SBATCH --error=$rundir/$casename.e |
|---|
| 248 | # Number of nodes |
|---|
| 249 | #SBATCH --nodes=$nnode |
|---|
| 250 | # Number of MPI tasks per node |
|---|
| 251 | #SBATCH --ntasks-per-node=$mpiprocs |
|---|
| 252 | |
|---|
| 253 | cd $rundir |
|---|
| 254 | |
|---|
| 255 | export KMP_STACKSIZE=1GB |
|---|
| 256 | export I_MPI_PIN_DOMAIN=omp |
|---|
| 257 | export I_MPI_WAIT_MODE=enable |
|---|
| 258 | export KMP_AFFINITY=verbose,granularity=fine,compact |
|---|
| 259 | export OASIS_OMP_NUM_THREADS=$threads |
|---|
| 260 | |
|---|
| 261 | time mpirun -np $nproces ./$exe1 |
|---|
| 262 | EOF |
|---|
| 263 | |
|---|
| 264 | elif [ ${library} == "ESMF" ]; then |
|---|
| 265 | |
|---|
| 266 | |
|---|
| 267 | cat <<EOF > $rundir/run_$casename.$arch |
|---|
| 268 | #!/bin/bash -l |
|---|
| 269 | #Partition |
|---|
| 270 | #SBATCH --partition prod |
|---|
| 271 | # Nom du job |
|---|
| 272 | #SBATCH --job-name ${n_p_t} |
|---|
| 273 | # Time limit for the job |
|---|
| 274 | #SBATCH --time=$timreq |
|---|
| 275 | #SBATCH --output=$rundir/$casename.o |
|---|
| 276 | #SBATCH --error=$rundir/$casename.e |
|---|
| 277 | # Number of nodes |
|---|
| 278 | #SBATCH --nodes=$nnode |
|---|
| 279 | # Number of MPI tasks per node |
|---|
| 280 | #SBATCH --ntasks-per-node=$mpiprocs |
|---|
| 281 | # Number of OpenMP threads per MPI task |
|---|
| 282 | #SBATCH --cpus-per-task=$threads |
|---|
| 283 | |
|---|
| 284 | cd $rundir |
|---|
| 285 | |
|---|
| 286 | export KMP_STACKSIZE=1GB |
|---|
| 287 | export I_MPI_PIN_DOMAIN=omp |
|---|
| 288 | export I_MPI_WAIT_MODE=enable |
|---|
| 289 | export KMP_AFFINITY=verbose,granularity=fine,compact |
|---|
| 290 | export OMP_NUM_THREADS=$threads |
|---|
| 291 | |
|---|
| 292 | python ./$OasisGridsToESMF $SGRID $rundir |
|---|
| 293 | python ./$OasisGridsToESMF $TGRID $rundir |
|---|
| 294 | # Generate ESMF weights |
|---|
| 295 | time mpirun -np $nproces ESMF_RegridWeightGen -s ${SGRID}_ESMF.nc -d ${TGRID}_ESMF.nc -m ${meth_esmfname} -w ESMFweights.nc --ignore_degenerate ${options} |
|---|
| 296 | |
|---|
| 297 | # Convert ESMF weight file in OASIS format |
|---|
| 298 | ./ESMFWeightsToOasis.sh ${SGRID} ${TGRID} ${remap} |
|---|
| 299 | |
|---|
| 300 | time mpirun -np $nproces ./$exe1 |
|---|
| 301 | EOF |
|---|
| 302 | elif [ ${library} == "XIOS" ]; then |
|---|
| 303 | cat <<EOF > $rundir/run_$casename.$arch |
|---|
| 304 | #!/bin/bash -l |
|---|
| 305 | #Partition |
|---|
| 306 | #SBATCH --partition prod |
|---|
| 307 | # Nom du job |
|---|
| 308 | #SBATCH --job-name ${n_p_t} |
|---|
| 309 | # Time limit for the job |
|---|
| 310 | #SBATCH --time=$timreq |
|---|
| 311 | #SBATCH --output=$rundir/$casename.o |
|---|
| 312 | #SBATCH --error=$rundir/$casename.e |
|---|
| 313 | # Number of nodes |
|---|
| 314 | #SBATCH --nodes=$nnode |
|---|
| 315 | # Number of MPI tasks per node |
|---|
| 316 | #SBATCH --ntasks-per-node=$mpiprocs |
|---|
| 317 | # Number of OpenMP threads per MPI task |
|---|
| 318 | #SBATCH --cpus-per-task=$threads |
|---|
| 319 | |
|---|
| 320 | cd $rundir |
|---|
| 321 | |
|---|
| 322 | export KMP_STACKSIZE=1GB |
|---|
| 323 | export I_MPI_PIN_DOMAIN=omp |
|---|
| 324 | export I_MPI_WAIT_MODE=enable |
|---|
| 325 | export KMP_AFFINITY=verbose,granularity=fine,compact |
|---|
| 326 | export OMP_NUM_THREADS=$threads |
|---|
| 327 | |
|---|
| 328 | # Generate XIOS weights |
|---|
| 329 | time mpirun -np $nproces ./$exexios |
|---|
| 330 | # Convert XIOS weight file in OASIS format |
|---|
| 331 | python $srcdir/XIOS/XiosWeightsToOasis.py |
|---|
| 332 | ln -sf rmp_${SGRID}_to_${TGRID}_xios_${xiosmethod}.nc rmp_${SGRID}_${TGRID}.nc |
|---|
| 333 | # |
|---|
| 334 | time mpirun -np $nproces ./$exe1 |
|---|
| 335 | EOF |
|---|
| 336 | fi |
|---|
| 337 | |
|---|
| 338 | elif [ $arch == belenos ] ; then |
|---|
| 339 | if [ ${library} == "SCRP" ]; then |
|---|
| 340 | cat <<EOF > $rundir/run_$casename.$arch |
|---|
| 341 | #!/bin/bash |
|---|
| 342 | #SBATCH --exclusive |
|---|
| 343 | #SBATCH --partition=normal256 |
|---|
| 344 | #SBATCH --job-name ${remap}_${nthreads} |
|---|
| 345 | #SBATCH --time=02:00:00 |
|---|
| 346 | #SBATCH -o $rundir/$casename.o |
|---|
| 347 | #SBATCH -e $rundir/$casename.e |
|---|
| 348 | #SBATCH -N $nnode |
|---|
| 349 | #SBATCH --ntasks-per-node=$mpiprocs |
|---|
| 350 | # |
|---|
| 351 | ulimit -s unlimited |
|---|
| 352 | cd $rundir |
|---|
| 353 | # |
|---|
| 354 | export KMP_STACKSIZE=1GB |
|---|
| 355 | export I_MPI_WAIT_MODE=enable |
|---|
| 356 | export KMP_AFFINITY=verbose,granularity=fine,compact |
|---|
| 357 | export OASIS_OMP_NUM_THREADS=$threads |
|---|
| 358 | export OMP_NUM_THREADS=$threads |
|---|
| 359 | # |
|---|
| 360 | time mpirun -np ${nproces} ./$exe1 |
|---|
| 361 | # |
|---|
| 362 | EOF |
|---|
| 363 | |
|---|
| 364 | elif [ ${library} == "ESMF" ]; then |
|---|
| 365 | cat <<EOF > $rundir/run_$casename.$arch |
|---|
| 366 | #!/bin/bash -l |
|---|
| 367 | #SBATCH --exclusive |
|---|
| 368 | #SBATCH --partition=normal256 |
|---|
| 369 | #SBATCH --job-name ${remap}_${nthreads} |
|---|
| 370 | #SBATCH --time=02:00:00 |
|---|
| 371 | #SBATCH -o $rundir/$casename.o |
|---|
| 372 | #SBATCH -e $rundir/$casename.e |
|---|
| 373 | #SBATCH -N $nnode |
|---|
| 374 | #SBATCH --ntasks-per-node=$mpiprocs |
|---|
| 375 | # |
|---|
| 376 | ulimit -s unlimited |
|---|
| 377 | |
|---|
| 378 | cd $rundir |
|---|
| 379 | |
|---|
| 380 | # |
|---|
| 381 | export KMP_STACKSIZE=1GB |
|---|
| 382 | export I_MPI_WAIT_MODE=enable |
|---|
| 383 | export KMP_AFFINITY=verbose,granularity=fine,compact |
|---|
| 384 | |
|---|
| 385 | python ./$OasisGridsToESMF $SGRID $rundir |
|---|
| 386 | python ./$OasisGridsToESMF $TGRID $rundir |
|---|
| 387 | # Generate ESMF weights |
|---|
| 388 | time mpirun -np $nproces ESMF_RegridWeightGen -s ${SGRID}_ESMF.nc -d ${TGRID}_ESMF.nc -m ${meth_esmfname} -w ESMFweights.nc --ignore_degenerate ${options} |
|---|
| 389 | |
|---|
| 390 | # Convert ESMF weight file in OASIS format |
|---|
| 391 | ./ESMFWeightsToOasis.sh ${SGRID} ${TGRID} ${remap} |
|---|
| 392 | |
|---|
| 393 | time mpirun -np $nproces ./$exe1 |
|---|
| 394 | EOF |
|---|
| 395 | |
|---|
| 396 | elif [ ${library} == "XIOS" ]; then |
|---|
| 397 | cat <<EOF > $rundir/run_$casename.$arch |
|---|
| 398 | #!/bin/bash -l |
|---|
| 399 | #SBATCH --exclusive |
|---|
| 400 | #SBATCH --partition=normal256 |
|---|
| 401 | #SBATCH --job-name ${remap}_${nthreads} |
|---|
| 402 | #SBATCH --time=02:00:00 |
|---|
| 403 | #SBATCH -o $rundir/$casename.o |
|---|
| 404 | #SBATCH -e $rundir/$casename.e |
|---|
| 405 | #SBATCH -N $nnode |
|---|
| 406 | #SBATCH --ntasks-per-node=$mpiprocs |
|---|
| 407 | # |
|---|
| 408 | ulimit -s unlimited |
|---|
| 409 | |
|---|
| 410 | cd $rundir |
|---|
| 411 | |
|---|
| 412 | # |
|---|
| 413 | export KMP_STACKSIZE=1GB |
|---|
| 414 | export I_MPI_WAIT_MODE=enable |
|---|
| 415 | export KMP_AFFINITY=verbose,granularity=fine,compact |
|---|
| 416 | # |
|---|
| 417 | # Generate XIOS weights |
|---|
| 418 | time mpirun -np $nproces ./$exexios |
|---|
| 419 | # Convert ESMF weight file in OASIS format |
|---|
| 420 | python $srcdir/XIOS/XiosWeightsToOasis.py |
|---|
| 421 | ln -sf rmp_${SGRID}_to_${TGRID}_xios_${xiosmethod}.nc rmp_${SGRID}_${TGRID}.nc |
|---|
| 422 | # |
|---|
| 423 | time mpirun -np $nproces ./$exe1 |
|---|
| 424 | EOF |
|---|
| 425 | fi |
|---|
| 426 | fi |
|---|
| 427 | |
|---|
| 428 | ###################################################################### |
|---|
| 429 | ### - Execute the model |
|---|
| 430 | |
|---|
| 431 | if [ $arch == kraken_intel_impi_openmp ]; then |
|---|
| 432 | echo 'Submitting the job to queue using sbatch' |
|---|
| 433 | sbatch $rundir/run_$casename.$arch |
|---|
| 434 | squeue -u $USER |
|---|
| 435 | elif [ $arch == belenos ]; then |
|---|
| 436 | echo 'Submitting the job to queue using sbatch' |
|---|
| 437 | sbatch $rundir/run_$casename.$arch |
|---|
| 438 | squeue -u $user |
|---|
| 439 | fi |
|---|
| 440 | |
|---|
| 441 | echo $casename 'is executed or submitted to queue.' |
|---|
| 442 | echo 'Results are found in rundir : '$rundir |
|---|
| 443 | |
|---|
| 444 | ###################################################################### |
|---|
| 445 | |
|---|
