source: configs/testing/arch/arch-X64_JEANZAY.sh

#!/bin/bash

# This script provides site-specific functions
# called by the site-independent scripts
# create_runs.sh and submit.sh

# NB : we must use the queue 'normal' because there seems to be a limit
# on the number of jobs that can be submitted to queue 'test'

#----------------------------- split_XXX ----------------------------

function split_mpi_32() {
    setvar nbp 33 nsplit_i 2 nsplit_j 4
}
function split_serial_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2
}
function split_mpi_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2
}
function split_mpi_80() {
    setvar nbp 81 nsplit_i 4 nsplit_j 2
}
function split_mpi_omp_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2 omp_level_size 4
}

#----------------- job submission ------------------

function submit_job_X64_IRENE() # $1=script $* = SLURM OPTIONS
{
    TMP=$(mktemp)
    ccc_msub $1 > $TMP
    cat $TMP 1>&2
    cat $TMP | awk '{print $NF }'
    rm -f $TMP
}

#------------------------------ job_XXX -----------------------------

# Serial jobs
function job_serial() { # EXP_NAME
    job_X64_JEANZAY $1 1 1 "./gcm.exe"
}
function job_serial_40() { # EXP_NAME
    job_serial $1
}

# MPI jobs
function job_mpi() { #EXP_NAME NB_MPI
    job_X64_JEANZAY $1 $2 1 "srun ./gcm.exe"
}
function job_mpi_32() { # EXP_NAME
    job_mpi $1 80
}
function job_mpi_40() { # EXP_NAME
    job_mpi $1 40
}
function job_mpi_80() { # EXP_NAME
    job_mpi $1 80
}

# MPI-OMP jobs
function job_mpi_omp() { #EXP_NAME NB_MPI NB_OMP
    job_X64_JEANZAY $1 $2 $3 "srun ./gcm.exe"
}
function job_mpi_omp_40() { # EXP_NAME
    job_mpi_omp $1 40 4
}

# Generic
function job_X64_JEANZAY() { # EXP_NAME MPI_TASKS OMP_TASKS MPIRUN
    cat <<EOF
#!/bin/bash
## Request name
#SBATCH --job-name=$1
## Number of tasks (=MPI processes) to use
#SBATCH --ntasks=$2
## Number of OpenMP threads
#SBATCH --cpus-per-task=$3

## computing project
#SBATCH -A wuu@cpu
## QoS for short jobs
#SBATCH --qos=qos_cpu-dev
## Elapsed time limit HH:MM:SS
#SBATCH --time=00:10:00

# do not use hyperthreading
#SBATCH --hint=nomultithread
# standard outputs
#SBATCH --output=DYNAMICO%j.out
#SBATCH --error=DYNAMICO%j.out

export OMP_NUM_THREADS=$3
# OpenMP binding
export OMP_PLACES=cores

# stack
export OMP_STACKSIZE=128M
ulimit -s unlimited

# move to submission directory
cd \${SLURM_SUBMIT_DIR}

# load the same modules as during compilation
source build/arch.env
module list

# cleanup execution directory and run
rm -rf gcm.log logs *.nc netcdf
date > gcm.log
ulimit -s unlimited
$4 >> gcm.log
date >> gcm.log

# move output files to netcdf subdir
mkdir -p netcdf
cp gcm.log *.def netcdf
mv *.nc netcdf

# keep log files, source code and executable for debug
mkdir -p logs
cp *.xml logs
cp -pr gcm.exe logs
cp -pr build/src logs
mv xios_client_*.err xios_client_*.out gcm.log logs

EOF
}

#------------------------------ post-processing job -----------------------------

function job_post_X64_IRENE() # LIST
{
    DEPLIST=afterany
    for ID in $* ; do
        DEPLIST="$DEPLIST:$ID"
    done
    cat <<EOF
#!/bin/bash
## Request name
#MSUB -r testing
#MSUB -n 1
## Elapsed time limit in seconds
#MSUB -T 600
#MSUB -q standard 
#MSUB -A $project
#MSUB -q skylake
#MSUB -m work
#MSUB -E "--dependency=$DEPLIST"
## Number of tasks (=MPI processes) to use
## Quality of Service required (long [3 days], normal [1 day], test [30 min])
#MSUB -Q normal

export OMP_NUM_THREADS=1
# this script is submitted from $ROOT/logs
cd \${BRIDGE_MSUB_PWD}/..

$(cat $ROOT/DYNAMICO/arch/arch-X64_IRENE.env)
module load python

bash/post.sh
EOF
}