#!/bin/bash

# This script provides site-specific functions
# called by the site-independent scripts
# create_runs.sh and submit.sh

#----------------------------- split_XXX ----------------------------

function split_mpi_32() {
    setvar nbp 32 nsplit_i 1 nsplit_j 2
}
function split_serial_40() {
    setvar nbp 40 nsplit_i 1 nsplit_j 2
}
function split_mpi_40() {
    setvar nbp 40 nsplit_i 1 nsplit_j 2
}
function split_mpi_80() {
    setvar nbp 80 nsplit_i 1 nsplit_j 2
}
function split_mpi_omp_40() {
    setvar nbp 40 nsplit_i 1 nsplit_j 2 omp_level_size 1
}

#----------------- job submission ------------------

function submit_job_X64_IRENE() # $1=script $* = SLURM OPTIONS
{
    TMP=$(mktemp)
    ccc_msub $1 > $TMP
    cat $TMP 1>&2
    cat $TMP | awk '{print $NF }'
    rm -f $TMP
}

#------------------------------ job_XXX -----------------------------

# Serial jobs
function job_serial() { # EXP_NAME
    job_generic $1 1 1 "./gcm.exe"
}
function job_serial_40() { # EXP_NAME
    job_serial $1
}

# MPI jobs
function job_mpi() { #EXP_NAME NB_MPI
    job_generic $* "srun ./gcm.exe"
}
function job_mpi_32() { # EXP_NAME
    job_mpi $1 1 4 4
}
function job_mpi_40() { # EXP_NAME
    job_mpi $1 1 4 4
}
function job_mpi_80() { # EXP_NAME
    job_mpi $1 1 4 4
}

# MPI-OMP jobs
function job_mpi_omp() { # EXP_NAME NODES GPUS GPU_PER_NODE
    job_generic $* "srun ./gcm.exe"
}

function job_mpi_omp_40() { # EXP_NAME
    job_mpi_omp $1 1 4 4
}

# Generic
function job_generic() { # EXP_NAME NODES GPUS GPU_PER_NODE CMD
    cat <<EOF
#!/bin/bash
## Request name
#SBATCH --job-name=$1

#SBATCH --partition=gpu_p1          # partition GPU choisie
#SBATCH --nodes=$2                  # nombre de noeud
#SBATCH --ntasks=$3                 # nombre de tache MPI (= nombre de GPU ici)
#SBATCH --ntasks-per-node=$4        # nombre de tache MPI par noeud (= nombre de GPU ici)
#SBATCH --gres=gpu:$4               # nombre de GPU par noeud
#SBATCH --cpus-per-task=1           # nombre de coeurs CPU par tache

## computing project
#SBATCH -A wuu@gpu
## Elapsed time limit HH:MM:SS
#SBATCH --time=00:10:00

# do not use hyperthreading
#SBATCH --hint=nomultithread
# standard outputs
#SBATCH --output=DYNAMICO%j.out
#SBATCH --error=DYNAMICO%j.out

export OMP_NUM_THREADS=1
# OpenMP binding
export OMP_PLACES=cores

# stack
export OMP_STACKSIZE=128M
ulimit -s unlimited

# move to submission directory
cd \${SLURM_SUBMIT_DIR}

# load the same modules as during compilation
source build/arch.env
module list

# cleanup execution directory and run
rm -rf gcm.log logs *.nc netcdf
date > gcm.log
ulimit -s unlimited
$5 >> gcm.log
date >> gcm.log

# move output files to netcdf subdir
mkdir -p netcdf
cp gcm.log *.def netcdf
mv *.nc netcdf

# keep log files, source code and executable for debug
mkdir -p logs
cp *.xml logs
cp -pr gcm.exe logs
cp -pr build/src logs
mv xios_client_*.err xios_client_*.out gcm.log logs

EOF
}

#------------------------------ post-processing job -----------------------------

function job_post_X64_IRENE() # LIST
{
    DEPLIST=afterany
    for ID in $* ; do
	DEPLIST="$DEPLIST:$ID"
    done
    cat <<EOF
#!/bin/bash
## Request name
#MSUB -r testing
#MSUB -n 1
## Elapsed time limit in seconds
#MSUB -T 600
#MSUB -q standard 
#MSUB -A $project
#MSUB -q skylake
#MSUB -m work
#MSUB -E "--dependency=$DEPLIST"
## Number of tasks (=MPI processes) to use
## Quality of Service required (long [3 days], normal [1 day], test [30 min])
#MSUB -Q normal

export OMP_NUM_THREADS=1
# this script is submitted from $ROOT/logs
cd \${BRIDGE_MSUB_PWD}/..

$(cat $ROOT/DYNAMICO/arch/arch-X64_IRENE.env)
module load python

bash/post.sh
EOF
}