#!/bin/bash
######################
## IRENE-AMD   TGCC/CEA ##
######################
#MSUB -r IPSLCM6.5_ENSEMBLES        # Job Name
#MSUB -o Script_Output_IPSLCM6.5_ENSEMBLES    # standard output
#MSUB -e Script_Output_IPSLCM6.5_ENSEMBLES    # error output
#MSUB -eo
#MSUB -n 369  # Number of MPI tasks (SPMD case) or cores (MPMD case)
#MSUB -x                    # exclusive node. To specify only for MPMD together with the one below
#MSUB -T 1800              # Wall clock limit (seconds)
#MSUB -Q test
#MSUB -A gen0826
#MSUB -q rome
#MSUB -X
#MSUB -m store,work,scratch

cd ${BRIDGE_MSUB_PWD}


module purge
module load intel/19.0.5.281
module load feature/openmpi/net/ib/ucx-verbs
module load flavor/openmpi/vanilla mpi/openmpi/4.0.5
module load flavor/hdf5/parallel
module load netcdf-fortran/4.4.4
module load hdf5/1.8.20
module load feature/bridge/heterogenous_mpmd
module load nco/4.9.1
module load cdo/1.9.5
module load c++/gnu/7.3.0
module load c/gnu/7.3.0
module load arm-forge/20.2.1


pool_number=5
pool_size=2

whole_ensemble_size=$((${pool_number}*${pool_size}))

rm -f xios.x ; cp ../bin/xios_server_prod.exe xios.x
rm -f lmdz.x ; cp ../bin/gcm_96x95x59_prod.e lmdz.x
rm -f opa.xx ; cp ../bin/opa_prod.exe opa.xx
rm -rf XIOS
mkdir XIOS

for (( pool_rank=0 ; pool_rank < $(($pool_number)) ; pool_rank++ ))
do
  ./create_runfile.sh $pool_size $pool_rank

  for (( pool_member_rank=0; pool_member_rank<$(($pool_size+1)); pool_member_rank++ ))
  do

    whole_ensemble_rank=$(($pool_size*$pool_rank+$pool_member_rank))
    if (( pool_member_rank < $pool_size ))
    then
      rm -rf MEMBER_$whole_ensemble_rank
      mkdir MEMBER_$whole_ensemble_rank
      cd MEMBER_$whole_ensemble_rank
    else
      cd XIOS
    fi
   
    ln -f -s ../ENSEMBLE_TEMPLATE/* .
    cp ../oasis_namcouple_body ../oasis_namcouple_epilog ../oasis_namcouple_header .
    ../create_ensemble_def $whole_ensemble_size $pool_size $pool_rank $pool_member_rank
    ../create_oasis_namecoupl.sh $pool_size $pool_rank
    ../create_oasis_restart.sh $pool_size $pool_rank 
    ../create_iodef_xml.sh $pool_size 
    cd ..
  done 
  #export DDT=TRUE
  /usr/bin/time ccc_mprun -E-K1 -f ./run_file
done


exit