#!/bin/bash
# Author: David Cugnet
# This script build a bunch of jobs, each one building limit.nc files, ozone files (optional)
# and initial state (optional, first year only) for a particular configuration.
# All those sequential light jobs can be submitted in parallel.
# The script checks used files are present, to avoid to submit nonfunctional jobs.
# Currently set up for CURIE, ADA and CICLAD machines, with corresponding paths for
# AMIP SST/SIC and ozone files.
# If you want to use other SST/SIC files, please change TYPE option. You will have to enter
# the path to the SST/SIC files manually, unless you include them in this script.
#
# Several types of sst/sic (variable V) files can be used, triggered by $TYPE keywork ; so far:
#                                   standard files name    variable name
# * AMIP for AMIP files             amipbc_${V}_1x1.nc     tosbcs/sicbcs
# * CPL  for coupled model outputs  cpl_atm_${V}.nc      SISUTESW/SIICECOV
# * HIST for atmosph model outputs  histmth_${V}.nc      tsol_oce/pourc_sic
# * DELK for ???                    sstk.nc/ci.nc            sstk/ci


#===============================================================================
#=== DETERMINE THE MACHINE NAME ================================================
#===============================================================================
if   [ ${HOSTNAME:0:3} = 'ada'   ]; then machine='ada';    work=$WORKDIR
elif [ ${HOSTNAME:0:5} = 'curie' ]; then machine='curie';  work=$CCCWORKDIR
elif [ ${HOSTNAME:0:6} = 'ciclad']; then machine='ciclad'; work=/data/$USER
else echo "Not set up for this machine yet, sorry."; exit; fi
export machine=$machine

#===============================================================================
#=== PARAMETERS YOU WILL PROBABLY NEED TO CHANGE ===============================
#===============================================================================
#=== GENERAL PARAMETERS
GENCI_ID='gen1168'                           #= GENCI project ID (for CURIE)
etat0=n                                      #= BUILD "start.nc" & "startphy.nc" (initial states) FILES (y/n)
limit=n                                      #= BUILD "limit.nc" (boundary conditions) FILES (y/n)
climoz=1                                     #= BUILD (1/2) OR NOT (0) climoz FILES USING:
                                             #=   1: "tro3"    2: "tro3" and "tro3_daylight"
#res=('96x95x39' '144x142x79' '280x280x79')   #= resolution(s)                         (at least one)
#cal='earth_360d earth_365d gregorian'        #= calendar(s): earth_366d / julian /... (at least one)
res=('96x95x39' '144x142x79')
cal='earth_360d'

#=== PARAMETERS RELATED TO SST/SIC
Ybs=1870; Yes=2008
STYPE='AMIP'                                  #= SST/SIC files type                      (single one)
#SVERS='2012 2014 20161020 20170419'          #= SST/SIC files versions, for AMIP only (at least one)
#LIMOUT='$work/LIMIT/$sstp/${rs}_${oatp}_$cl' #= SST/SIC output files storage folder
SVERS='20170419'
LIMOUT='$work/IGCM/ATM/LIMIT/$sstp/interpol/${rs}_${oatp}_$cl'

#=== PARAMETERS RELATED TO OZONE
Ybo=1850; Yeo=2014
#OTYPE=('HYBRIDE' 'UReading_2016')            #= ozone   files type                    (at least one)
#OVERS=('v2.clim' '2D_14Fields')              #= corresponding versions tags  (one each o3 file type)
#OZOOUT='$work/Ozone/$o3tp/$o3vs/${rs}'       #= ozone   output files storage folder
OTYPE=('UReading')                           #= ozone   files type                    (at least one)
OVERS=('historical.v20160711')               #= corresponding versions tags  (one each o3 file type)
OZOOUT='$work/IGCM/ATM/OZONE/$o3tp/$o3vs/interpol/${rs}' #= ozone   output files storage folder

#=== PARAMETERS RELATED TO INITIAL STATES
INIOUT='$work/IGCM/ATM/START/${rs}_${oatp}'  #= initial states storage folder (1st year only)

#===============================================================================
#=== PARAMETERS NOT SUPPOSED TO BE CHANGED OFTEN ===============================
#===============================================================================
#--- EXECUTABLE PATH AND NAME
ce0l="./ce0lseq.e"                           #= command to run ce0l executable in jobs
BINDIR=$PWD/bin                              #= folder containing ce0l executable(s)

#--- CONFIGURATION FILES
DEFDIR=$PWD/DefLists                         #= folder containing parameters files
                                             #= List of *.def files ; any change of name: after ":".
defslist='physiq.def config.def traceur.def  run.def' #conv_param.def wake_param.def
                                             #= keys to change. Syntax: <file>:<key>=<value>

#--- RUN COMMAND, LOCAL STORAGE SPACE, INPUT DATA FOLDER, NEEDED FILES (EXCLUDING SIC/SST/O3, NO ".nc")
case $machine in
  ada)    work=$WORKDIR;    DATAIN='/workgpfs/rech/psl/rpsl035/IGCM'    ; run="";;
  curie)  work=$CCCWORKDIR; DATAIN='/ccc/work/cont003/igcmg/igcmg/IGCM' ; run="ccc_mprun";;
  ciclad) work=/data/$USER; DATAIN='/prodigfs/ipslfs/igcmg/IGCM'        ; run="";;
esac
DATAIN_TMP=$work/IGCM

#--- CLIMATOLOGIES FOLDER AND CORRESPONDING NEEDED FILES
CLIDIR=$DATAIN/ATM
climslist="Albedo ECDYN:ECDYN ECDYN:ECPHY landiceref Relief Rugos"

#--- INPUT SST/SIC FILES AND VARIABLES NAMES + STANDARD NAMES KNOWN BY ce0l
#SSTSIN='$DATAIN/ATM/$sstp'
SSTSIN='$DATAIN/ATM/LIMIT/$sstp/original/360x180/BCS'

case $STYPE in
  AMIP) sstsic_in=('amipbc_sst_360x180_${Y}.nc' 'amipbc_sic_360x180_${Y}.nc')
        sstsic_ou=('amipbc_sst_1x1.nc'          'amipbc_sic_1x1.nc')
        sstsic_cl=('sst_bc_clim.nc'             'sic_bc_clim.nc') ;;
  CPL)  sstsic_in=('cpl_atm_sst_${Y}.nc' 'cpl_atm_sic_${Y}.nc')
        sstsic_ou=('cpl_atm_sst.nc'      'cpl_atm_sic.nc')
        sstsic_cl=('sst_cpl_clim.nc'     'sic_cpl_clim.nc') ;;
  HIST) sstsic_in=('histmth_sst_${Y}.nc' 'histmth_sic_${Y}.nc')
        sstsic_ou=('histmth_sst.nc'      'histmth_sic.nc')
        sstsic_cl=('sst_hist_clim.nc'    'sic_hist_clim.nc') ;;
  DELK) sstsic_in=('sstk_${Y}.nc' 'ci_${Y}.nc')
        sstsic_ou=('sstk.nc'      'ci.nc')
        sstsic_cl=('sstk_clim.nc' 'sick_clim.nc') ;;
esac

#--- INPUT OZONE FILES NAMES + STANDARD NAMES KNOW BY ce0l
#OZONIN='$DATAIN/ATM/Ozone/$o3tp/$o3vs'
OZONIN='$DATAIN_TMP/ATM/OZONE/$o3tp/$o3vs/original/144x96x66'
ozonefile='tro3_${Y}.nc'
ozonehybr='tro3_${Y}.new.nc'

#--- PARAMETERS RELATED TO o2a.nc
O2CDIR=$DATAIN/ATM/IPSLCM6

#===============================================================================
#=== FEW FUNCTIONS =============================================================
#===============================================================================
function addc {
  local s
  s="$1"; while [ ${#s} -lt $2 ]; do s="_$s"; done ; echo "$s"; unset s
}
#==================================================================================
function progress_bar {
# $1: iteration  $2: iterations number  $3: length of the bar
  local n bar
  bar=`echo "scale=1 ; (100*$1)/$2" | bc`
  bar="`addc "$bar" 5`% ["
  n=`echo "scale=0 ; ($3*$1)/$2" | bc`
  if [ $n -ne 0 ]; then bar="$bar"`eval "printf '='%.0s {1..$n}"`; fi
  n=$(($3-$n))
  if [ $n -ne 0 ]; then bar="$bar"`eval "printf '.'%.0s {1..$n}"`']\r'; fi
  echo -ne $bar; if [ $2 -eq $1 ]; then echo; fi
  unset n bar
}
#==================================================================================
function check_file {
# Purpose: Check if a file is present. Return in $method the way to get it:
#  1) untar for files from tar archives.
#  2) simple link for files on mounted disks.
#  3) mfget for files stored on ERGON.
#   $1=<archive name>:<input file name> (case 1)
#   $1=<input file name>                (cases 2,3)
#   Global variable used: $method (+ variables used to evaluate $1).
  local f=${1##*:} a="" fnam arch
  if [ "${1%%:*}" != "$1" ]; then a=${1%%:*}; fi
  fnam=$(eval echo $f); arch=$(eval echo $a)

  #=== CHECK WETHER ARGUMENTS ARE CORRECT IN CASE OF A *.tar ARCHIVE
  if [ "$arch" != "" ]; then
    #--- NOT A TAR ARCHIVE
    if [ "${arch##*.tar}" = "${arch}" ]; then
      echo "Argument provided is not a *.tar file: ${arch}"; return 1
    fi
    #--- TAR ARCHIVE UNFOUND
    if [ ! -f "${arch}" ]; then
      echo "Missing archive ${arch}"; return 1
    fi
    #--- FILE NOT IN THE ARCHIVE
    if [ ! `tar tf $arch $fnam 2> /dev/null` ]; then
      echo "Archive ${arch} does not contain a file named $fnam"; return 1
    fi
  fi

  #=== SEVERAL POSSIBILITIES FOR A PRESENT FILE
  if [ -f $fnam ]; then                      #=== REACHABLE FILE: SIMPLE LINK
    export method='ln -sf $f_in $f_ou' ; return 0
  elif [ "${arch}" != "" ]; then             #=== TAR ARCHIVE ; MOSTLY USEFULL FOR CURIE
    export method='tar xvf $arch $f_in --strip-components=$(grep -o "/" <<< "$f_in" | wc -l) ; mv ${f_in##*/} $f_ou'; return 0
  elif [ `mfls $fnam 2> /dev/null` ]; then   #=== STORED ON ERGON
    export method='mfget $f_in $f_ou' ; return 0
  fi

  #=== FILE NOT FOUND ; DISPLAY A MESSAGE IF NEEDED.
  if [ $# -eq 3 ]; then                      #=== NOT FOUND
    if [ $3 = '-v' ]; then echo "No way to deal with file described by argument $1"; fi
  fi
  return 1
}

#===============================================================================
function check_fyears {
# $1: Ystart  $2: Ystop  $3: files  $4: type
  echo ">>> CHECKING $4 FILES: ${3##*/}..."
  Yok=(); Yno=()
#-------------------------------------------------------------------------------
  for Y in $(eval echo {$1..$2}); do         #--- LOOP ON YEARS
#-------------------------------------------------------------------------------
    check_file $(Y=$Y; eval echo $3)
    if [ $? -eq 0 ]; then Yok=(${Yok[@]} $Y); else Yno=(${Yno[@]} $Y); fi
    progress_bar $((Y-$1+1)) $(($2-$1+1)) 50
#-------------------------------------------------------------------------------
  done                                       #--- YEARS
#-------------------------------------------------------------------------------
  if [ ${#Yok} -eq 0 ]; then echo "    No $4 files found: $3 $4"; return 1; fi
  nY=$((${#Yok[@]}-1)); Yi=$1; Yf=$2
  if [ $nY -ne $(($2-$1)) ]; then Yi=${Yok[0]}; Yf=${Yok[$nY]}
    echo "    Reduced interval for $4 files ${3##*/}: ${Yi}-${Yf} (instead of $1-$2)"
    i=0; while [ $i -le ${#Yno} ]; do
      if [[ ${Yno[$i]} -lt $Yi || ${Yno[$i]} -gt $Yf ]]; then unset Yno[$i]; fi; ((i++))
    done
  elif [ ${#Yno} -ne 0 ]; then
    echo "    Missing years for $4 files (${3##*/}): ${Yno[@]}"; return 1
  else
    echo "    Default interval for $4 files ${3##*/}: $1-$2"
  fi
  export Yi=$Yi Yf=$Yf method="$method"
  return 0
}

#===============================================================================
function change_key {
# Purpose: write in $3 the command to set the value of key ${1%%=*} to ${1##*=} in file $2.def
#    $1: <key=value>  $2: parameter file name (without .def suffix)  $3: job name
  if ! test $(grep ${1%%=*}= $2.def); then echo "  echo $1 >> ${2##*/}.def" >> $3
  else                     echo "  sed -i s%${1%%=*}=.*%${1}% ${2##*/}.def" >> $3 ; fi
  return 0
}

#===============================================================================
function header {
# Generate a job header ; $1=<job_name> ; $2=resolution
#    Used global paths: TMPDIR/SCRATCHDIR/USER, depending on the machine.
  local jobn=$(t=${1##*/}; eval echo ${t%.*}) suff tmp time
  suff=${jobn##*ob_}; tmp=CE0L_${suff}_$$

#-- Number of hours required (4 by default)
  time=4
  case $2 in
    96x95x39)   time=2 ;;
    144x142x79) time=4 ;;
    280x280x79) time=12 ;;
  esac
  case $machine in
    ada) tmp=\$TMPDIR/$tmp; time=$time:00:00
      cat > $1 << ______fin
# @ job_name = $jobn
# @ output   = \$(job_name).\$(jobid)
# @ error= \$(job_name).\$(jobid)
# @ job_type = serial
# @ as_limit = 20.0Gb
# @ wall_clock_limit = $time
# @ queue
______fin
    ;;
    curie) tmp=\$SCRATCHDIR/$tmp; time=$((3600*time))
      cat > $1 << ______fin
#!/bin/bash 
#MSUB -r $jobn
#MSUB -n 1
#MSUB -T $time
#MSUB -o $1.%I.o
#MSUB -e $1.%I.e
#MSUB -A $GENCI_ID
#MSUB -q large
______fin
    ;;
    ciclad) tmp=/data/$USER/$tmp
      cat > $1 << ______fin
#!/bin/bash
#PBS -qstd
#PBS -k oe
#PBS -j oe
#PBS -l mem=10gb
#PBS -l vmem=12gb
______fin
    ;;
  esac
  cat >> $1 << ______fin
  set -x
  tmp=$tmp ; if [ ! -d \$tmp ]; then mkdir \$tmp; fi; cd \$tmp
______fin
}


#===============================================================================
#=== MAIN PROGRAM
#===============================================================================

#=== CHECK EXECUTABLE ARE THERE FOR EACH RESOLUTION
mrs=""
#-------------------------------------------------------------------------------
for rs in ${res[@]}; do                      #--- LOOP ON RESOLUTIONS
#-------------------------------------------------------------------------------
  exe=ce0l_${rs}_phylmd_seq.e                #= name of used executable
  check_file $BINDIR/$exe
  if [ $? -ne 0 ]; then echo "Missing executable $BINDIR/$exe"; mrs="$mrs $rs"; fi
  for rrs in $mrs; do res=(${res[@]/$rrs}); done
#-------------------------------------------------------------------------------
done
#-------------------------------------------------------------------------------
if [ "$mrs" != "" ]; then echo "No executable available."; exit; fi

#=== IF SELECTED TYPE IS "AMIP", THEN USE SEVERAL VERSIONS FROM $SVERS VARIABLE.
TYPE=$STYPE;
if [ "$TYPE" = "AMIP" ]; then TYPE=""; for vr in $SVERS; do TYPE="$TYPE AMIP.v$vr"; done; fi


#=== CHECK THE SST/SIC FILES ARE THERE FOR EACH REQUIRED YEAR
itype=0; Ybsst=(); Yesst=(); Ybsic=(); Yesic=()
#-------------------------------------------------------------------------------
for sstp in $TYPE; do                        #--- LOOP ON SST/SIC TYPES
#-------------------------------------------------------------------------------
  SSTDIR=$(sstp=$sstp; DATAIN=$DATAIN;     eval echo $SSTSIN)
  if [ ! -d $SSTDIR ]; then
  SSTDIR=$(sstp=$sstp; DATAIN=$DATAIN_TMP; eval echo $SSTSIN)
  fi
  iv=0
  for V in sst sic; do
    check_fyears $Ybs $Yes $SSTDIR/${sstsic_in[$iv]} $sstp ; if [ $? -ne 0 ]; then exit; fi
    Ybsst[$itype]=$Yi; Yesst[$itype]=$Yf
    eval clim$V[$itype]=\"$(ls $SSTDIR/${sstsic_cl[$iv]} 2> /dev/null) $(ls $SSTDIR/$(Y=*_*_clim; eval echo ${sstsic_in[$iv]}) 2> /dev/null)\"
    ((iv++))
  done
  get_sstsic[$itype]=$method; ((itype++))
#-------------------------------------------------------------------------------
done                                         #--- TYPES
#-------------------------------------------------------------------------------

#=== CHECK THE OZONE FILES ARE THERE FOR EACH REQUIRED YEAR
if [ $climoz -ne 0 ]; then
itype=0; Ybozo=(); Yeozo=()
#-------------------------------------------------------------------------------
for o3tp in ${OTYPE[@]}; do                  #--- LOOP ON OZONE TYPES
#-------------------------------------------------------------------------------
  o3vs=${OVERS[$itype]}
  OZODIR=$(o3tp=$o3tp; o3vs=$o3vs; DATAIN=$DATAIN;     eval echo $OZONIN)
  if [ ! -d $OZODIR ]; then
  OZODIR=$(o3tp=$o3tp; o3vs=$o3vs; DATAIN=$DATAIN_TMP; eval echo $OZONIN)
  fi
  o3f=$ozonefile; if [ "$o3tp" = "HYBRIDE" ]; then o3f=$ozonehybr; fi
  check_fyears $Ybo $Yeo $OZODIR/$o3f $o3tp ; if [ $? -ne 0 ]; then exit; fi
  Ybozo[$itype]=$Yi; Yeozo[$itype]=$Yf
  eval climozo[$itype]=\"`ls $OZODIR/$(Y=*_*_clim; eval echo $o3f) 2> /dev/null`\"
  get_ozone[$itype]=$method; ((itype++))
#-------------------------------------------------------------------------------
done                                         #--- TYPES
#-------------------------------------------------------------------------------
fi

#iv=0; ttp=($TYPE)
#for V in sst sic; do
#  for k in {1..3}; do n=$k; ((n--))
#    echo type: ${ttp[$n]}   Yb: ${Ybsst[$n]}   Ye: ${Yesst[$n]}   clim$V: $(eval echo \${clim$V[$n]})
#  done; ((iv++))
#done
#exit

#=== JOBS CREATION
echo ">>> GENERATING JOBS..."
no2a=0
for rs in ${res[@]}; do
  n=$(ls $O2CDIR/*LMD${rs%x*}/o2a.nc 2> /dev/null|wc -l); no2a=$((no2a+n+1))
done
njobs=$(($(echo $TYPE|wc -w)*$(echo $cal|wc -w)*$no2a))
itype=0; ijob=0
#-------------------------------------------------------------------------------
for sstp in $TYPE; do                        #--- LOOP ON SST/SIC TYPES
#-------------------------------------------------------------------------------
  #=== CHECK WETHER OZONE FILES HAVE TO BE COMPUTED
  okO3='n'; if [[ $itype -lt ${#OTYPE[@]} && $climoz -ne 0 ]]; then okO3='y'; fi

  #--- DETERMINE PATH FOR SST/SIC FILES
  SSTDIR=$(sstp=$sstp; DATAIN=$DATAIN; eval echo $SSTSIN)
  if [ ! -d $SSTDIR ]; then
    SSTDIR=$(sstp=$sstp; DATAIN=$DATAIN_TMP; eval echo $SSTSIN)
  fi

  #--- DETERMINE PATH FOR OZONE FILES + MODIFICATION
  if [ $okO3 = 'y' ]; then o3tp=${OTYPE[$itype]}; o3vs=${OVERS[$itype]}
    OZODIR=$(o3tp=$o3tp; o3vs=$o3vs; DATAIN=$DATAIN;     eval echo $OZONIN)
    if [ ! -d $OZODIR ]; then
    OZODIR=$(o3tp=$o3tp; o3vs=$o3vs; DATAIN=$DATAIN_TMP; eval echo $OZONIN)
    fi
    o3f=$ozonefile; if [ "$o3tp" = "HYBRIDE" ]; then o3f=$ozonehybr; fi
  fi
#-------------------------------------------------------------------------------
  for rs in ${res[@]}; do                    #--- LOOP ON RESOLUTIONS
#-------------------------------------------------------------------------------
    hres=${rs%x*}
    exe=ce0l_${rs}_phylmd_seq.e              #= name of the executable used
    case ${rs##*x} in                        #= Additional *.def file (depends on resolution)
      39) DEFLIST="$defslist gcm.def_96x95x39_NPv3.1:gcm.def" ;;
      79) DEFLIST="$defslist gcm.def vert_L79.def:vert.def"   ;;
       *) DEFLIST="$defslist gcm.def" ;;
    esac
    export DEFLIST=$DEFLIST
    ical=0
#-------------------------------------------------------------------------------
    for cl in $cal; do                       #--- LOOP ON CALENDARS
#-------------------------------------------------------------------------------
      export cl=$cl
      if [ $okO3 = 'y' ]; then
        OZONOU=$(o3tp=$o3tp; o3vs=$o3vs; rs=$hres; eval echo $OZOOUT)
      fi
      io2a=0
#-------------------------------------------------------------------------------
      for o2a in default $(ls $O2CDIR/*LMD${hres}/o2a.nc 2> /dev/null); do
#-------------------------------------------------------------------------------
        okOzone=$okO3;  if [[ $ical -ne 0 || $io2a  -ne 0 ]]; then okOzone='n'; fi
        oketat0=$etat0; if [[ $ical -ne 0 || $itype -ne 0 ]]; then oketat0='n'; fi
        oatp=$(a=${o2a%xLMD*}; echo ${a##*/})
        job=$PWD/job_ce0l-${sstp}-${rs}-${cl##*_}-${oatp}.bash
        LIMTOU=$(sstp=$sstp; oatp=$oatp; rs=$hres; cl=${cl##*_}; eval echo $LIMOUT)
        header $job $rs
        cat >> $job << ______fin

  #=== SOME PATHS
  DEFDIR=$DEFDIR
  BINDIR=$BINDIR
  CLIDIR=$CLIDIR
  SSTDIR=$SSTDIR
  LIMTOU=$LIMTOU
  if [ ! -d \$LIMTOU ]; then mkdir -p \$LIMTOU; fi
______fin
        if [ $okOzone = 'y' ]; then
          cat >> $job << ______fin
  OZODIR=$OZODIR
  OZONOU=$OZONOU
  if [ ! -d \$OZONOU ]; then mkdir -p \$OZONOU; fi
______fin
        fi
        if [ $oketat0 = 'y' ]; then
          cat >> $job << ______fin
  INITOU=$(eval echo $INIOUT)
  if [ ! -d \$INITOU ]; then mkdir -p \$INITOU; fi
______fin
        fi
        echo ""                                         >> $job
        echo "  #=== GET AND ALTER CONFIGURATION FILES" >> $job
        for fi in $DEFLIST; do fo=.
          if [ $(grep -o ":" <<< "$fi" | wc -l) -ne 0 ]; then
            fo="${fi##*:}"; fi="${fi%:*}"
          fi
          check_file $DEFDIR/$fi
          if [ $? -ne 0 ]; then echo "Missing file $DEFDIR/$fi"; isOK=1; fi
          echo "  cp \$DEFDIR/$fi $fo"                  >> $job
        done
        change_key calend=${cl}  $DEFDIR/run               $job
        change_key ok_etat0=y    $DEFDIR/run               $job
        change_key ok_limit=y    $DEFDIR/run               $job
        change_key read_climoz=0 $DEFDIR/config            $job
        echo ""                                         >> $job
        echo "  #=== LINK THE MODEL"                    >> $job
        check_file $BINDIR/$exe
        if [ $? -ne 0 ]; then echo "Missing executable $BINDIR/$exe"; isOK=1; fi
        echo "  $(f_in=\$BINDIR/$exe; f_ou=$ce0l; eval echo $method)" >> $job
        echo ""                                         >> $job
        echo "  #=== LINK MISCELLANOUS DATA"            >> $job
        for clim in $climslist; do
          fi=$clim.nc; fo=.
          if [ $(grep -o ":" <<< "$clim" | wc -l) -ne 0 ]; then
            fo="${clim##*:}.nc"; fi="${clim%:*}.nc"
          fi
          check_file $CLIDIR/$fi
          if [ $? -ne 0 ]; then echo "Missing climatology $fi"; isOK=1; fi
          echo "  $(f_in=\$CLIDIR/$fi; f_ou=$fo; eval echo $method)" >> $job 
        done
        if [ $o2a != 'default' ]; then
          echo "  ln -sf $o2a o2a.nc"                   >> $job
        fi
        Yb1=${Ybsst[$itype]}; Ye1=${Yesst[$itype]}
        Yb2=$Yb1; Ye2=$Ye1; if [ $okOzone = 'y' ]; then Yb2=${Ybozo[$itype]}; Ye2=${Yeozo[$itype]}; fi
        Yb0=$((Yb1<Yb2?Yb1:Yb2)); Ye0=$((Ye1>Ye2?Ye1:Ye2))
        cat >> $job << ______fin

  #-----------------------------------------------------------------------------
  for Y in {$Yb0..$Ye0}; do                                   #--- LOOP ON YEARS
  #-----------------------------------------------------------------------------
    Y1=\$Y; Y1=\$((Y1<$Yb1?$Yb1:Y1)); Y1=\$((Y1>$Ye1?$Ye1:Y1))
    Y2=\$Y; Y2=\$((Y2<$Yb2?$Yb2:Y2)); Y2=\$((Y2>$Ye2?$Ye2:Y2))
    sed -i s%anneeref=.*%anneeref=\${Y}% run.def
______fin
        if [ $okOzone = 'y' ]; then
          fin=$o3f; fou=climoz.nc; fom=${fou%.*}_m.nc; fop=${fou%.*}_p.nc
          cat >> $job << ______fin

    #=== OZONE FILES
    read_climoz=$climoz; if [[ \$Y -lt $Yb2 || \$Y -gt $Ye2 ]]; then read_climoz=0; fi
    sed -i s%read_climoz=.*%read_climoz=\$read_climoz% config.def
    fin=\$(Y=\$Y2;       eval echo $fin)
    fip=\$(Y=\$((Y2+1)); eval echo $fin)
    fou=$fou; fom=$fom; fop=$fop
    if   [ \$Y -eq $Yb0 ]; then $(f_in=\$OZODIR/\$fin; f_ou=\$fou; eval echo ${get_ozone[$itype]}); fi
    if  [[ \$Y -ge $Yb2 && \$Y -le $Ye2 ]]; then
      if [ \$Y -ne $Yb2 ]; then mv \$fou \$fom; mv \$fop \$fou; fi
      if [ \$Y -ne $Ye2 ]; then $(f_in=\$OZODIR/\$fip; f_ou=\$fop; eval echo ${get_ozone[$itype]}); fi
    fi
    if [ \$Y -eq $((Ye2+1)) ]; then rm \$fom; fi
______fin
        fi
        iv=0
        for V in sst sic; do
          fin=${sstsic_in[$iv]}; fou=${sstsic_ou[$iv]}; fom=${fou%.*}_m.nc; fop=${fou%.*}_p.nc
          cat >> $job << ______fin

    #=== $V FILES
    fin=\$(Y=\$Y1;       eval echo $fin)
    fip=\$(Y=\$((Y1+1)); eval echo $fin)
    fou=$fou; fom=$fom; fop=$fop
    if   [ \$Y -eq $Yb0 ]; then $(f_in=\$SSTDIR/\$fin; f_ou=\$fou; eval echo ${get_sstsic[$itype]}); fi
    if  [[ \$Y -ge $Yb1 && \$Y -le $Ye1 ]]; then
      if [ \$Y -ne $Yb1 ]; then mv \$fou \$fom; mv \$fop \$fou; fi
      if [ \$Y -ne $Ye1 ]; then $(f_in=\$SSTDIR/\$fip; f_ou=\$fop; eval echo ${get_sstsic[$itype]}); fi
    fi
    if [ \$Y -eq $((Ye1+1)) ]; then rm \$fom; fi
______fin
          ((iv++))
        done
        cat >> $job << ______fin

    ls

    #=== RUN THE MODEL
    $run $ce0l

    #=== COPY OUTPUTS
______fin
        if [ $limit = 'y' ]; then Y1=${Ybsst[$itype]}; Y2=${Yesst[$itype]}
          echo "    if [[ \$Y -ge $Y1 && \$Y -le $Y2 ]]; then"         >> $job
          echo "      mv limit.nc \$LIMTOU/limit_\${Y}.nc"             >> $job
          echo "    fi"                                                >> $job
        fi
        if [ $okOzone = 'y' ]; then Y1=${Ybozo[$itype]}; Y2=${Yeozo[$itype]}
          echo "    if [[ \$Y -ge $Y1 && \$Y -le $Y2 ]]; then"         >> $job
          echo "      mv climoz_LMDZ.nc \$OZONOU/climoz_LMDZ_\${Y}.nc" >> $job
          echo "    fi"                                                >> $job
        fi
        if [ $oketat0 = 'y' ]; then
          echo "    if [ \$Y -eq $Yb1 ]; then"                         >> $job
          echo "      mv start.nc \$INITOU/start.nc"                   >> $job
          echo "      mv startphy.nc \$INITOU/startphy.nc"             >> $job
          echo "    fi"                                                >> $job
        fi
        cat >> $job << ______fin
  #-----------------------------------------------------------------------------
  done
  #-----------------------------------------------------------------------------

  #=== COMPUTE CLIMATOLOGIES (IF SOME CORRESPONDING FILES ARE AVAILABLE)
______fin
        #--- SIC/SST climatologies for current combination (itype)
        csic=(${climsic[$itype]})
        csst=(${climsst[$itype]}); ns=${#csst[@]}; nmax=$ns
        #--- Same for oeone if required
        if [ $okOzone = 'y' ]; then fou=climoz.nc
          cozo=(${climozo[$itype]}); no=${#cozo[@]}; nmax=$((ns>no?ns:no))
          echo "  rm -f ${fou%.*}_m.nc ${fou%.*}_p.nc"                 >> $job
        fi
        #--- Get rid of the adjacent files (previous/next months)
        if [ $nmax -gt 0 ]; then iv=0
          for V in sst sic; do fou=${sstsic_ou[$iv]}; ((iv++))
            echo "  rm -f ${fou%.*}_m.nc ${fou%.*}_p.nc"               >> $job
          done
        fi
        for iclim in $(eval echo {1..$nmax}); do   #=== LOOP ON CLIMATOLOGIES
          if [ $iclim -le $ns ]; then isst=$((iclim-1))
            iv=0
            for V in sst sic; do
              #--- Get SST/SIC files names a,nd link/copy them
              eval fin=\${c$V[$isst]}; fou=${sstsic_ou[$iv]}
              echo "  $(f_in=\$SSTDIR/${fin##*/}; f_ou=$fou; eval echo ${get_sstsic[$itype]})" >> $job
              #--- Build the output file name
              m=${sstsic_in[$iv]}; p=${m%%'${Y}'*}; n=_clim${m##*'${Y}'}
              Yspan=_$(p=$p ; n=$n ; t=${fin##*$p}; t=${t%%$n*}; echo ${t##*/})
              if ! [[ ${Yspan:1:4} =~ ^-?[0-9]+$ && ${Yspan:6:4} =~ ^-?[0-9]+$ ]]; then Yspan=""; fi
              lim_ou=limit${Yspan}_clim.nc
              ((iv++))
            done
          fi
          if [[ $iclim -le $no && $okOzone = 'y' ]]; then iozo=$((iclim-1))
            eval fin=\${cozo[$iozo]}; fou=climoz.nc
            echo "  sed -i s%read_climoz=.*%read_climoz=$climoz% config.def"                 >> $job
            echo "  $(f_in=\$OZODIR/${fin##*/}; f_ou=$fou; eval echo ${get_ozone[$itype]})"  >> $job
            p=${o3f%%'${Y}'*}; n=_clim${o3f##*'${Y}'}
            Yspan=_$(p=$p ; n=$n ; t=${fin##*$p}; t=${t%%$n*}; echo ${t##*/})
            if ! [[ ${Yspan:1:4} =~ ^-?[0-9]+$ && ${Yspan:6:4} =~ ^-?[0-9]+$ ]]; then Yspan=""; fi
            ozo_ou=climoz_LMDZ${Yspan}_clim.nc
          fi
          echo "  sed -i s%anneeref=.*%anneeref=1980% run.def"       >> $job
          echo "  $run $ce0l"                                        >> $job
          if [ $iclim -le $ns ]; then
            if [ "$cl" = "gregorian" ]; then
              echo "  mv limit.nc \$LIMTOU/${lim_ou%.nc*}_leap.nc"   >> $job
              echo "  sed -i s%anneeref=.*%anneeref=1982% run.def"   >> $job
              echo "  $run $ce0l"                                    >> $job
              echo "  mv limit.nc \$LIMTOU/${lim_ou%.nc*}_noleap.nc" >> $job
            else
              echo "  mv limit.nc \$LIMTOU/$lim_ou"                  >> $job
            fi
          fi
          if [[ $iclim -le $no && $okOzone = 'y' ]]; then
            echo "  mv climoz_LMDZ.nc \$OZONOU/$ozo_ou"              >> $job
          fi
        done
        echo   "  cd .. ; rm -rf \$tmp"                              >> $job
        ((io2a++))
        ((ijob++)); progress_bar $ijob $njobs 50
#-------------------------------------------------------------------------------
      done; ((ical++))
#-------------------------------------------------------------------------------
    done
#-------------------------------------------------------------------------------
  done ;    ((itype++))
#-------------------------------------------------------------------------------
done
#-------------------------------------------------------------------------------
echo

