Index: /branches/libIGCM_concurrent/AA_job
===================================================================
--- /branches/libIGCM_concurrent/AA_job	(revision 1628)
+++ /branches/libIGCM_concurrent/AA_job	(revision 1629)
@@ -55,4 +55,5 @@
 #-Q- irene-amd #MSUB -c ::openMPthreads::  # Number of openMP threads. To specify only for SMPD
 #-Q- irene-amd #MSUB -x                    # exclusive node. To specify only for MPMD together with the one below
+#-Q- irene-amd #MSUB -N ::NodeNumber::
 #-Q- irene-amd #MSUB -T ::WallTime::              # Wall clock limit (seconds)
 #-Q- irene-amd #MSUB -A ::default_project::
Index: /branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene-amd.ksh
===================================================================
--- /branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene-amd.ksh	(revision 1628)
+++ /branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene-amd.ksh	(revision 1629)
@@ -1172,64 +1172,12 @@
   typeset file
   file=$1
-
-   # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
-   # Compute the number of core to add to force the use of a new node for xios servers
-
-   if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
-       (( IOS_OK_DEDICATED = 0 ))
-   fi
-
-
-   if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
-      (( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
-      (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
-      (( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
-      (( coreNumber_final = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
-   else
-      (( coreNumber_final = coreNumber ))
-   fi
-
-  if [ ${executionType} -eq 1 ] ; then
-    # MPMD + MPI
+    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
+	(( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
+	[ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
     sed -e "/::openMPthreads::/d"                  \
-        -e "s/::JobNumProcTot::/${coreNumber_final}/"    \
+        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
+	  -e "s/::NodeNumber::/${nodeNumber}/"    \
+          -e "/--cpu_bind=none/d"                    \
       ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 2 ] ; then
-    # MPMD + MPI + OMP :ccc_mprun
-    sed -e "/::openMPthreads::/d"                  \
-        -e "s/::JobNumProcTot::/${coreNumber_final}/"    \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 3 ] ; then
-    # SPMD + MPI/OMP
-    sed -e "s/::openMPthreads::/${openMPthreads}/" \
-        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
-        -e "/#MSUB -x/d"                           \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 4 ] ; then
-    # SPMD + MPI only
-    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
-        -e "/::openMPthreads::/d"                  \
-        -e "/#MSUB -x/d"                           \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 5 ] ; then
-    # SPMD + OMP only
-    sed -e "s/::openMPthreads::/${openMPthreads}/" \
-        -e "/::JobNumProcTot::/d"                  \
-        -e "/#MSUB -x/d"                           \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 6 ] ; then
-    # SEQUENTIAL THEN
-    sed -e "s/::JobNumProcTot::/1/"                \
-        -e "/::openMPthreads::/d"                  \
-        -e "/#MSUB -x/d"                           \
-      ${file} > ${file}.tmp
-
-  fi
-
   IGCM_sys_Mv ${file}.tmp ${file}
 
@@ -1263,4 +1211,279 @@
     echo "IGCM_sys_build_execution_scripts " $@
   fi
+  if ( [ "X${config_UserChoices_ExecutionMode}" = "Xslurm" ] ) ; then
+
+      EXECUTION="/usr/bin/time srun "
+
+        if ( ${OK_PARA_MPMD} ) ; then
+
+    # MPMD mode
+    # 1 MPI only : executionType=1
+    # 2 MPI/OpenMP : executionType=2
+
+    if [ -f run_file ] ; then
+      IGCM_sys_Rm -f run_file
+    fi
+    if [ -f RUNDIR_2/run_file ] ; then
+	IGCM_sys_Rm -f RUNDIR_2/run_file
+    fi
+    touch run_file
+
+    # case 1 : Only MPI (MPMD)
+    # if  ( ! ${OK_PARA_OMP} ) ; then
+    #   first_slurm_comp=0
+    #   # Build run_file
+    #   current_core=0
+    #   # First loop on the components for the coupler ie oasis (only if oasis3)
+    #   # the coupler ie oasis3 must be the first one
+    #   for comp in ${config_ListOfComponents[*]} ; do
+    # 	  number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+    # 	  if [ X${number_rundir} != X ] ; then
+    # 	      [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+    # 	      cd RUNDIR_${number_rundir} 
+    # 	      if [ ${first_slurm_comp} = "0" ] ; then
+    # 		  current_core=0 ; first_slurm_comp=1 ;
+    # 	      fi
+    # 	  fi
+    #     eval ExeNameIn=\${config_Executable_${comp}[0]}
+    #     eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+    #     # Only if the component has an executable
+    #     if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
+
+    #       eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+    #       (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
+    #       echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
+    #       (( current_core = ${end_core} + 1 ))
+    #     fi
+    # 	if [ X${number_rundir} != X ] ; then 
+    # 	    cd $RUN_DIR ;
+    # 	fi
+    #   done
+
+    #   if [ X${number_rundir} != X ] ; then
+    # 	  echo "cd $RUN_DIR ; /usr/bin/time srun --ntasks=${config_UserChoices_NbMPItasks_run1} --multi-prog ./run_file > out_execution  2>&1 &" > EXECUTION.exe
+    # 	  echo "cd RUNDIR_2 ; /usr/bin/time srun --ntasks=${config_UserChoices_NbMPItasks_run2} --multi-prog ./run_file > out_execution  2>&1 &" >> EXECUTION.exe
+    # 	  echo "wait" >> EXECUTION.exe
+    # 	  chmod u+x EXECUTION.exe 
+    #   else
+    # 	  EXECUTION="/usr/bin/time srun --multi-prog ./run_file"
+    #   fi
+
+    #   IGCM_sys_Chmod u+x run_file
+    #   if ( $DEBUG_sys ) ; then
+    #     echo "run_file contains : "
+    #     cat run_file
+    #   fi
+
+    # else 
+
+    # 2 MPI/OpenMP : executionType=2
+
+    # MPI-OpenMP (MPMD)
+    # export SLURM_HOSTFILE=./hostlist 
+    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
+    # example of  hostlist file :
+    # r3i3n33
+    # r3i3n33
+    # ...
+    # example of run_file :
+    # 0-70 ./prog_lmdz.x.sh %o %t
+    # 71-430 ./prog_opa.xx.sh %o %t
+    # 431-431 ./prog_xios.x.sh %o %t
+    # examples of prog_file :
+    # prog_lmdz.x.sh :
+    # (( init = 0 + $1 ))
+    # (( index = init * 10 ))
+    # (( slot = index % 40 ))
+    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
+    # that will become
+    # taskset -c 0-9 ./script_lmdz.x.ksh
+    # ...
+    # with script_lmdz.x.ksh
+    # export OMP_STACKSIZE=3g
+    # export OMP_PLACES=cores
+    # export OMP_NUM_THREADS=10
+    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
+
+    #  Hosts treatment
+    _bkIFS=$IFS;
+	IFS=$'\n'; set -f
+	listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
+	IFS=$_bkIFS; set +f
+	rm -f hostlist
+
+     # Loop on the components to build run_file and script_exec files
+	rank=0
+	current_core=0
+        current_core_mpi=0
+	current_core_tmp=0
+        current_core_mpi_tmp=0
+	first_slurm_comp=0
+
+      for comp in ${config_ListOfComponents[*]} ; do
+
+	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	  if [ X${number_rundir} != X ] ; then
+	      [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	      cd RUNDIR_${number_rundir} 
+	      if [ ${first_slurm_comp} = "0" ] ; then
+		  (( NbMPItasks_run1 = current_core_mpi_tmp ))
+		  ### On change de noeud pour le prochain srun
+		  if [ $(( $current_core % $NB_CORE_PER_NODE )) -ne 0 ] ; then
+		      (( current_core = current_core + NB_CORE_PER_NODE - current_core % NB_CORE_PER_NODE ))
+		  fi
+		  current_core_tmp=0 ; current_core_mpi_tmp=0 ; first_slurm_comp=1 ;
+	      fi
+	  fi
+
+	eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # Not possible if oasis has an executable (i.e old version of oasis3)
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
+          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
+          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
+          IGCM_debug_Verif_Exit
+        fi
+
+        # Only if we really have an executable for the component :
+        if [ "X${ExeNameOut}" != X\"\" ] ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
+     
+
+          # Build script files
+
+          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+          echo ""  >> script_${ExeNameOut}.ksh
+          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
+            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
+          fi
+
+          # to have out/err per process on different files
+          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
+
+          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+        # Build run_file
+        # Only if the component has an executable
+        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          (( end_core = ${current_core_mpi_tmp} + ${comp_proc_mpi_loc} - 1 ))
+          echo "${current_core_mpi_tmp}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
+          (( current_core_mpi_tmp = ${end_core} + 1 ))
+        fi
+
+          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
+              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
+          else
+              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
+          fi
+
+          # Build configuration file
+ 
+	      echo "#!/bin/sh" > prog_${ExeNameOut}.sh
+          echo "(( init = $current_core_tmp + \$1 ))" >> prog_${ExeNameOut}.sh
+          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
+          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
+	      echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
+          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
+
+          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
+
+        # Build hostlist file
+
+          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+             (( index_host = current_core / NB_CORE_PER_NODE ))
+             host_value=${listnodes[${index_host}]}
+             echo "$host_value" >> hostlist
+	     if [ ${DRYRUN_DEBUG} = 4 ] ; then
+		 echo "node_${index_host}_X" >> hostlist_template
+	     fi
+             (( current_core = current_core + offset_comp_proc_loc ))
+             (( current_core_tmp = current_core_tmp + offset_comp_proc_loc ))
+          done
+        fi
+	if [ X${number_rundir} != X ] ; then 
+	    cd $RUN_DIR ;
+	fi
+      done
+
+      ## variable added to stop after 60s instead of 600s by default. 
+      ## This is used when no error comes from executables and when something stopped an executable without notice. 
+      export SLURM_WAIT=60
+
+      if [ X${number_rundir} != X ] ; then
+	  echo "cd $RUN_DIR ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run1} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" > EXECUTION.exe
+	  echo "cd RUNDIR_2 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${current_core_mpi_tmp} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" >> EXECUTION.exe
+	  echo "wait" >> EXECUTION.exe
+	  chmod u+x EXECUTION.exe 
+      else
+	  EXECUTION="/usr/bin/time srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
+      fi
+      IGCM_sys_Chmod u+x run_file
+      if ( $DEBUG_sys ) ; then
+        echo "run_file contains : "
+        cat run_file
+      fi
+
+#    fi # if ${OK_PARA_MPMD}
+
+  else
+  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
+
+    for comp in ${config_ListOfComponents[*]} ; do
+
+      # Only if we really have an executable for the component :
+      eval ExeNameOut=\${config_Executable_${comp}[1]}
+      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
+
+        # Build script files
+
+        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+        echo ""  >> script_${ExeNameOut}.ksh
+        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+        if ( ${OK_PARA_OMP} ) ; then
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+	    # Check if the number of threads is correct
+	    case ${comp_proc_omp_loc} in
+		2|4|5|10|20)
+		    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+		    ;;
+		*)
+		    IGCM_debug_Exit "ERROR with OMP parameters !"
+		    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+		    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
+		    IGCM_debug_Verif_Exit
+		    ;;
+	    esac 
+            echo ""  >> script_${ExeNameOut}.ksh
+            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
+            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+        fi
+
+        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+        
+        # To have out/err per process on different files
+        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
+        EXECUTION="/usr/bin/time srun ./script_${ExeNameOut}.ksh"
+
+        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
+        cat script_${ExeNameOut}.ksh
+
+      fi
+
+    done
+
+  fi # ${OK_PARA_MPMD}
+      
+  else
 
   EXECUTION=${HOST_MPIRUN_COMMAND}
@@ -1634,4 +1857,5 @@
   fi # ${OK_PARA_MPMD}
 
+  fi
   IGCM_debug_Print 1 "sys Irene-amd : execution command is "
   IGCM_debug_Print 1 "$EXECUTION"
