Index: /trunk/libIGCM/ins_job
===================================================================
--- /trunk/libIGCM/ins_job	(revision 1304)
+++ /trunk/libIGCM/ins_job	(revision 1305)
@@ -27,5 +27,6 @@
   or on curie/TGCC :
   ${b_n} [-h] [-v] [-e] [-p project] [-c number of cores]
-
+  or on ada /IDRIS
+  ${b_n} [-h] [-v] [-e] [-m MPI environment]
 Options :
   -h                  : help
@@ -35,4 +36,6 @@
   -p project          : add default project on curie
   -c number of cores  : add default number of cores for postprocessing on curie 1-16 or 1-128
+on ada only :
+  - m MPI environment : add default MPI environment (Intel MPI or IBM MPI)
 "
 }
@@ -74,7 +77,7 @@
 x_e=false;
 x_p=false;
-x_q=false;
 x_c=false;
-while getopts :hvec:p:q: V ; do
+x_m=false;
+while getopts :hvec:p:m: V ; do
   case $V in
   (h)  ins_job_Usage; exit 0;;
@@ -85,4 +88,6 @@
   (c)  x_c=true
        ProjectCore=${OPTARG} ;;
+  (m)  x_m=true
+       MPIEnvironment=${OPTARG} ;;
   (:)  echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
        exit 2;;
@@ -190,7 +195,27 @@
   echo ProjectCore is ${ProjectCore}
   #- ProjectCore is set (option or answer)
+# let check minimum/maximum value 1/${ProjectCoreMax}
+
+elif [ X"${SYSTEM}" == "Xada" ] ; then
+    if ( ! ${x_m} ) ; then
+	MPIEnvironment=IBM 
+#
+# Commented : waiting for full validation of Intel MPI Environment
+#	answerOK=false   	
+#       while ( ! ${answerOK} ) ; do
+#	    answer=""
+#	    print - "Hit Enter or give MPI Environement (default is ${MPIEnvironment}), possible MPI environments are IBM (MPI IBM) and Intel (MPI Intel) :"
+#	    read answer
+#	    [ "X${answer}" == "X" ] || [ "X${answer}" == "XIBM" ] || [ "X${answer}" == "XIntel" ] && answerOK=true
+#	done
+
+#	if [ "X${answer}" != "X" ] ; then
+#	    MPIEnvironment=${answer}
+#	fi
+    fi # if ( ! ${x_pm} )
+#  echo MPIEnvironment is ${MPIEnvironment}
 
 else
-  echo No option for Header regarding ProjectID, ProjectNode and ProjectCore because we are not on curie
+  echo No option for Header regarding ProjectID, ProjectNode, ProjectCore or MPIEnvironment because we are not on curie or ada
 fi # if [ X"${SYSTEM}" == "Xcurie" ]
 
Index: /trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh
===================================================================
--- /trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh	(revision 1304)
+++ /trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh	(revision 1305)
@@ -93,5 +93,10 @@
 #====================================================
 source /smplocal/pub/Modules/default/init/ksh
-. /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
+if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
+  . /smphome/rech/psl/rpsl035/.atlas_env_ada_intel_2013_0_bash > /dev/null 2>&1
+else
+  module load intel/2016.2 > /dev/null 2>&1
+  . /smphome/rech/psl/rpsl035/.atlas_env_ada_bash > /dev/null 2>&1
+fi
 [ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
 [ ! X${TaskType} = Xchecking ] && module list
@@ -136,6 +141,9 @@
 #====================================================
 #- HOST_MPIRUN_COMMAND
-typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
-
+if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
+  typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time poe"}
+else
+  typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time mpirun"}
+fi
 #====================================================
 #- Max number of arguments passed to nco operator or demigration command
@@ -145,4 +153,8 @@
 #- set PackDefault to true on ada
 PackDefault=true
+
+#====================================================
+#- Number of cores per node
+core_per_node=32
 
 #====================================================
@@ -401,6 +413,6 @@
   [ ${#@} = 2 ] &&  REP_FOR_JOB=${2}
   sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Output}:" \
-      -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
-      $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
+    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Output}:"   \
+    $1 > ${REP_FOR_JOB}/JOB_FOR_IGCM
   cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
 
@@ -433,6 +445,6 @@
 
   sed -e "s:\# \@ output *= .*:\# \@ output = ${Script_Post_Output}.out:" \
-      -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
-      ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
+    -e "s:\# \@ error *= .*:\# \@ error = ${Script_Post_Output}.out:"   \
+    ${libIGCM_POST}/$1.job > ${REP_FOR_JOB}/JOB_FOR_IGCM_$$
 
   cd $REP_FOR_JOB ; /usr/bin/llsubmit JOB_FOR_IGCM_$$ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1 ; status=$? ; cd - ;
@@ -858,4 +870,25 @@
 function IGCM_sys_activ_variables {
   IGCM_debug_PushStack "IGCM_sys_activ_variables"
+  if ( [ ! "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] && [ ! "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
+    # Temporary modifications for Intel MPI Environment
+    # To use DAPL Provider instead of OFA for inter-node communications
+    # and shm for intra-node communications
+    export I_MPI_FABRICS=shm:dapl
+    # To use UD protocol instead of RC (default with DAPL)
+    export I_MPI_DAPL_UD=1
+    # Some optimizations
+    export DAPL_UCM_REP_TIME=2000 # REQUEST timer, waiting for REPLY in millisecs
+    export DAPL_UCM_RTU_TIME=2000 # REPLY timer, waiting for RTU in millisecs
+    export DAPL_UCM_CQ_SIZE=2000 # CM completion queue
+    export DAPL_UCM_QP_SIZE=2000 # CM message queue
+    export DAPL_UCM_RETRY=7 # REQUEST and REPLY retries
+    export DAPL_ACK_RETRY=7 # IB RC Ack retry count
+    export DAPL_ACK_TIMER=20 # IB RC Ack retry timer
+    # To enable RDMA
+    export I_MPI_DAPL_UD_RDMA_MIXED=enable
+    # To use DAPL 2.1.7
+    export DAT_OVERRIDE=/smplocal/pub/DAPL/2.1.7/etc/dat.conf
+    export LD_LIBRARY_PATH=/smplocal/pub/DAPL/2.1.7/lib:${LD_LIBRARY_PATH}
+  fi
   if ( $DEBUG_sys ) ; then
     echo "IGCM_sys_activ_variables"
@@ -928,25 +961,57 @@
   if [ ${executionType} -eq 1 ] ; then
     # MPMD + MPI
-    sed -e "/::openMPthreads::/d"                  \
+    if [ X${MPIEnvironment} = XIBM ] ; then
+      sed -e "/::openMPthreads::/d"                  \
         -e "s/::JobNumProcTot::/${coreNumber}/"    \
-      ${file} > ${file}.tmp
+        ${file} > ${file}.tmp
+    else
+      sed -e "/::openMPthreads::/d"                  \
+        -e "s/@ job_type = parallel/@ job_type = mpich/" \
+        -e "s/::JobNumProcTot::/${coreNumber}/"    \
+        ${file} > ${file}.tmp
+    fi
 
   elif [ ${executionType} -eq 2 ] ; then
     # MPMD + MPI + OMP
-    sed -e "s/::openMPthreads::/${openMPthreads}/" \
+    if [ X${MPIEnvironment} = XIBM ] ; then
+      sed -e "s/::openMPthreads::/${openMPthreads}/" \
         -e "s/::JobNumProcTot::/${mpiTasks}/"    \
-      ${file} > ${file}.tmp
+        ${file} > ${file}.tmp
+    else
+      (( nodeNumber = coreNumber / core_per_node ))
+      [ $(( ${coreNumber} % ${core_per_node} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
+      sed -e "/::openMPthreads::/d"                  \
+        -e "s/@ job_type = parallel/@ job_type = mpich/" \
+        -e "s/@ total_tasks = ::JobNumProcTot::/@ node = ${nodeNumber} /"    \
+        -e "/@ as_limit = 3.5gb/d"      \
+        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
+        ${file} > ${file}.tmp
+    fi
 
   elif [ ${executionType} -eq 3 ] ; then
     # SPMD + MPI/OMP
-    sed -e "s/::openMPthreads::/${openMPthreads}/" \
+    if [ X${MPIEnvironment} = XIBM ] ; then
+      sed -e "s/::openMPthreads::/${openMPthreads}/" \
         -e "s/::JobNumProcTot::/${mpiTasks}/"      \
-      ${file} > ${file}.tmp
+        ${file} > ${file}.tmp
+    else
+      sed -e "s/::openMPthreads::/${openMPthreads}/" \
+        -e "s/@ job_type = parallel/@ job_type = mpich/" \
+        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
+        ${file} > ${file}.tmp
+    fi
 
   elif [ ${executionType} -eq 4 ] ; then
     # SPMD + MPI only
-    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
+    if [ X${MPIEnvironment} = XIBM ] ; then
+      sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
         -e "/::openMPthreads::/d"                  \
-      ${file} > ${file}.tmp
+        ${file} > ${file}.tmp
+    else
+      sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
+        -e "s/@ job_type = parallel/@ job_type = mpich/" \
+        -e "/::openMPthreads::/d"                  \
+        ${file} > ${file}.tmp
+    fi
 
   elif [ ${executionType} -eq 5 ] ; then
@@ -1005,91 +1070,277 @@
   if ( ${OK_PARA_MPMD} ) ; then
 
-    if [ -f run_file ] ; then
-      IGCM_sys_Rm -f run_file
-    fi
-    touch run_file
-
-    # Build run_file
-
-    # First loop on the components for the coupler ie oasis (only if oasis3)
-    # the coupler ie oasis3 must be the first one
-    for comp in ${config_ListOfComponents[*]} ; do
-
-      eval ExeNameIn=\${config_Executable_${comp}[0]}
-      eval ExeNameOut=\${config_Executable_${comp}[1]}
-
-      # for CPL component only
-      if [ "X${comp}" = "XCPL" ] ; then
-
-        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-
-        if ( ${OK_PARA_MPI} ) ; then
-          (( mpi_count = 1 ))
-          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
-            echo "./${ExeNameOut}" >> run_file
-            (( mpi_count = mpi_count + 1 ))
-          done
-        else
-          echo "./${ExeNameOut} " >> run_file
+    # MPI IBM Environment
+    if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
+      IGCM_debug_Print 1 "You use IBM MPI environment"
+      if [ -f run_file ] ; then
+        IGCM_sys_Rm -f run_file
+      fi
+      touch run_file
+
+      # Build run_file
+
+      # First loop on the components for the coupler ie oasis (only if oasis3)
+      # the coupler ie oasis3 must be the first one
+      for comp in ${config_ListOfComponents[*]} ; do
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # for CPL component only
+        if [ "X${comp}" = "XCPL" ] ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+          if ( ${OK_PARA_MPI} ) ; then
+            (( mpi_count = 1 ))
+            until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
+              echo "./${ExeNameOut}" >> run_file
+              (( mpi_count = mpi_count + 1 ))
+            done
+          else
+            echo "./${ExeNameOut} " >> run_file
+          fi
+        fi
+
+      done
+
+      # Then second loop on the components
+      for comp in ${config_ListOfComponents[*]} ; do
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # Only if we really have an executable for the component and not the coupler ie oasis:
+        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+          if ( ${OK_PARA_OMP} ) ; then
+            # Check if the number of threads is correct
+            case ${comp_proc_omp_loc} in
+            1|2|4)
+              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads on IBM MPI Environment"
+              IGCM_debug_Print 1 "Beware : it may you use more CPU than needed."
+              IGCM_debug_Print 1 "Think to the Intel MPI Environment to do what you want to do !"
+              ;;
+            8|16)
+              IGCM_debug_Exit "ERROR with OMP parameters !"
+              IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
+              IGCM_debug_Print 2 "You will use more CPU than needed : try to use Intel-MPI environment to do such a thing !"
+              IGCM_debug_Verif_Exit
+              ;;
+            *)
+              IGCM_debug_Exit "ERROR with OMP parameters !"
+              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+              IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
+              IGCM_debug_Verif_Exit
+              ;;
+            esac
+
+          fi
+
+          if ( ${OK_PARA_MPI} ) ; then
+            (( mpi_count = 1 ))
+            until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
+              if ( ${OK_PARA_OMP} ) ; then
+                echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
+              else
+                echo "./${ExeNameOut}" >> run_file
+              fi
+              (( mpi_count = mpi_count + 1 ))
+            done
+          else
+            echo "./${ExeNameOut} " >> run_file
+          fi
+        fi
+      done
+      if ( ${OK_PARA_OMP} ) ; then
+        export KMP_STACKSIZE=200m
+      fi
+
+      EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
+
+      IGCM_sys_Chmod u+x run_file
+      if ( $DEBUG_sys ) ; then
+        echo "run_file contains : "
+        cat run_file
+      fi
+
+
+    # MPI Intel Environment
+    else
+      IGCM_debug_Print 1 "You use Intel MPI environment"
+
+      # Only MPI (MPMD)
+      if  ( ! ${OK_PARA_OMP} ) ; then
+	init_exec=n
+  
+        # First loop on the components for the coupler ie oasis (only if oasis3)
+        # the coupler ie oasis3 must be the first one
+        for comp in ${config_ListOfComponents[*]} ; do
+
+          eval ExeNameIn=\${config_Executable_${comp}[0]}
+          eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+          # for CPL component only
+          if [ "X${comp}" = "XCPL"  ]  && [ "X${ExeNameOut}" != X\"\" ] ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+            echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+            echo ""  >> script_${ExeNameOut}.ksh
+            echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+            IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+            init_exec=y
+          fi
+        done
+        # Then second loop on the components
+        for comp in ${config_ListOfComponents[*]} ; do
+
+          eval ExeNameIn=\${config_Executable_${comp}[0]}
+          eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+          # Only if we really have an executable for the component and not the coupler ie oasis:
+          if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+            echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+            echo ""  >> script_${ExeNameOut}.ksh
+            echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+            IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+            if [ ${init_exec} = y ] ; then
+              EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+            else
+              EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+              init_exec=y
+            fi
+          fi
+        done
+      # MPI-OpenMP (MPMD)
+      else
+
+        # Execution specifications
+        EXECUTION="${EXECUTION} -configfile run_file"
+        export KMP_STACKSIZE=200m
+        if [ -f run_file ] ; then
+          IGCM_sys_Rm -f run_file
+        fi
+        touch run_file
+
+        # Initialisation of variables
+        string_final=""
+        string_final_hexa=""
+        current_core=0
+        executable_nb=1
+
+        #  Hosts treatment
+        for nodes in `echo $LOADL_PROCESSOR_LIST`
+        do
+          host[$i]=$nodes
+          i=$((i+1))
+        done
+
+        # Loop on the components
+        for comp in ${config_ListOfComponents[*]} ; do
+
+          eval ExeNameIn=\${config_Executable_${comp}[0]}
+          eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+          # Not possible if oasis has an executable (i.e old version of oasis3)
+          if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
+            IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
+            IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
+            IGCM_debug_Verif_Exit
+          fi
+
+          # Only if we really have an executable for the component :
+          if [ "X${ExeNameOut}" != X\"\" ] ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+            # Check if the number of threads is correct
+            case ${comp_proc_omp_loc} in
+            1|2|4|8|16)
+              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+              ;;
+            *)
+              IGCM_debug_Exit "ERROR with OMP parameters !"
+              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+              IGCM_debug_Print 2 "Only 1,2,4,8,16 as number of OMP threads are possible "
+              IGCM_debug_Verif_Exit
+              ;;
+            esac
+
+
+            # Build run_file for Ada Intel-MPI environment : method used to assign cores and nodes for the MPI process by using hexadecimal mask
+            # Example of run_file :
+            #-host host0 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
+            #-host host1 -n 4 -env I_MPI_PIN_DOMAIN=[ff,ff00,ff0000,ff000000] ./a.out
+            #-host host2 -n 10 -env I_MPI_PIN_DOMAIN=[3,c,30,c0,300,c00,3000,c000,30000,c0000,100000,200000,400000,800000,1000000,2000000,4000000,8000000,10000000,20000000] ./b.out
+            #-host host2 -n 10 ./c.out
+            # Example of final command :
+            # mpirun -configfile run_file
+
+            rank=0
+            # For one specific executable, loop on mpi process
+            for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+              (( index_host = current_core / core_per_node ))
+              host_value=${host[${index_host}]}
+              (( slot =  current_core % core_per_node ))
+              # loop on omp threads for each mpi process (set the appropriate bit to 1 and append it to previous one)
+              for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
+                string_final="1"$string_final
+              done
+              # convert binary mask to hexadecimal mask
+              if [ $rank -ne 0 ] ; then
+                string_final_hexa=$string_final_hexa","$( printf '%x\n' "$((2#$string_final))" )
+              else
+                string_final_hexa=$( printf '%x\n' "$((2#$string_final))" )
+              fi
+              # replace bit 1 by bit 0 in order to append next one (next one wil be 1)
+              string_final=$( echo $string_final | sed "s/1/0/g" )
+              # mpi rank = mpi_rank + 1
+              (( rank = rank + 1 ))
+              # current core takes into account the number of omp threads which was previously append
+              (( current_core = current_core + comp_proc_omp_loc ))
+              # We write to the configuration file either we switch to the next node or we switch to the next executable
+              if ( [ $(( current_core / core_per_node )) -ne $index_host ] || [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ) ; then
+                # I_MPI_PIN_DOMAIN variable must be given once per node
+                if [ $executable_nb -eq 1 ] ; then
+                  echo "-host $host_value -n $rank -env I_MPI_PIN_DOMAIN=[$string_final_hexa] ./$ExeNameOut" >> run_file
+                else
+                  sed -i "/$host_value/s/\]/\,$string_final_hexa\]/g" run_file
+                  echo "-host $host_value -n $rank ./$ExeNameOut" >> run_file
+                fi
+                # +1 for the number of executbale on the same node
+                if [ $nb_proc_mpi -eq $(($comp_proc_mpi_loc-1)) ] ; then
+                  (( executable_nb = executable_nb + 1 ))
+                fi
+                # Some initializations if we switch to the next node
+                if [ $(( current_core / core_per_node )) -ne $index_host ] ; then
+                  string_final=""
+                  string_final_hexa=""
+                  rank=0
+                  executable_nb=1
+                fi
+              fi
+            done
+
+          fi
+        done
+        IGCM_sys_Chmod u+x run_file
+        if ( $DEBUG_sys ) ; then
+          echo "run_file contains : "
+          cat run_file
         fi
       fi
-    done
-
-    # Then second loop on the components
-    for comp in ${config_ListOfComponents[*]} ; do
-
-      eval ExeNameIn=\${config_Executable_${comp}[0]}
-      eval ExeNameOut=\${config_Executable_${comp}[1]}
-
-      # Only if we really have an executable for the component and not the coupler ie oasis:
-      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
-
-        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-
-        if ( ${OK_PARA_MPI} ) ; then
-          (( mpi_count = 1 ))
-          until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
-            if ( ${OK_PARA_OMP} ) ; then
-              # Check if the number of threads is correct
-              case ${comp_proc_omp_loc} in
-              1|2|4)
-                IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
-                ;;
-              8|16)
-                IGCM_debug_Exit "ERROR with OMP parameters !"
-                IGCM_debug_Print 2 "Beware : ${comp_proc_omp_loc} is too much for MPMD mode"
-                IGCM_debug_Print 2 "You will use more CPU than needed"
-                IGCM_debug_Verif_Exit
-                ;;
-              *)
-                IGCM_debug_Exit "ERROR with OMP parameters !"
-                IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
-                IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
-                IGCM_debug_Verif_Exit
-                ;;
-              esac
-              echo "env OMP_NUM_THREADS=$comp_proc_omp_loc ./${ExeNameOut}" >> run_file
-            else
-              echo "./${ExeNameOut}" >> run_file
-            fi
-            (( mpi_count = mpi_count + 1 ))
-          done
-        else
-          echo "./${ExeNameOut} " >> run_file
-        fi
-        if ( ${OK_PARA_OMP} ) ; then
-          export KMP_STACKSIZE=200m
-        fi
-      fi
-    done
-
-    EXECUTION="${HOST_MPIRUN_COMMAND} -pgmmodel mpmd -cmdfile ./run_file"
-
-    IGCM_sys_Chmod u+x run_file
-    if ( $DEBUG_sys ) ; then
-      echo "run_file contains : "
-      cat run_file
     fi
 
@@ -1111,5 +1362,14 @@
           #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
         fi
-        EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
+        if  ( ${OK_PARA_MPI} ) ; then
+          if ( [ "X${LOADL_STEP_TYPE}" = "XPARALLEL" ] || [ "X${LOADL_STEP_TYPE}" = "XSERIAL" ] ) ; then
+            EXECUTION="${HOST_MPIRUN_COMMAND} ./${ExeNameOut}"
+          else
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
+          fi
+        else
+          EXECUTION="/usr/bin/time ./${ExeNameOut}"
+        fi
       fi
 
@@ -1220,5 +1480,5 @@
   # Print only the full (-W) JobID (%id) and JobName (%jn)
   ID=$( llq -u $2 -W -f %id %jn | \
-        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
+    gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
 
   eval ${3}=${ID}
@@ -1258,5 +1518,5 @@
   # With -W option, column width is as large as necessary
   set -A JobList $( llq -u $1 -W -f %jn | head -n -2 | tail -n +3 | \
-                    gawk '( $1 != /TS/      && \
+    gawk '( $1 != /TS/      && \
                             $1 !~ /PACK/    && \
                             $1 !~ /REBUILD/ && \
@@ -1286,14 +1546,14 @@
   if [ ${status} -gt 0 ] ; then
     echo "IGCM_sys_atlas : error code ${status}"
-
-    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 
-    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 
- 	            IGCM_debug_PopStack "IGCM_sys_atlas" 
-	    return 1 
-  else 
-    IGCM_debug_PopStack "IGCM_sys_atlas" 
-	 	    return 0 
-  fi 
-
-  IGCM_debug_PopStack "IGCM_sys_atlas" 
-} 
+    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
+    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
+    IGCM_debug_PopStack "IGCM_sys_atlas"
+    return 1
+  else
+    IGCM_debug_PopStack "IGCM_sys_atlas"
+    return 0
+  fi
+
+  IGCM_debug_PopStack "IGCM_sys_atlas"
+}
+
