Index: branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_curie.ksh
===================================================================
--- branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh	(revision 534)
+++ branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_curie.ksh	(revision 563)
@@ -68,5 +68,5 @@
 typeset  LOGIN=${LOGIN:=$( whoami )}
 # $hostname of the MASTER job
-typeset MASTER=titane
+typeset MASTER=curie
 
 #D-
@@ -80,5 +80,5 @@
 typeset -r RSYNC_opt="-va"
 # ie storage filesystem
-typeset -r RHOST=titane
+typeset -r RHOST=curie
 
 #====================================================
@@ -90,5 +90,5 @@
 # Set environment tools (ferret, nco, cdo)
 #====================================================
-. /home/cont003/p86ipsl/.atlas_env_titane_ksh
+. /ccc/cont003/home/dsm/p86ipsl/.atlas_env_curie_ksh
 
 #====================================================
@@ -107,32 +107,21 @@
 #====================================================
 #- SUBMIT_DIR : submission dir
-typeset SUBMIT_DIR=${SUBMIT_DIR:=${LS_SUBCWD}}
+typeset SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
 
 #====================================================
 #- ARCHIVE
-typeset -r ARCHIVE=${DMFDIR}
-
-#====================================================
-#- Mirror libIGCM from titane to cesium if needed
-ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $2}' )
-if [ ! ${ROOTSYS} = "home" ] ; then
-    typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
-else
-    typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
-fi
+typeset -r ARCHIVE=${CCCSTOREDIR}
+
+#====================================================
+typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
 
 #====================================================
 #- libIGCM_POST
-if ( ${MirrorlibIGCM} ) ; then
-    PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
-    typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
-else
-    typeset -r libIGCM_POST=${libIGCM}
-fi
+typeset -r libIGCM_POST=${libIGCM}
 
 #====================================================
 #- IN
-typeset -r R_IN=${R_IN:=/dmnfs/cont003/p86ipsl/IGCM}
-typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
+typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
+typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
 
 #====================================================
@@ -146,12 +135,12 @@
 #====================================================
 #- OUT_POST
-typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
+typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT_POST
 
 #====================================================
 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
-if [ ! X${LSB_QUEUE} = Xmono ] ; then
-    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${LSB_JOBID}}
+if [ ! X${BRIDGE_MSUB_NPROC} = X1 ] ; then
+    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}}
 else
-    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${LSB_JOBID}}
+    typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${BRIDGE_MSUB_JOBID}}
 fi
 
@@ -162,5 +151,5 @@
 #====================================================
 #- HOST_MPIRUN_COMMAND
-typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
+typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time ccc_mprun"}
 
 #====================================================
@@ -224,56 +213,9 @@
     fi
 
-    typeset NB_ESSAI DELAI status i
-    # keep standard input for the loop onto temporary file
-    cat >tmp_IGCM_sys_RshPost_$$
-
-    #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
-    #cat tmp_IGCM_sys_RshPost_$$
-
-    if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
-	# little hack so that rebuild submission is done on titane not an cesium
-	#
-	libIGCM_POST_sed=$( echo $libIGCM_POST | sed 's/\//\\\//g' )
-	POST_DIR_sed=$( echo ${POST_DIR} | sed 's/\//\\\//g' )
-	sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
-	sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
-	sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
-	sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
-	#\mv tmp.txt tmp_IGCM_sys_RshPost_$$
-	#
-	#echo cat tmp_IGCM_sys_RshPost_$$ AFTER
-	#cat tmp_IGCM_sys_RshPost_$$
-	/bin/ksh <tmp_IGCM_sys_RshPost_$$
+	/bin/ksh ${@}
 	if [ $? -gt 0 ] ; then
 	    echo "IGCM_sys_RshPost : erreur."
 	    IGCM_debug_Exit "IGCM_sys_RshPost"
 	fi
-        # delete temporary file
-        \rm tmp_IGCM_sys_RshPost_$$
-    else
-        # number of tentative
-	NB_ESSAI=10
-        # time delay between tentative
-	DELAI=10
-	i=0
-	while [ $i -ne $NB_ESSAI ] ; do
-            ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
-            status=$?
-            if [ ${status} -ne 0 ];
-            then
-		sleep $DELAI
-            else
-		break
-            fi
-            let i=$i+1
-	done
-        # delete temporary file
-	\rm tmp_IGCM_sys_RshPost_$$
-
-	if [ ${status} -gt 0 ] ; then
-	    echo "IGCM_sys_RshPost : erreur."
-	    IGCM_debug_Exit "IGCM_sys_RshPost"
-	fi
-    fi
     IGCM_debug_PopStack "IGCM_sys_RshPost"
 }
@@ -295,14 +237,4 @@
 	status=completed
     fi
-
-    ssh -t titane996 ssh platine /bin/ksh <<-EOF
-    export LOGIN=${LOGIN}
-    export config_UserChoices_JobName=${config_UserChoices_JobName}
-    export config_UserChoices_MailName=${config_UserChoices_MailName}
-    export DateBegin=${DateBegin}
-    export DateEnd=${DateEnd}
-    export R_SAVE=${R_SAVE}
-    export SUBMIT_DIR=${SUBMIT_DIR}
-    export status=${status}
 
     cat  << END_MAIL > job_end.mail
@@ -317,12 +249,10 @@
 
     if [ ! -z ${config_UserChoices_MailName} ] ; then
-	mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
+	mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail
     elif [ -f ~/.forward ] ; then
-	mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
-    fi
-
-    sleep 10
-    rm -f job_end.mail
-EOF
+	mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
+    else
+    mail -s "${config_UserChoices_JobName} ${status}" ${USER} < job_end.mail
+    fi
 
     if [ $? -gt 0 ] ; then
@@ -350,5 +280,5 @@
 	fi
     fi
-    # vérification :
+    # verification :
     if [ ! -d ${1} ] ; then
 	echo "IGCM_sys_Mkdir : erreur."
@@ -584,7 +514,7 @@
 
     if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then
-        /usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} < $1
+        /usr/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} < $1
     else
-        /usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod} < $1
+        /usr/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod} < $1
     fi
 
@@ -606,5 +536,5 @@
 	echo "IGCM_sys_QsubPost :" $@
     fi
-    /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
+    /usr/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job
     if [ $? -gt 0 ] ; then
 	echo "IGCM_sys_QsubPost : erreur " $@
@@ -940,6 +870,6 @@
 	#
 	#USUAL WAY
-        # add dmget (to demigrate all offline files) to reduce time of this command :
-        dmget $1/*
+        # add get (to demigrate all offline files) to reduce time of this command :
+        ccc_hsm get $1/*
 	\cp -r $1 $2 > out_rsync 2>&1
   	RET=$?
@@ -1167,5 +1097,5 @@
 	fi
 
-	dmget ${dm_liste[*]} > out_rsync 2>&1
+	ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
   	RET=$?
 
@@ -1273,5 +1203,5 @@
 	echo "IGCM_sys_rebuild :" $@
     fi
-    /home/cont003/p86ipsl/X64/bin/rebuild -f -o $@
+    ~p86ipsl/CESIUM/bin/rebuild -f -o $@
     if [ $? -gt 0 ] ; then
        echo "IGCM_sys_rebuild : erreur ${@}."
@@ -1290,30 +1220,6 @@
 	echo "IGCM_sys_activ_variables"
     fi
-#    ulimit -s 2097152
-
-    typeset max_omp
 
     ulimit -s unlimited
-
-    (( max_omp = 0 ))
-    for comp in ${config_ListOfComponents[*]} ; do
-	
-	eval ExeNameIn=\${config_Executable_${comp}[0]}
-	eval ExeNameOut=\${config_Executable_${comp}[1]}
-	
-    
-	# Only if we really have an executable for the component :
-	if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
-	    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-
-	    if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
-
-		(( max_omp = comp_proc_omp_loc ))
-	    fi
-	fi
-    done
-    if [ ${max_omp} -gt 1 ] ; then
-	module load openmp/${max_omp}thds
-    fi
 
     IGCM_debug_PopStack "IGCM_sys_activ_variables"
@@ -1332,30 +1238,11 @@
 
 
-# function IGCM_sys_build_run_file {
+function IGCM_sys_build_run_file {
 ############################################################
 # Build run file
 
-#     IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-#     if ( $DEBUG_sys ) ; then
-# 	echo "IGCM_sys_build_run_file " $@
-#     fi
-
-#     # set Number of processors for OCE here
-#     NUM_PROC_OCE=5
-
-#     (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
-#     (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
-#     if [ $1 = MPI1 ]; then
-# 	cat <<EOF > run_file
-# -np 1 ./oasis
-# -np ${NUM_PROC_ATM} ./lmdz.x
-# -np ${NUM_PROC_OCE} ./opa.xx
-# EOF
-# 	config_UserChoices_JobRunOptions='"--app"'
-# 	IGCM_sys_Chmod u+x run_file
-#     fi
-#     IGCM_debug_PopStack "IGCM_sys_build_run_file"
-  
-# }
+ IGCM_debug_Print 1 "Dummy IGCM_sys_build_run_file"
+
+}
 
 ############################################################
@@ -1368,199 +1255,84 @@
     fi
 
-    typeset nodes listnodes init_node node_num_current start_num init_exec comp ExeNameIn ExeNameOut 
-    typeset nombre_restant_node node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc 
-    typeset num_corempi nombre_restant_node nombre_restant_comp
-
-    # Verification with PBS parameter
-    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
-
-	if ( ${OK_PARA_MPI} ) ; then
-	    EXECUTION=${HOST_MPIRUN_COMMAND}
-	else
-	    if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then
-		EXECUTION=${MPIRUN_COMMAND:="time "}
-	    fi
-	fi
-    else
-	if ( ! ${OK_PARA_MPI} ) ; then
-            EXECUTION=${MPIRUN_COMMAND:="time "}
-	fi
-    fi
-
-    if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then
-
-    #  Hosts treatment
-
-	${EXECUTION} hostname | sort | uniq > hosts.tmp
-	
-	i=0
-	rm -f hosts
-	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
-	for nodes in `cat hosts.tmp`
-	do
-	    host[$i]=$nodes
-	    echo "${host[$i]} slots=8 max_slots=8" >> hosts
-	    IGCM_debug_Print 1 ${host[$i]}
-	    i=$((i+1))
-	done
-	rm -f hosts.tmp
-
-	listnodes=${host[*]}
-    
-	EXECUTION="${EXECUTION} -hostfile hosts" 
-    fi
-
-# Initialisation
-
-    init_node=y
-    node_num_current=0
-    start_num=0
-    init_exec=n
-
-
-# Test : if oasis is there, we put it at the first position
-		
+    # Verification with MSUB parameter
+	EXECUTION=${HOST_MPIRUN_COMMAND}
+
+    if ( ${OK_PARA_MPMD} ) ; then
+
+        if [ -f run_file ] ; then
+            IGCM_sys_Rm -f run_file
+        fi
+        touch run_file
+
+# run_file construction
+
+# Then first loop on the components for the coupler ie oasis
+
+### the coupler ie oasis must be the first one
     for comp in ${config_ListOfComponents[*]} ; do
-	
-	if [ "X${comp}" = "XCPL" ]  ; then
-	    
-	    eval ExeNameIn=\${config_Executable_${comp}[0]}
-	    eval ExeNameOut=\${config_Executable_${comp}[1]}
-	    
-	    echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
-	    echo ""  >> script_${ExeNameOut}.ksh
-	    echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
-	    echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
-	    echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
-	    echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
-	    IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
-	    
-	    init_node=n
-	    
-	    (( nombre_restant_node = NUM_COREPERNODE - 1 ))
-	    node_num_current=0
-	    node_current=${host[${node_num_current}]}
-	    
-	    EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 
-	    
-            init_exec=y
-	    start_num=1
-	    
-	fi
-	
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # for CPL component only
+        if [ "X${comp}" = "XCPL" ] ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+        fi
     done
-    
-# Then loop on the components (except for oasis)
+
+# Then second loop on the components
 
     for comp in ${config_ListOfComponents[*]} ; do
-	
-	eval ExeNameIn=\${config_Executable_${comp}[0]}
-	eval ExeNameOut=\${config_Executable_${comp}[1]}
-	
-    
-	# Only if we really have an executable for the component :
-	if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
-
-	    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-	    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-	    
-	    if ( ${OK_PARA_MPI} ) ; then 
-		
-		echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
-#		echo "set -vx" >> script_${ExeNameOut}.ksh
-		echo ""  >> script_${ExeNameOut}.ksh
-		echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
-		echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
-		echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
-		echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
- 		echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
-		echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
-		IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
-		
-		node_num=0
-		
-# We define the number of MPI process to be assigned for the component 
-
-		nombre_restant_comp=${comp_proc_mpi_loc}
-		
-# Loop on the allocated nodes
-
-		for node in ${listnodes} ; do
-
-# We go to the current node
-
-		    if [ ${node_num} = ${node_num_current} ] ; then
-
-			node_current=${host[${node_num_current}]}
-
-# If first time on the node : initialisation 
-
-			if [ ${init_node} = y ] ; then
-			    nombre_restant_node=${NUM_COREPERNODE}
-			fi
-	
-# Test on the number of OMP threads
-
-			if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
-			    (( node_num = node_num + 1 ))
-			    node_num_current=${node_num}
-			    init_node=y
-			    continue
-			fi
-
-# Number of MPI process to assign
-
-			(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
- 
-			if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
-			    num_corempi=${nombre_restant_comp}
-			fi
-
-			(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
-			(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
-		    
-			if [ ${init_exec} = y ] ; then
-			    EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
-			else	
-			    EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
-			    init_exec=y
-			fi
-
-			((  start_num = num_corempi + start_num ))
-	
-		    else
-			
-			(( node_num = node_num + 1 )) 
-			continue
-		    fi
-		
-# Test on the number of core/process remaining on the node/component 
-		
-		    if [ ${nombre_restant_node} = 0 ] ; then
-			(( node_num = node_num + 1 ))
-			node_num_current=${node_num}
-			init_node=y
-
-			if [ ${nombre_restant_comp} = 0 ] ; then
-			    break 1
-			fi
-		    else
-
-			node_num_current=${node_num}
-			init_node=n
-			
-			if [ ${nombre_restant_comp} = 0 ] ; then
-			    break 1
-			fi
-		    fi
-		done
-	    else
-		EXECUTION="${EXECUTION} ./${ExeNameOut}"
-	    fi
-	fi
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # Only if we really have an executable for the component and not the coupler ie oasis:
+        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+        fi
     done
 
-    IGCM_debug_Print 1 "sys Titane : La commande d execution est "
-    IGCM_debug_Print 1 $EXECUTION
+    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+    IGCM_sys_Chmod u+x run_file
+    if ( $DEBUG_sys ) ; then
+      echo "run_file contains : "
+      cat run_file
+    fi
+
+else # Only one executable. launch it.
+
+    # 
+    for comp in ${config_ListOfComponents[*]} ; do
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        if  [ "X${ExeNameOut}" != X\"\" ] ; then
+            if ( ${OK_PARA_MPI} ) ; then
+                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                EXECUTION="${EXECUTION} -n ${comp_proc_mpi_loc} ./${ExeNameOut}"
+            else
+                EXECUTION="${EXECUTION} ./${ExeNameOut}"
+            fi
+            if ( ${OK_PARA_OMP} ) ; then
+                eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                EXECUTION="export OMP_NUM_THREADS=${comp_proc_omp_loc} ; ${EXECUTION} "
+            fi
+        fi
+
+    done
+
+fi
+
+    IGCM_debug_Print 1 "sys Curie : execution command is "
+    IGCM_debug_Print 1 "$EXECUTION"
 
     IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
