Changeset 490 for branches/libIGCM_MPI_OpenMP/libIGCM_sys

Timestamp:

06/16/11 12:31:47 (13 years ago)

Author:

mmaipsl

Message:

First version for MPI/OpenMP integration in libIGCM.
Work only on titane for instance.

File:

• branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh (modified) (3 diffs)

branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh

@@ -167 +167 @@
 #- Max number of arguments passed to nco operator or demigration command
 UNIX_MAX_LIMIT=360
+
+NUM_COREPERNODE=8
+        
 
 #D-#==================================================
@@ -1266 +1269 @@
         echo "IGCM_sys_activ_variables"
     fi
-    ulimit -s 2097152
+#    ulimit -s 2097152
+
+    typeset max_omp
+
+    ulimit -s unlimited
+
+    (( max_omp = 0 ))
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+        
+    
+        # Only if we really have an executable for the component :
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+            if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
+
+                (( max_omp = comp_proc_omp_loc ))
+            fi
+        fi
+    done
+    if [ ${max_omp} -gt 1 ] ; then
+        module load openmp/${max_omp}thds
+    fi
+
     IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1281 +1310 @@
 }
 
+
+# function IGCM_sys_build_run_file {
 ############################################################
 # Build run file
 
-function IGCM_sys_build_run_file {
-    IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-    if ( $DEBUG_sys ) ; then
-        echo "IGCM_sys_build_run_file " $@
-    fi
-
-    # set Number of processors for OCE here
-    NUM_PROC_OCE=5
-
-    (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
-    (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
-    if [ $1 = MPI1 ]; then
-        cat <<EOF > run_file
--np 1 ./oasis
--np ${NUM_PROC_ATM} ./lmdz.x
--np ${NUM_PROC_OCE} ./opa.xx
-EOF
-        config_UserChoices_JobRunOptions='"--app"'
-        IGCM_sys_Chmod u+x run_file
-    fi
-    IGCM_debug_PopStack "IGCM_sys_build_run_file"
+#     IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
+#     if ( $DEBUG_sys ) ; then
+#       echo "IGCM_sys_build_run_file " $@
+#     fi
+
+#     # set Number of processors for OCE here
+#     NUM_PROC_OCE=5
+
+#     (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
+#     (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
+#     if [ $1 = MPI1 ]; then
+#       cat <<EOF > run_file
+# -np 1 ./oasis
+# -np ${NUM_PROC_ATM} ./lmdz.x
+# -np ${NUM_PROC_OCE} ./opa.xx
+# EOF
+#       config_UserChoices_JobRunOptions='"--app"'
+#       IGCM_sys_Chmod u+x run_file
+#     fi
+#     IGCM_debug_PopStack "IGCM_sys_build_run_file"
   
-}
+# }
+
+############################################################
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+    IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+    if ( $DEBUG_sys ) ; then
+        echo "IGCM_sys_build_execution_scripts " $@
+    fi
+
+    # Verification with PBS parameter
+    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
+
+        if ( ${OK_PARA_MPI} ) ; then
+            EXECUTION=${HOST_MPIRUN_COMMAND}
+        else
+            if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then
+                EXECUTION=${MPIRUN_COMMAND:="time "}
+            fi
+        fi
+    else
+        if ( ! ${OK_PARA_MPI} ) ; then
+            EXECUTION=${MPIRUN_COMMAND:="time "}
+        fi
+    fi
+
+    if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then
+
+    #  Hosts treatment
+
+        ${EXECUTION} hostname | sort | uniq > hosts.tmp
+        
+        i=0
+        rm -f hosts
+        IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
+        for nodes in `cat hosts.tmp`
+        do
+            host[$i]=$nodes
+            echo "${host[$i]} slots=8 max_slots=8" >> hosts
+            IGCM_debug_Print 1 ${host[$i]}
+            i=$((i+1))
+        done
+        rm -f hosts.tmp
+
+        listnodes=${host[*]}
+    
+        EXECUTION="${EXECUTION} -hostfile hosts" 
+    fi
+
+# Initialisation
+
+    init_node=y
+    node_num_current=0
+    start_num=0
+    init_exec=n
+
+
+# Test : if oasis is there, we put it at the first position
+                
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        if [ "X${comp}" = "XCPL" ]  ; then
+            
+            eval ExeNameIn=\${config_Executable_${comp}[0]}
+            eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+            echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+            echo ""  >> script_${ExeNameOut}.ksh
+            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+            echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
+            IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+            
+            init_node=n
+            
+            (( nombre_restant_node = NUM_COREPERNODE - 1 ))
+            node_num_current=0
+            node_current=${host[${node_num_current}]}
+            
+            EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 
+            
+            init_exec=y
+            start_num=1
+            
+        fi
+        
+    done
+    
+# Then loop on the components (except for oasis)
+
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+        
+    
+        # Only if we really have an executable for the component :
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+                
+                echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+#               echo "set -vx" >> script_${ExeNameOut}.ksh
+                echo ""  >> script_${ExeNameOut}.ksh
+                echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+                echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+                echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+                echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+                echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
+                echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
+                IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                
+                node_num=0
+                
+# We define the number of MPI process to be assigned for the component 
+
+                nombre_restant_comp=${comp_proc_mpi_loc}
+                
+# Loop on the allocated nodes
+
+                for node in ${listnodes} ; do
+
+# We go to the current node
+
+                    if [ ${node_num} = ${node_num_current} ] ; then
+
+                        node_current=${host[${node_num_current}]}
+
+# If first time on the node : initialisation 
+
+                        if [ ${init_node} = y ] ; then
+                            nombre_restant_node=${NUM_COREPERNODE}
+                        fi
+        
+# Test on the number of OMP threads
+
+                        if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
+                            (( node_num = node_num + 1 ))
+                            node_num_current=${node_num}
+                            init_node=y
+                            continue
+                        fi
+
+# Number of MPI process to assign
+
+                        (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
+ 
+                        if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
+                            num_corempi=${nombre_restant_comp}
+                        fi
+
+                        (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
+                        (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
+                    
+                        if [ ${init_exec} = y ] ; then
+                            EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+                        else    
+                            EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+                            init_exec=y
+                        fi
+
+                        ((  start_num = num_corempi + start_num ))
+        
+                    else
+                        
+                        (( node_num = node_num + 1 )) 
+                        continue
+                    fi
+                
+# Test on the number of core/process remaining on the node/component 
+                
+                    if [ ${nombre_restant_node} = 0 ] ; then
+                        (( node_num = node_num + 1 ))
+                        node_num_current=${node_num}
+                        init_node=y
+
+                        if [ ${nombre_restant_comp} = 0 ] ; then
+                            break 1
+                        fi
+                    else
+
+                        node_num_current=${node_num}
+                        init_node=n
+                        
+                        if [ ${nombre_restant_comp} = 0 ] ; then
+                            break 1
+                        fi
+                    fi
+                done
+            else
+                EXECUTION="${EXECUTION} ./${ExeNameOut}"
+            fi
+        fi
+    done
+
+    IGCM_debug_Print 1 "sys Titane : La commande d execution est "
+    IGCM_debug_Print 1 $EXECUTION
+
+    IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
+}
+
+
 
 ##############################################################