Changeset 1000 for trunk/libIGCM
- Timestamp:
- 03/28/14 16:45:03 (10 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh
r996 r1000 2083 2083 EXECUTION=${HOST_MPIRUN_COMMAND} 2084 2084 2085 # MPMD mode 2086 2085 2087 if ( ${OK_PARA_MPMD} ) ; then 2086 2088 2087 if [ -f run_file ] ; then 2088 IGCM_sys_Rm -f run_file 2089 fi 2090 touch run_file 2091 2092 # run_file construction 2093 2094 # Then first loop on the components for the coupler ie oasis 2095 2096 ## the coupler ie oasis must be the first one 2097 for comp in ${config_ListOfComponents[*]} ; do 2098 2099 eval ExeNameIn=\${config_Executable_${comp}[0]} 2100 eval ExeNameOut=\${config_Executable_${comp}[1]} 2089 # Only MPI (MPMD) 2090 if ( ! ${OK_PARA_OMP} ) ; then 2091 2092 if [ -f run_file ] ; then 2093 IGCM_sys_Rm -f run_file 2094 fi 2095 touch run_file 2096 2097 # Build run_file 2098 2099 # First loop on the components for the coupler ie oasis (only if oasis3) 2100 2101 ## the coupler ie oasis3 must be the first one 2102 for comp in ${config_ListOfComponents[*]} ; do 2103 2104 eval ExeNameIn=\${config_Executable_${comp}[0]} 2105 eval ExeNameOut=\${config_Executable_${comp}[1]} 2101 2106 2102 2107 # for CPL component only 2103 if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then 2104 2105 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 2106 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 2107 echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 2108 if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then 2109 2110 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 2111 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 2112 echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 2113 fi 2114 done 2115 2116 # Then second loop on the components 2117 2118 for comp in ${config_ListOfComponents[*]} ; do 2119 2120 eval ExeNameIn=\${config_Executable_${comp}[0]} 2121 eval ExeNameOut=\${config_Executable_${comp}[1]} 2122 2123 # Only if we really have an executable for the component and not the coupler ie oasis: 2124 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then 2125 2126 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 2127 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 2128 echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 2129 fi 2130 done 2131 2132 EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file" 2133 2134 IGCM_sys_Chmod u+x run_file 2135 if ( $DEBUG_sys ) ; then 2136 echo "run_file contains : " 2137 cat run_file 2138 fi 2139 2140 # MPI-OpenMP (MPMD) 2141 else 2142 2143 # Use of mpirun instead of ccc_mprun 2144 EXECUTION="time mpirun" 2145 2146 # Hosts treatment 2147 ${EXECUTION} hostname | sort | uniq > hosts.tmp 2148 2149 i=0 2150 rm -f hosts rankfile 2151 IGCM_debug_Print 1 "sys Curie, Hosts available :" 2152 for nodes in `cat hosts.tmp` 2153 do 2154 host[$i]=$nodes 2155 echo "${host[$i]}" >> hosts 2156 IGCM_debug_Print 1 ${host[$i]} 2157 i=$((i+1)) 2158 done 2159 rm -f hosts.tmp 2160 2161 listnodes=${host[*]} 2162 2163 EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile" 2164 2165 # Initialisation 2166 rank=0 2167 current_core=0 2168 core_per_node=16 2169 init_exec=n 2170 2171 # Loop on the components 2172 for comp in ${config_ListOfComponents[*]} ; do 2173 2174 eval ExeNameIn=\${config_Executable_${comp}[0]} 2175 eval ExeNameOut=\${config_Executable_${comp}[1]} 2176 2177 # Not possible if oasis has an executable (i.e old version of oasis3) 2178 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then 2179 IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version" 2180 IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler" 2181 IGCM_debug_Verif_Exit 2182 fi 2183 2184 # Only if we really have an executable for the component : 2185 if [ "X${ExeNameOut}" != X\"\" ] ; then 2186 2187 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 2188 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 2189 2190 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 2191 echo "" >> script_${ExeNameOut}.ksh 2192 if [ ${comp_proc_omp_loc} -gt 1 ] ; then 2193 2194 # Check if the number of threads is correct 2195 case ${comp_proc_omp_loc} in 2196 2|4|8|16) 2197 IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads" 2198 ;; 2199 *) 2200 IGCM_debug_Exit "ERROR with OMP parameters !" 2201 IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads" 2202 IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible " 2203 IGCM_debug_Verif_Exit 2204 ;; 2205 esac 2206 echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 2207 echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 2208 echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 2209 echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 2210 fi 2211 # echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh 2212 # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh 2213 echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 2214 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 2215 2216 if [ ${init_exec} = y ] ; then 2217 EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 2218 else 2219 EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 2220 init_exec=y 2221 fi 2222 2223 # Build rankfile : method used to assign cores and nodes for the MPI process 2224 # Ex : 2225 #rank 0=curie5296 slot=0,1,2,3 2226 #rank 1=curie5296 slot=4,5,6,7 2227 # Example of final command : 2228 # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh 2229 # with script_lmdz.x.ksh : 2230 # #!/bin/ksh 2231 #export KMP_STACKSIZE=3g 2232 #export KMP_LIBRARY=turnaround 2233 #export MKL_SERIAL=YES 2234 #OMP_NUM_THREADS=4 2235 #./lmdz.x 2236 # 2237 2238 for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do 2239 (( index_host = current_core / core_per_node )) 2240 host_value=${host[${index_host}]} 2241 (( slot = current_core % core_per_node )) 2242 virg="," 2243 string_final="" 2244 for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do 2245 string=$index$virg 2246 string_final=$string_final$string 2247 done 2248 string_final=$( echo $string_final | sed "s/.$//" ) 2249 echo "rank $rank=$host_value slot=$string_final" >> rankfile 2250 (( rank = rank + 1 )) 2251 (( current_core = current_core + comp_proc_omp_loc )) 2252 done 2253 fi 2254 2255 done 2108 2256 fi 2109 done 2110 2111 # Then second loop on the components 2112 2113 for comp in ${config_ListOfComponents[*]} ; do 2114 2115 eval ExeNameIn=\${config_Executable_${comp}[0]} 2116 eval ExeNameOut=\${config_Executable_${comp}[1]} 2117 2118 # Only if we really have an executable for the component and not the coupler ie oasis: 2119 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then 2120 2121 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 2122 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 2123 echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 2124 fi 2125 done 2126 2127 EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file" 2128 2129 IGCM_sys_Chmod u+x run_file 2130 if ( $DEBUG_sys ) ; then 2131 echo "run_file contains : " 2132 cat run_file 2133 fi 2134 2135 else # Only one executable. launch it. 2257 2258 # Only one executable (SPMD mode). 2259 else 2136 2260 2137 2261 for comp in ${config_ListOfComponents[*]} ; do
Note: See TracChangeset
for help on using the changeset viewer.