Changeset 2659

Timestamp:

09/23/24 16:10:58 (4 days ago)

Author:

jderouillat

Message:

Integrating a Kokkos build into the XIOS one.

Location:

XIOS3/dev/XIOS_KOKKOS

Files:

• arch/arch-X64_JEANZAY_KOKKOS.env (modified) (1 diff)
• arch/arch-X64_JEANZAY_KOKKOS.fcm (modified) (1 diff)
• arch/arch-X64_JEANZAY_KOKKOS_GPU.env (modified) (1 diff)
• arch/arch-X64_JEANZAY_KOKKOS_GPU.fcm (modified) (2 diffs)
• make_xios (modified) (1 diff)

XIOS3/dev/XIOS_KOKKOS/arch/arch-X64_JEANZAY_KOKKOS.env

@@ -1 +1 @@
 module purge
+
+XIOS_ARCH=OPENMP
 
 module load gcc/9.1.0 nvidia-compilers/23.1 cuda/12.1.0
 
-export KOKKOS_HOME=${LSCEWORK}/KOKKOS/INSTALL/kokkos.traccs.oMP
-export PATH=${KOKKOS_HOME}/bin:${PATH}
-export LD_LIBRARY_PATH=${KOKKOS_HOME}/lib64:${LD_LIBRARY_PATH}
-export CPATH=${KOKKOS_HOME}/include:${CPATH}
-
 module load openmpi/4.1.1-cuda
+# Kokkos wrapper
 export OMPI_CXX=nvcc_wrapper
 module load hdf5/1.12.0-mpi-cuda

XIOS3/dev/XIOS_KOKKOS/arch/arch-X64_JEANZAY_KOKKOS.fcm

@@ -19 +19 @@
 
 %BASE_INC       -D__NONE__
-%BASE_LD        -lstdc++ -L${KOKKOS_HOME}/lib64 -lkokkoscore -ldl -L/gpfslocalsys/cuda/12.2.0/lib64 -lcudart -fopenmp
+%BASE_LD        -lstdc++ -L${KOKKOS_HOME}/lib64 -lkokkoscore -ldl -fopenmp -L/gpfslocalsys/cuda/12.2.0/lib64 -lcudart
 
 %CPP            cpp

XIOS3/dev/XIOS_KOKKOS/arch/arch-X64_JEANZAY_KOKKOS_GPU.env

@@ -1 +1 @@
 module purge
+
+XIOS_ARCH=NVIDIA
 
 module load gcc/9.1.0 nvidia-compilers/23.1 cuda/12.1.0
 
-export KOKKOS_HOME=${LSCEWORK}/KOKKOS/INSTALL/kokkos
-export PATH=${KOKKOS_HOME}/bin:${PATH}
-export LD_LIBRARY_PATH=${KOKKOS_HOME}/lib64:${LD_LIBRARY_PATH}
-export CPATH=${KOKKOS_HOME}/include:${CPATH}
-
 module load openmpi/4.1.1-cuda
+# Kokkos wrapper
 export OMPI_CXX=nvcc_wrapper
 module load hdf5/1.12.0-mpi-cuda

XIOS3/dev/XIOS_KOKKOS/arch/arch-X64_JEANZAY_KOKKOS_GPU.fcm

@@ -8 +8 @@
 %LINKER         mpif90
 
-%BASE_CFLAGS    -I${KOKKOS_HOME}/include -std=c++17 -extended-lambda -arch=sm_70 -fopenmp
+%BASE_CFLAGS    -I${KOKKOS_HOME}/include -std=c++17 -extended-lambda -arch=sm_70
 %PROD_CFLAGS    -O3 -w
 %DEV_CFLAGS     -g -traceback
@@ -19 +19 @@
 
 %BASE_INC       -D__NONE__
-%BASE_LD        -acc -gpu=cc70 -lstdc++ -L${KOKKOS_HOME}/lib64 -lkokkoscore -ldl -L/gpfslocalsys/cuda/12.2.0/lib64 -lcudart -fopenmp
+%BASE_LD        -acc -gpu=cc70 -lstdc++ -L${KOKKOS_HOME}/lib64 -lkokkoscore -ldl -L/gpfslocalsys/cuda/12.2.0/lib64 -lcudart -lcuda
 
 %CPP            cpp

XIOS3/dev/XIOS_KOKKOS/make_xios

@@ -168 +168 @@
 fi
 
+
+##############################
+# Installation de Kokkos
+export CURRENT_DIR=${PWD}
+module load cmake
+module load git
+git clone https://github.com/kokkos/kokkos.git extern/kokkos
+
+if [  ! -d extern/kokkos ]
+then
+    # github not reachable from compute nodes ?
+    echo "github is not not reachable from the resource used"
+    echo "Download kokkos in extern/kokkos bu your own."
+    exit 1
+fi
+
+if [ ! -f ${install_dir}/kokkos_${XIOS_ARCH}/lib64/libkokkoscore.a ]
+then 
+
+    export KOKKOS_SRC=${PWD}/extern/kokkos
+    mkdir -p ${KOKKOS_SRC}/build_${XIOS_ARCH}
+    cd ${KOKKOS_SRC}/build_${XIOS_ARCH}
+
+    if [[ "$XIOS_ARCH" == "NVIDIA" ]]        # GPU
+    then
+        cmake .. \
+              -DCMAKE_CXX_COMPILER=${KOKKOS_SRC}/bin/nvcc_wrapper \
+              -DCMAKE_INSTALL_PREFIX=${install_dir}/kokkos_${XIOS_ARCH} \
+              -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VOLTA70=ON
+        ##../generate_makefile.bash \
+        ##    --compiler=g++ \
+        ##    --prefix=${install_dir}/kokkos_${XIOS_ARCH} \
+        ##    --with-cuda --arch=Volta70  # --with-openmp
+    elif [[ "$XIOS_ARCH" == "OPENMP" ]]      # OMP
+    then
+        cmake .. \
+              -DCMAKE_CXX_COMPILER=${KOKKOS_SRC}/bin/nvcc_wrapper \
+              -DCMAKE_INSTALL_PREFIX=${install_dir}/kokkos_${XIOS_ARCH} \
+              -DKokkos_ENABLE_OPENMP=ON
+    elif [[ "$XIOS_ARCH" == "NONE" ]]      # Serial
+    then    
+        cmake .. \
+              -DCMAKE_CXX_COMPILER=${KOKKOS_SRC}/bin/nvcc_wrapper \
+              -DCMAKE_INSTALL_PREFIX=${install_dir}/kokkos_${XIOS_ARCH} \
+              -DKokkos_ENABLE_SERIAL=ON
+    fi
+
+    make -j ${job}
+    make install
+
+fi
+
+export KOKKOS_HOME=${install_dir}/kokkos_${XIOS_ARCH}
+export PATH=${KOKKOS_HOME}/bin:${PATH}
+export LD_LIBRARY_PATH=${KOKKOS_HOME}/lib64:${LD_LIBRARY_PATH}
+export CPATH=${KOKKOS_HOME}/include:${CPATH}
+
+cd ${CURRENT_DIR}
+# Installation de Kokkos (Fin)
+##############################
+
+
 rm -r $PWD/extern/netcdf4