Merge pull request CFT-HY#36 from timohyva/master

lumi-hip-CC.mk main.mk change for GPU app building optimazation; Antiperidic BC debug
timohyva · Jan 8, 2025 · 6ae558a · 6ae558a
2 parents 26f6a68 + 980749a
commit 6ae558a
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 12 deletions.
diff --git a/libraries/main.mk b/libraries/main.mk
@@ -200,7 +200,7 @@ build/%.cpt: %.cpp Makefile $(MAKEFILE_LIST) $(ALL_DEPEND) $(APP_HEADERS)
 	$(HILAPP) $(HILAPP_OPTS) $(APP_OPTS) $(HILA_OPTS) $< -o $@ $(HILAPP_TRAILING_OPTS)
 
 build/%.o : build/%.cpt
-	$(CC) $(CXXFLAGS) $(APP_OPTS) $(HILA_OPTS) $< -c -o $@
+	$(CC) $(APP_OPTS) $(HILA_OPTS) $(CXXFLAGS) $< -c -o $@
 
 build/%.cpt: $(LIBRARIES_DIR)/plumbing/%.cpp $(ALL_DEPEND) $(HILA_HEADERS)
 	@mkdir -p build
@@ -210,7 +210,6 @@ build/%.cpt: $(LIBRARIES_DIR)/tools/%.cpp $(ALL_DEPEND) $(HILA_HEADERS)
 	@mkdir -p build
 	$(HILAPP) $(HILAPP_OPTS) $(APP_OPTS) $(HILA_OPTS) $< -o $@ $(HILAPP_TRAILING_OPTS)
 
-
 # This one triggers only for cuda targets
 build/%.cpt: $(LIBRARIES_DIR)/plumbing/backend_gpu/%.cpp $(ALL_DEPEND) $(HILA_HEADERS)
 	@mkdir -p build

diff --git a/libraries/plumbing/backend_gpu/field_storage_backend.h b/libraries/plumbing/backend_gpu/field_storage_backend.h
@@ -290,6 +290,7 @@ void field_storage<T>::gather_comm_elements(T *buffer,
 
     // Call the kernel to build the list of elements
     int N_blocks = n / N_threads + 1;
+#ifdef SPECIAL_BOUNDARY_CONDITIONS    
     if (antiperiodic) {
 
         if constexpr (hila::has_unary_minus<T>::value) {
@@ -301,6 +302,10 @@ void field_storage<T>::gather_comm_elements(T *buffer,
         gather_comm_elements_kernel<<<N_blocks, N_threads>>>(*this, d_buffer, d_site_index, n,
                                                              lattice.field_alloc_size());
     }
+#else
+    gather_comm_elements_kernel<<<N_blocks, N_threads>>>(*this, d_buffer, d_site_index, n,
+                                                         lattice.field_alloc_size());
+#endif    
 
 #ifndef GPU_AWARE_MPI
     // Copy the result to the host

diff --git a/libraries/target_arch/lumi-hip-CC.mk b/libraries/target_arch/lumi-hip-CC.mk
@@ -6,7 +6,15 @@
 
 $(info ########################################################################)
 $(info Target lumi-hip-CC: remember to )
-$(info module load CrayEnv PrgEnv-cray craype-accel-amd-gfx90a cray-mpich rocm )
+$(info   module load CrayEnv PrgEnv-cray craype-accel-amd-gfx90a cray-mpich rocm )
+$(info )
+$(info Arcording to note of changes of  update of August-September 2024 from LUMI team,)
+$(info LUMI/24.04 is the only truly-supported software stack, you may condider use )
+$(info LUMI/24.03 and its toolchains to make workflow smooth. )
+$(info To load LUMI/24.03 with Cray Clang compiler, AMD GPU gfx90a libs, ROCm and toolchain run )
+$(info )
+$(info   module --force purge && module --force unload LUMI)
+$(info   module load LUMI/24.03 partition/G cpeCray/24.03 buildtools/24.03 rocm/6.0.3 )
 $(info ########################################################################)
 
 
@@ -20,9 +28,9 @@ LD := CC
 #CXXFLAGS  := -Ofast -flto -x c++ --std=c++17 -fno-rtti
 #CXXFLAGS := -g -x c++ --std=c++17
 # CXXFLAGS := -std=c++17 -fno-rtti --rocm-path=${ROCM_PATH} --offload-arch=gfx908 -x hip -fgpu-rdc
-CXXFLAGS := -std=c++17 -fno-rtti -O3 -xhip -fgpu-rdc --offload-arch=gfx90a -D__HIP_PLATFORM_AMD__=1
-CXXFLAGS += -D__HIP_PLATFORM_HCC__=1 
-CXXFLAGS += -D__HIP_ROCclr__ -D__HIP_ARCH_GFX90A__=1
+CXXFLAGS := -std=c++17 -fno-rtti -O3 -fgpu-rdc --offload-arch=gfx90a -D__HIP_PLATFORM_AMD__=1
+CXXFLAGS += -D__HIP_PLATFORM_AMD__=1 
+CXXFLAGS += -D__HIP_ROCclr__ -D__HIP_ARCH_GFX90A__=1 -xhip
 # CXXFLAGS := -std=c++17 --offload-arch=gfx908 -x c++
 #
 # hilapp needs to know where c++ system include files are located.  This is not a problem if
@@ -56,9 +64,12 @@ HILA_INCLUDES += -I${ROCM_PATH}/hipfft/include/ -I${ROCM_PATH}/hipcub/include/hi
 # Linker libraries and possible options
 
 # LDLIBS  := -lfftw3 -lfftw3f -lm
-LDFLAGS := $(CXXFLAGS) -fgpu-rdc --hip-link --rocm-path=${ROCM_PATH} -L${ROCM_PATH}/lib -lamdhip64
-LDFLAGS += -L${ROCM_PATH}/hipfft/lib/ -lhipfft
-LDFLAGS += -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib
+LDFLAGS := --offload-arch=gfx90a -D__HIP_PLATFORM_AMD__=1 --hip-link --rocm-path=${ROCM_PATH} -L${ROCM_PATH}/lib 
+LDFLAGS += -L${ROCM_PATH}/hipfft/lib
+LDFLAGS += -L${MPICH_DIR}/lib -L${CRAY_MPICH_ROOTDIR}/gtl/lib
+
+# libraries flags of amdhip, rocm-fft, mpich
+LDLIBS := -lamdhip64 -lhipfft -lmpi
 
 # These variables must be defined here
 #

diff --git a/libraries/target_arch/lumi.mk b/libraries/target_arch/lumi.mk
@@ -5,8 +5,17 @@
 #
 
 $(info ########################################################################)
-$(info Target lumi: remember to )
-$(info   module load PrgEnv-gnu cray-mpich cray-fftw )
+$(info Target lumi for LUMI-C: remember to )
+$(info   module load PrgEnv-gnu cray-mpich cray-fftw if you want to use Cray PE.)
+$(info )
+$(info Arcording to note of changes of  update of August-September 2024 from LUMI team,)
+$(info LUMI/24.04 is the only truly-supported software stack, you may condider use LUMI/24.03 and its toolchains to make workflow smooth.)
+$(info To load LUMI/24.03 with GNU compiler and its toolchains such as fftw, cmake run )
+$(info )
+$(info   module --force purge && module --force unload LUMI )
+$(info   module load LUMI/24.03 partition/C cpeGNU/24.03 buildtools/24.03 cray-fftw/3.3.10.7 )
+$(info )
+$(info LUMI/24.03 contains EasyBuild extra-packages building system, read LUMI documentaion to learn more.)
 $(info ########################################################################)
 
 
@@ -17,7 +26,7 @@ CC := CC
 LD := CC
 
 # Define compilation flags
-CXXFLAGS  := -Ofast -flto -x c++ --std=c++17 -fno-rtti
+CXXFLAGS  := -Ofast -flto=auto -x c++ --std=c++17 -fno-rtti
 #CXXFLAGS := -g -x c++ --std=c++17
 
 # hilapp needs to know where c++ system include files are located.  This is not a problem if