From 97dd146e0226c7f17911dc320e1b3b61d46bd6aa Mon Sep 17 00:00:00 2001 From: Alfio Lazzaro Date: Fri, 6 Mar 2020 15:44:52 +0100 Subject: [PATCH 1/6] Makefile limited to CP2K only, MISC improvements (#307) * Simply Makefile targets * GPU flags set by default * Remove Makefile from the documentation * Makefile only available for CP2K (initial step) * move files related to makefile to hidden .cp2k directory * Move makefile to .cp2k directory and document it * Update README.md * Override flags for ACC * Do not set __DBCSR_ACC if input configuration is provided * Add more MACROS description * Remove C-bindings from CP2K * Move build variants * add overview image * First draft of flags and macros * Add Macros * Comment macros * Add error on warnings * Add warnings and debug flags, add debug travis test * Add debugging symbols to coverage compilation * Avoid uninitialized error on cmake test * Add more debugging flags * Add more compiler checks (FPE traps...) * Update travis VM * Update tested compilers * Fix broken symb link * Makefile requires only BLAS * Avoid RMA for OpenMPI 2.1 and 3.1 * Adjust tests env for travis * Fix travis * Fix maybe-uninitialized * Remote test on macOS with BLAS because of FPE in BLAS * Remove warnings for conversion * Remove reloc for matmul * Upgrade to cmake 3.12 * Remove warning on type conversion * Include Shoshana's comment for HIP compilation * Add Shoshana's suggestion on user gender * Add compiler flags to the documentation --- .ci/daint.cscs.ch/Jenkinsfile | 11 - .ci/daint.cscs.ch/gnu-cmake-3.10.build.sh | 43 -- .cp2k/Makefile | 384 ++++++++++++++ .cp2k/README.md | 6 + .../build_utils => .cp2k}/check_archives.py | 0 {tools/build_utils => .cp2k}/makedep.py | 0 {tools/build_utils => .cp2k}/test_makedep.py | 0 .travis.yml | 114 ++-- CMakeLists.txt | 17 +- CONTRIBUTING.md | 4 +- DBCSR.md | 2 +- Makefile | 498 ------------------ Makefile.inc | 257 --------- README.md | 4 +- cmake/CompilerConfiguration.cmake | 12 +- .../f95-reshape-order-allocatable.f90 | 3 +- .../2-user-guide/1-installation/1-install.md | 9 +- .../1-installation/2-cmake-build-recipes.md | 8 +- .../1-installation/3-supported-compilers.md | 2 +- .../4-using-dbcsr-in-a-cmake-project.md | 2 +- .../3-developer-guide/1-tooling/index.md | 20 +- .../1-overview/dbcsr_mm_overview.png | Bin 0 -> 49976 bytes .../3-programming/1-overview/index.md | 29 + examples/Makefile | 33 -- src/acc/acc_event.cpp | 4 +- src/acc/libsmm_acc/README.md | 2 +- src/acc/libsmm_acc/libsmm_acc.cpp | 3 +- src/acc/libsmm_acc/libsmm_acc_benchmark.cpp | 2 +- ...ibsmm_acc_predictive_modeling_features.png | 2 +- src/mm/dbcsr_mm.F | 13 +- tests/CMakeLists.txt | 4 + tests/Makefile.inc | 45 -- tests/libsmm_acc_timer_multiply.cpp.template | 2 +- 33 files changed, 534 insertions(+), 1001 deletions(-) delete mode 100755 .ci/daint.cscs.ch/gnu-cmake-3.10.build.sh create mode 100644 .cp2k/Makefile create mode 100644 .cp2k/README.md rename {tools/build_utils => .cp2k}/check_archives.py (100%) rename {tools/build_utils => .cp2k}/makedep.py (100%) rename {tools/build_utils => .cp2k}/test_makedep.py (100%) delete mode 100644 Makefile delete mode 100644 Makefile.inc create mode 100644 docs/guide/3-developer-guide/3-programming/1-overview/dbcsr_mm_overview.png delete mode 100644 examples/Makefile delete mode 100644 tests/Makefile.inc diff --git a/.ci/daint.cscs.ch/Jenkinsfile b/.ci/daint.cscs.ch/Jenkinsfile index 25d5b8e992e..ceab014dfe6 100644 --- a/.ci/daint.cscs.ch/Jenkinsfile +++ b/.ci/daint.cscs.ch/Jenkinsfile @@ -59,17 +59,6 @@ pipeline { } } } - stage("GNU CMake 3.10 build-only") { - // we don't need a full build, - // just checking that we can still configure using CMake 3.10 - stages { - stage('build') { - steps { - run_batch("0:15:00", "gnu-cmake-3.10", "build") - } - } - } - } stage("Intel") { stages { stage('build') { diff --git a/.ci/daint.cscs.ch/gnu-cmake-3.10.build.sh b/.ci/daint.cscs.ch/gnu-cmake-3.10.build.sh deleted file mode 100755 index 0ce53567c1a..00000000000 --- a/.ci/daint.cscs.ch/gnu-cmake-3.10.build.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -l - -#SBATCH --export=ALL -#SBATCH --exclusive -#SBATCH --constraint="mc" -#SBATCH --partition="cscsci" -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=4 -#SBATCH --cpus-per-task=3 -#SBATCH --ntasks-per-core=1 # 1=no HT, 2=HT - -set -o errexit -set -o nounset -set -o pipefail - -module swap PrgEnv-cray PrgEnv-gnu -module load daint-gpu cudatoolkit -module unload cray-libsci_acc -module list - -set -o xtrace # do not set earlier to avoid noise from module - -umask 0002 # make sure group members can access the data - -mkdir --mode=0775 -p "${SCRATCH}/${BUILD_TAG}.gnu-cmake-3.10" -cd "${SCRATCH}/${BUILD_TAG}.gnu-cmake-3.10" - -cmake --version - -cmake \ - -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \ - -DCMAKE_CROSSCOMPILING_EMULATOR="" \ - -DUSE_CUDA=ON \ - -DUSE_CUBLAS=ON \ - -DWITH_GPU=P100 \ - -DBLAS_FOUND=ON -DBLAS_LIBRARIES="-lsci_gnu_mpi_mp" \ - -DLAPACK_FOUND=ON -DLAPACK_LIBRARIES="-lsci_gnu_mpi_mp" \ - -DMPIEXEC_EXECUTABLE="$(command -v srun)" \ - -DTEST_MPI_RANKS=${SLURM_NTASKS} \ - -DTEST_OMP_THREADS=${SLURM_CPUS_PER_TASK} \ - "${WORKSPACE}" |& tee -a "${STAGE_NAME}.out" - -make VERBOSE=1 -j |& tee -a "${STAGE_NAME}.out" diff --git a/.cp2k/Makefile b/.cp2k/Makefile new file mode 100644 index 00000000000..bd42b3bfab8 --- /dev/null +++ b/.cp2k/Makefile @@ -0,0 +1,384 @@ +# This Makefile is only used for the inclusion of the DBCSR library into CP2K. +# It is not supported by DBCSR development anymore. +############################################################ +#### DO NOT CHANGE AS PART OF THE DBCSR DEVELOPMENT!!!! #### +############################################################ +# CP2K team will update it, following the compilation flags suggested in the +# CP2K installation file. +# For this reason, this Makefile must be compatible with the CP2K compilation. +# +# For testing, by default, the Makefile compiles the OpenMP+MPI+CUDA toolchain +# with GNU compiler on a P100 GPU. +# Make sure that the env variable ${BLAS_PATH} is set. +# You can compile DBCSR by running from the DBCSR main directory with: +# +# > make -f .cp2k/Makefile +# +# Alternatively, you can provide an ARCH file as described in the CP2K installation: +# +# > make -f .cp2k/Makefile INCLUDEMAKE= +# +# The ARCH file can set the variables: +# +# CXX => C++ compiler, e.g. g++ or mpicxx +# CC => C compiler, e.g. gcc or mpicc +# FC => Fortran compiler, e.g. gfortran or mpifort +# LD => Linker, e.g. gfortran or mpifort +# AR => Archive command, e.g. ar -r +# CXXFLAGS => C++ compilation flags +# CFLAGS => C compilation flags +# FCFLAGS => Fortran compilation flags +# LDFLAGS => Linker flags +# LIBS => Libraries +# ACC => ACC can be nvcc (CUDA) or hipcc (HIP) +# ACCFLAGS => ACC flags +# GPUVER => +# - for CUDA, possible values correspond to NVIDIA GPUs: +# possible values are K20X, K40, K80, P100, V100 +# - for HIP, possible values correspond to NVIDIA and AMD GPUs: +# possible values are K20X, K40, K80, P100, V100, Mi50 +# +# Libraries for accelerator: +# - e.g. for CUDA: LIBS += -lstdc++ -lcudart -lnvrtc -lcuda -lcublas +# - e.g. for HIP: LIBS += -lstdc++ -lhiprtc -lhipblas +# + +# +SHELL = /bin/sh +# +# the home dir is taken from the current directory +# +DBCSRHOME := $(CURDIR) +DBCSRCP2K := $(DBCSRHOME)/.cp2k +MAKEFILE := $(DBCSRCP2K)/Makefile +LIBDIR := $(DBCSRHOME)/lib +OBJDIR := $(DBCSRHOME)/obj +PRETTYOBJDIR := $(OBJDIR)/prettified +TOOLSDIR := $(DBCSRHOME)/tools +FYPPEXE := $(TOOLSDIR)/build_utils/fypp/bin/fypp +SRCDIR := $(DBCSRHOME)/src +TESTSDIR := $(DBCSRHOME)/tests +INCLUDEMAKE := + +# Default Target ============================================================ +LIBNAME := dbcsr +LIBRARY := lib$(LIBNAME) +default_target: $(LIBRARY) + +# Set default values and read the configuration (if provided) =============== +MODDEPS = "lower" +CXX = mpicxx +CC = mpicc +FC = mpif90 +LD = mpif90 +AR = ar -r +CXXFLAGS = -O3 -g -std=c++11 -fopenmp +CFLAGS = -O3 -g +FCFLAGS = -D__parallel -O3 -g -fopenmp -std=f2008ts -ffree-form -fimplicit-none -ffree-line-length-512 -fno-omit-frame-pointer -funroll-loops \ + -Werror=aliasing -Werror=ampersand -Werror=c-binding-type \ + -Werror=intrinsic-shadow -Werror=intrinsics-std \ + -Werror=line-truncation \ + -Werror=tabs -Werror=target-lifetime \ + -Werror=underflow \ + -Werror=unused-but-set-variable -Werror=unused-variable \ + -Werror=unused-dummy-argument -Werror=conversion \ + -Werror=uninitialized -Wno-maybe-uninitialized +LDFLAGS = $(FCFLAGS) +LIBS = -L${BLAS_PATH}/lib -lblas -lstdc++ -lcudart -lnvrtc -lcuda +ACC = nvcc +ACCFLAGS = -O3 -g -w --std=c++11 +GPUVER = P100 + +ifneq (,$(INCLUDEMAKE)) +include $(INCLUDEMAKE) +endif + +# Read the version ========================================================== +include $(DBCSRHOME)/VERSION +ifeq ($(DATE),) +DATE = "Development Version" +endif + +# Set the compute version and ACCFLAGS ======================================= +ifneq ($(ACC),) +# Set ARCH version +ifeq ($(GPUVER),K20X) + ARCH_NUMBER = 35 +else ifeq ($(GPUVER),K40) + ARCH_NUMBER = 35 +else ifeq ($(GPUVER),K80) + ARCH_NUMBER = 37 +else ifeq ($(GPUVER),P100) + ARCH_NUMBER = 60 +else ifeq ($(GPUVER),V100) + ARCH_NUMBER = 70 +else ifeq ($(GPUVER),) # Default to the P100 + ARCH_NUMBER = 60 +# Remaining ARCH only for HIP +else ifneq (,$(findstring nvcc,$(ACC))) + $(error GPUVER requires HIP or not recognized) +else ifeq ($(GPUVER),Mi50) + ARCH_NUMBER = gfx906 +else + $(error GPUVER not recognized) +endif + +# enable ACC compilation +ifeq (,$(INCLUDEMAKE)) +FCFLAGS += -D__DBCSR_ACC +CFLAGS += -D__DBCSR_ACC +CXXFLAGS += -D__DBCSR_ACC +endif + +# If compiling with nvcc +ifneq (,$(findstring nvcc,$(ACC))) +override ACCFLAGS += -D__CUDA +CXXFLAGS += -D__CUDA +#if "-arch" has not yet been set in ACCFLAGS +ifeq (,$(findstring "-arch", $(ACCFLAGS))) +override ACCFLAGS += -arch sm_$(ARCH_NUMBER) +endif +# If compiling with hipcc +else ifneq (,$(findstring hipcc,$(ACC))) +override ACCFLAGS += -D__HIP +CXXFLAGS += -D__HIP +#if "--amdgpu-target" has not yet been set in ACCFLAGS +ifeq (,$(findstring "--amdgpu-target", $(ACCFLAGS))) +override ACCFLAGS += --amdgpu-target=$(ARCH_NUMBER) +endif +endif +endif + +# Set the configuration ============================================ +# +ifneq ($(LD_SHARED),) + ARCHIVE_EXT := .so +else + ARCHIVE_EXT := .a +endif + +# Declare PHONY targets ===================================================== +.PHONY : dirs makedep \ + default_target $(LIBRARY) \ + pretty prettyclean \ + clean realclean \ + version + +# Discover files and directories ============================================ +ALL_SRC_DIRS := $(shell find $(SRCDIR) -type d | awk '{printf("%s:",$$1)}') +ALL_SRC_DIRS += $(TESTSDIR) +LIBSMM_ACC_DIR := $(shell cd $(SRCDIR) ; find . -type d -name "libsmm_acc") +LIBSMM_ACC_ABS_DIR := $(shell find $(SRCDIR) -type d -name "libsmm_acc") + +ALL_PKG_FILES := $(shell find $(SRCDIR) -name "PACKAGE") +OBJ_SRC_FILES = $(shell cd $(SRCDIR); find . ! -name "dbcsr_api_c.F" -name "*.F") +OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -name "*.c") + +# if compiling with GPU acceleration +ifneq ($(ACC),) + # All *.cpp files belong to the accelerator backend + OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . ! -name "acc_cuda.cpp" ! -name "acc_hip.cpp" -name "*.cpp") + # if compiling with nvcc + ifneq (,$(findstring nvcc,$(ACC))) + OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../cuda/acc_cuda.cpp + # Exclude autotuning files + OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . ! -name "tune_*_exe*_part*.cu" ! -name "tune_*_exe*_main*.cu" -name "*.cu") + # if compiling with hipcc + else ifneq (,$(findstring hipcc,$(ACC))) + OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../hip/acc_hip.cpp + endif +endif + +# OBJECTS used for pretty +ALL_OBJECTS := $(addsuffix .o, $(basename $(notdir $(OBJ_SRC_FILES)))) +ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.F")))) +ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.c")))) +ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.cpp")))) +ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.cu")))) + +# Included files used by Fypp preprocessor and standard includes +INCLUDED_SRC_FILES := $(filter-out base_uses.f90, $(notdir $(shell find $(SRCDIR) -name "*.f90"))) +INCLUDED_SRC_FILES += $(notdir $(shell find $(TESTSDIR) -name "*.f90")) + +# Include also source files which won't compile into an object file +ALL_SRC_FILES = $(strip $(subst $(NULL) .,$(NULL) $(SRCDIR),$(NULL) $(OBJ_SRC_FILES))) +ALL_SRC_FILES += $(filter-out base_uses.f90, $(shell find $(SRCDIR) -name "*.f90")) +ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.h") +ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.hpp") + +# stage 1: create dirs and run makedep.py. +# Afterwards, call make recursively again with -C $(OBJDIR) and INCLUDE_DEPS=true +ifeq ($(INCLUDE_DEPS),) +$(LIBRARY): dirs makedep + @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME) + +dirs: + @mkdir -p $(OBJDIR) + @mkdir -p $(LIBDIR) + +version: + @echo "DBCSR Version: "$(MAJOR)"."$(MINOR)"."$(PATCH)" ("$(DATE)")" + +else +# stage 2: Include $(OBJDIR)/all.dep, expand target all, and get list of dependencies. + +# Check if FYPP is available =============================================== +ifeq (, $(shell which $(FYPPEXE) 2>/dev/null )) +$(error "No FYPP submodule available, please read README.md on how to properly download DBCSR") +endif + +endif + +clean: + rm -f $(TESTSDIR)/libsmm_acc_unittest_multiply.cpp + rm -f $(TESTSDIR)/libsmm_acc_timer_multiply.cpp + rm -rf $(OBJDIR) + rm -f $(LIBSMM_ACC_ABS_DIR)/parameters.h $(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h $(LIBSMM_ACC_ABS_DIR)/*.so + +# +# delete the intermediate files, the programs and libraries and anything that might be in the objdir or libdir directory +# Use this if you want to fully rebuild an executable (for a given compiler) +# +realclean: clean + rm -rf $(LIBDIR) $(PREFIX) + rm -rf `find $(DBCSRHOME) -name "*.pyc"` + rm -rf `find $(DBCSRHOME) -name "*.callgraph"` + +# Prettyfier stuff ========================================================== +vpath %.pretty $(PRETTYOBJDIR) + +pretty: $(addprefix $(PRETTYOBJDIR)/, $(ALL_OBJECTS:.o=.pretty)) $(addprefix $(PRETTYOBJDIR)/, $(INCLUDED_SRC_FILES:.f90=.pretty_included)) + +prettyclean: + -rm -rf $(PRETTYOBJDIR) + +define pretty_func + @mkdir -p $(PRETTYOBJDIR) + @touch $2 + $(TOOLSDIR)/fprettify/fprettify.py --disable-whitespace $1 +endef + +$(PRETTYOBJDIR)/%.pretty: %.F + $(call pretty_func, $<, $@) + +$(PRETTYOBJDIR)/%.pretty_included: %.f90 + $(call pretty_func, $<, $@) + +$(PRETTYOBJDIR)/%.pretty: %.c +# TODO: call indent here? + @mkdir -p $(PRETTYOBJDIR) + @touch $@ + +$(PRETTYOBJDIR)/%.pretty: %.cpp +# TODO: call indent here? + @mkdir -p $(PRETTYOBJDIR) + @touch $@ + +$(PRETTYOBJDIR)/%.pretty: %.cu +# TODO: call indent here? + @mkdir -p $(PRETTYOBJDIR) + @touch $@ + +# Libsmm_acc stuff ========================================================== +$(LIBSMM_ACC_ABS_DIR)/parameters.h: $(LIBSMM_ACC_ABS_DIR)/generate_parameters.py $(wildcard $(LIBSMM_ACC_ABS_DIR)/parameters_*.txt) + cd $(LIBSMM_ACC_ABS_DIR); ./generate_parameters.py --gpu_version=$(GPUVER) + +$(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h: $(LIBSMM_ACC_ABS_DIR)/generate_kernels.py $(wildcard $(LIBSMM_ACC_ABS_DIR)/kernels/*.h) + cd $(LIBSMM_ACC_ABS_DIR); ./generate_kernels.py + + +# automatic dependency generation =========================================== +MAKEDEPMODE = "normal" +ifeq ($(HACKDEP),yes) +MAKEDEPMODE = "hackdep" +endif + +# this happens on stage 1 +makedep: $(ALL_SRC_FILES) $(ALL_PKG_FILES) dirs +ifeq ($(LD_SHARED),) + @echo "Removing stale archives ... " + @$(DBCSRCP2K)/check_archives.py $(firstword $(AR)) $(SRCDIR) $(LIBDIR) +endif + @echo "Resolving dependencies ... " + @$(DBCSRCP2K)/makedep.py $(OBJDIR)/all.dep dbcsr $(MODDEPS) $(MAKEDEPMODE) $(ARCHIVE_EXT) $(SRCDIR) $(OBJ_SRC_FILES) + +# on stage 2, load the rules generated by makedep.py +ifeq ($(INCLUDE_DEPS), true) +include $(OBJDIR)/all.dep +endif + + +# ================= Stuff need for compiling (stage 2) ====================== +# These rules are executed in a recursive call to make -C $(OBJDIR) +# The change of $(CURDIR) allows to find targets without abs paths and vpaths. + + +### Slave rules ### +vpath %.F $(ALL_SRC_DIRS) +vpath %.h $(ALL_SRC_DIRS) +vpath %.hpp $(ALL_SRC_DIRS) +vpath %.f90 $(ALL_SRC_DIRS) +vpath %.cu $(ALL_SRC_DIRS) +vpath %.c $(ALL_SRC_DIRS) +vpath %.cpp $(ALL_SRC_DIRS) + +# $(FCLOGPIPE) can be used to store compiler output, e.g. warnings, for each F-file separately. +# This is used e.g. by the convention checker. + +FYPPFLAGS ?= -n + +%.o: %.F + $(FYPPEXE) $(FYPPFLAGS) $< $*.F90 + $(FC) -c $(FCFLAGS) -D__SHORT_FILE__="\"$(notdir $<)\"" -I'$(dir $<)' -I'$(SRCDIR)' $*.F90 $(FCLOGPIPE) + +%.mod: %.o + @true + +%.o: %.c + $(CC) -c $(CFLAGS) $< + +# Compile the CUDA/HIP files +ifneq ($(ACC),) +%.o: %.cpp + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< + +libsmm_acc.o: libsmm_acc.cpp parameters.h smm_acc_kernels.h + $(ACC) -c $(ACCFLAGS) -DARCH_NUMBER=$(ARCH_NUMBER) $< + +libsmm_acc_benchmark.o: libsmm_acc_benchmark.cpp parameters.h + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< + +libsmm_acc_init.o: libsmm_acc_init.cpp libsmm_acc_init.h parameters.h + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< +endif + +ifneq (,$(findstring nvcc,$(ACC))) +%.o: %.cpp + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< + +acc_cuda.o: acc_cuda.cpp acc_cuda.h + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< + +%.o: %.cu + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< +else ifneq (,$(findstring hipcc,$(ACC))) +%.o: %.cpp + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< + +acc_hip.o: acc_hip.cpp acc_hip.h + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< + +hipblas.o: hipblas.cpp + $(ACC) -c $(ACCFLAGS) -I'$(SRCDIR)' $< +endif + +$(LIBDIR)/%: +ifneq ($(LD_SHARED),) + @echo "Creating shared library $@" + @$(LD_SHARED) $(LDFLAGS) -o $(@:.a=.so) $^ $(LIBS) +else + @echo "Updating archive $@" + @$(AR) $@ $? +endif + +#EOF diff --git a/.cp2k/README.md b/.cp2k/README.md new file mode 100644 index 00000000000..5c5e5d9f786 --- /dev/null +++ b/.cp2k/README.md @@ -0,0 +1,6 @@ + +This directory is used for the inclusion of the DBCSR library in CP2K. +It is not supported by DBCSR development anymore and eventually it will disappear in the next releases. + +**DO NOT CHANGE AS PART OF THE DBCSR DEVELOPMENT!!!!** + diff --git a/tools/build_utils/check_archives.py b/.cp2k/check_archives.py similarity index 100% rename from tools/build_utils/check_archives.py rename to .cp2k/check_archives.py diff --git a/tools/build_utils/makedep.py b/.cp2k/makedep.py similarity index 100% rename from tools/build_utils/makedep.py rename to .cp2k/makedep.py diff --git a/tools/build_utils/test_makedep.py b/.cp2k/test_makedep.py similarity index 100% rename from tools/build_utils/test_makedep.py rename to .cp2k/test_makedep.py diff --git a/.travis.yml b/.travis.yml index af08662dfba..a4cef5fc1af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ os: - linux - osx -dist: xenial -osx_image: xcode11.2 +dist: bionic +osx_image: xcode11.3 addons: apt: @@ -33,27 +33,73 @@ env: - LIBXSMM_VERSION=1.14 - CMAKE_VERSION=3.16.20191124-g54082ed - NINJA_VERSION=1.9.0.g99df1.kitware.dyndep-1.jobserver-1 - matrix: + jobs: - USE_MPI=ON USE_OPENMP=ON USE_SMM=blas - USE_MPI=OFF USE_OPENMP=ON USE_SMM=blas - USE_MPI=ON USE_OPENMP=OFF USE_SMM=blas - - USE_MPI=OFF USE_OPENMP=OFF USE_SMM=blas + - USE_MPI=ON USE_OPENMP=ON USE_SMM=libxsmm - USE_MPI=OFF USE_OPENMP=ON USE_SMM=libxsmm + - USE_MPI=ON USE_OPENMP=OFF USE_SMM=libxsmm - USE_MPI=OFF USE_OPENMP=OFF USE_SMM=libxsmm -matrix: - include: - - os: linux - python: 3.6 - -matrix: +jobs: # the following jobs inherit only the first OS exclude: - os: osx env: USE_MPI=ON USE_OPENMP=ON USE_SMM=blas - os: osx env: USE_MPI=OFF USE_OPENMP=ON USE_SMM=blas + - os: osx + env: USE_MPI=ON USE_OPENMP=OFF USE_SMM=blas + - os: osx + env: USE_MPI=ON USE_OPENMP=ON USE_SMM=libxsmm - os: osx env: USE_MPI=OFF USE_OPENMP=ON USE_SMM=libxsmm + include: + - name: "Run pre-commit hook on changed files" + if: tag IS NOT present + script: + - set -o pipefail + # fetch remotes first to ensure that referenced branches (other than the default) are available + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + - git fetch + - git diff --name-only --diff-filter=AM HEAD "origin/${TRAVIS_BRANCH}" | xargs pre-commit run --files + + - name: "Run pre-commit hook 'check-ast' on all files" + if: tag IS NOT present + script: + - pre-commit run --hook-stage manual --all-files check-ast + - stage: release + name: "Build release tarball and documentation and deploy to GitHub" + if: tag IS present + script: + - mkdir -p build + - cd build + - cmake .. + - make dist doc + - touch doc/.nojekyll + deploy: + - provider: releases + token: # encrypted token of the `dbcsr` GitHub service user + secure: ID+AX1LOPs+BRGpZiYgP0H9gvOBF/a/o1tNUNDY7Mbo6uShpwVcmYxpqW/2zbWrHXeG4TNZEATED2Yw1nSSp0wj7z/t6ScDuttvYwuN8AExauW7y4WrU5o3qGCGuc9ClDmj3lnQsjqjkmFkqZasmnh2MKHldcDM5cluNs1o/48a/nJ55TEsplKvsPn6gzi+XIs0zY/yIFEZxWEeXs7SlR3Jr+0nDMBHdllnEV/AxdQGQ7y2w6sXN26B7PZNM97kQ8qBGUPnnj41vHtAvT1rGu/N7TcRu+P0n9JmPtC+ZpzBNU8RfI0YJgFOjhORP9gwUNrwNSLZBqJECo2+s6QhjZ7x+UVF9CBQePUnRPJxMjkFVKqfykd7NtREaFJiwwfcWXSq7naVj1n4aNN19jO1myeSP/4WAGMBmEHXSlaDj1jqyaijwPC9YxVeMovTwlDdg0Iv/ZsjE4+MFcgyUU8f0h5x94gbsxgmKPUMaQeuMhdw41wAQm0KcDANwWNyiiGiyw5h0QJcjvg1/HDsrfID0vgqXXMXCfbLEMwpJLWj8F5Htc/D2gGfvepLy9+chqGF/53ntpTPEkF9V1f8ke6QZv7cJhi7U7IOmxWE6OFObPB1auT6g+Got10zkzNDWulk7g6thIFZlarpYqVKUWMfhb3i0ndJLk24L6iJhRuxtp90= + cleanup: true + file_glob: true + file: dist/* + draft: true + # the following 2 lines work around an issue in TravisCI with draft=true: + tag_name: $TRAVIS_TAG + target_commitish: $TRAVIS_COMMIT + on: + repo: cp2k/dbcsr + tags: true # default is "restricted by branch" + - provider: pages + token: # encrypted token of the `dbcsr` GitHub service user + secure: ID+AX1LOPs+BRGpZiYgP0H9gvOBF/a/o1tNUNDY7Mbo6uShpwVcmYxpqW/2zbWrHXeG4TNZEATED2Yw1nSSp0wj7z/t6ScDuttvYwuN8AExauW7y4WrU5o3qGCGuc9ClDmj3lnQsjqjkmFkqZasmnh2MKHldcDM5cluNs1o/48a/nJ55TEsplKvsPn6gzi+XIs0zY/yIFEZxWEeXs7SlR3Jr+0nDMBHdllnEV/AxdQGQ7y2w6sXN26B7PZNM97kQ8qBGUPnnj41vHtAvT1rGu/N7TcRu+P0n9JmPtC+ZpzBNU8RfI0YJgFOjhORP9gwUNrwNSLZBqJECo2+s6QhjZ7x+UVF9CBQePUnRPJxMjkFVKqfykd7NtREaFJiwwfcWXSq7naVj1n4aNN19jO1myeSP/4WAGMBmEHXSlaDj1jqyaijwPC9YxVeMovTwlDdg0Iv/ZsjE4+MFcgyUU8f0h5x94gbsxgmKPUMaQeuMhdw41wAQm0KcDANwWNyiiGiyw5h0QJcjvg1/HDsrfID0vgqXXMXCfbLEMwpJLWj8F5Htc/D2gGfvepLy9+chqGF/53ntpTPEkF9V1f8ke6QZv7cJhi7U7IOmxWE6OFObPB1auT6g+Got10zkzNDWulk7g6thIFZlarpYqVKUWMfhb3i0ndJLk24L6iJhRuxtp90= + cleanup: true + keep_history: true + local_dir: build/doc/ + on: + repo: cp2k/dbcsr + tags: true # default is "restricted by branch" install: - DEPS_DIR="${HOME}/deps" @@ -118,51 +164,3 @@ after_success: cmake --build . -- cov-info bash <(curl -s https://codecov.io/bash) -f coverage.info || echo "Codecov did not collect coverage reports" fi - -jobs: # the following jobs inherit only the first OS - include: - - name: "Run pre-commit hook on changed files" - if: tag IS NOT present - script: - - set -o pipefail - # fetch remotes first to ensure that referenced branches (other than the default) are available - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch - - git diff --name-only --diff-filter=AM HEAD "origin/${TRAVIS_BRANCH}" | xargs pre-commit run --files - - - name: "Run pre-commit hook 'check-ast' on all files" - if: tag IS NOT present - script: - - pre-commit run --hook-stage manual --all-files check-ast - - stage: release - name: "Build release tarball and documentation and deploy to GitHub" - if: tag IS present - script: - - mkdir -p build - - cd build - - cmake .. - - make dist doc - - touch doc/.nojekyll - deploy: - - provider: releases - api_key: # encrypted token of the `dbcsr` GitHub service user - secure: ID+AX1LOPs+BRGpZiYgP0H9gvOBF/a/o1tNUNDY7Mbo6uShpwVcmYxpqW/2zbWrHXeG4TNZEATED2Yw1nSSp0wj7z/t6ScDuttvYwuN8AExauW7y4WrU5o3qGCGuc9ClDmj3lnQsjqjkmFkqZasmnh2MKHldcDM5cluNs1o/48a/nJ55TEsplKvsPn6gzi+XIs0zY/yIFEZxWEeXs7SlR3Jr+0nDMBHdllnEV/AxdQGQ7y2w6sXN26B7PZNM97kQ8qBGUPnnj41vHtAvT1rGu/N7TcRu+P0n9JmPtC+ZpzBNU8RfI0YJgFOjhORP9gwUNrwNSLZBqJECo2+s6QhjZ7x+UVF9CBQePUnRPJxMjkFVKqfykd7NtREaFJiwwfcWXSq7naVj1n4aNN19jO1myeSP/4WAGMBmEHXSlaDj1jqyaijwPC9YxVeMovTwlDdg0Iv/ZsjE4+MFcgyUU8f0h5x94gbsxgmKPUMaQeuMhdw41wAQm0KcDANwWNyiiGiyw5h0QJcjvg1/HDsrfID0vgqXXMXCfbLEMwpJLWj8F5Htc/D2gGfvepLy9+chqGF/53ntpTPEkF9V1f8ke6QZv7cJhi7U7IOmxWE6OFObPB1auT6g+Got10zkzNDWulk7g6thIFZlarpYqVKUWMfhb3i0ndJLk24L6iJhRuxtp90= - skip_cleanup: true - file_glob: true - file: dist/* - draft: true - # the following 2 lines work around an issue in TravisCI with draft=true: - tag_name: $TRAVIS_TAG - target_commitish: $TRAVIS_COMMIT - on: - repo: cp2k/dbcsr - tags: true # default is "restricted by branch" - - provider: pages - github-token: # encrypted token of the `dbcsr` GitHub service user - secure: ID+AX1LOPs+BRGpZiYgP0H9gvOBF/a/o1tNUNDY7Mbo6uShpwVcmYxpqW/2zbWrHXeG4TNZEATED2Yw1nSSp0wj7z/t6ScDuttvYwuN8AExauW7y4WrU5o3qGCGuc9ClDmj3lnQsjqjkmFkqZasmnh2MKHldcDM5cluNs1o/48a/nJ55TEsplKvsPn6gzi+XIs0zY/yIFEZxWEeXs7SlR3Jr+0nDMBHdllnEV/AxdQGQ7y2w6sXN26B7PZNM97kQ8qBGUPnnj41vHtAvT1rGu/N7TcRu+P0n9JmPtC+ZpzBNU8RfI0YJgFOjhORP9gwUNrwNSLZBqJECo2+s6QhjZ7x+UVF9CBQePUnRPJxMjkFVKqfykd7NtREaFJiwwfcWXSq7naVj1n4aNN19jO1myeSP/4WAGMBmEHXSlaDj1jqyaijwPC9YxVeMovTwlDdg0Iv/ZsjE4+MFcgyUU8f0h5x94gbsxgmKPUMaQeuMhdw41wAQm0KcDANwWNyiiGiyw5h0QJcjvg1/HDsrfID0vgqXXMXCfbLEMwpJLWj8F5Htc/D2gGfvepLy9+chqGF/53ntpTPEkF9V1f8ke6QZv7cJhi7U7IOmxWE6OFObPB1auT6g+Got10zkzNDWulk7g6thIFZlarpYqVKUWMfhb3i0ndJLk24L6iJhRuxtp90= - skip-cleanup: true - keep-history: true - local-dir: build/doc/ - on: - repo: cp2k/dbcsr - tags: true # default is "restricted by branch" diff --git a/CMakeLists.txt b/CMakeLists.txt index ff8bfdd1f64..16689eb87d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.10) +cmake_minimum_required (VERSION 3.12) # include our cmake snippets set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -111,18 +111,16 @@ find_package(PkgConfig) # In CMake <3.15, the system is searched before the virtual environment. if (NOT Python_EXECUTABLE) # If the python interpreter isn't specified as a command line option, look for it: - if (CMAKE_VERSION VERSION_LESS 3.12) - find_package(PythonInterp REQUIRED) - set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) - else () - find_package(Python COMPONENTS Interpreter REQUIRED) - endif () + find_package(Python COMPONENTS Interpreter REQUIRED) endif () # =================================== MPI if (USE_MPI) get_property(REQUIRED_MPI_COMPONENTS GLOBAL PROPERTY ENABLED_LANGUAGES) list(REMOVE_ITEM REQUIRED_MPI_COMPONENTS CUDA) # CUDA does not have an MPI component + if (NOT CMAKE_CROSSCOMPILING) # when cross compiling, assume the users know what they are doing + set(MPI_DETERMINE_LIBRARY_VERSION TRUE) + endif () find_package(MPI COMPONENTS ${REQUIRED_MPI_COMPONENTS} REQUIRED) if (NOT MPI_Fortran_HAVE_F90_MODULE) @@ -131,6 +129,11 @@ The listed MPI implementation does not provide the required mpi.mod interface. \ When using the GNU compiler in combination with Intel MPI, please use the \ Intel MPI compiler wrappers. Check the INSTALL.md for more information.") endif () + if ("${MPI_Fortran_LIBRARY_VERSION_STRING}" MATCHES "Open MPI v2.1" OR "${MPI_Fortran_LIBRARY_VERSION_STRING}" MATCHES "Open MPI v3.1") + message(WARNING + "RMA with ${MPI_Fortran_LIBRARY_VERSION_STRING} is not supported due to issues with its implementation." + " Please use a newer version of OpenMPI or switch to MPICH if you plan on using MPI-RMA.") + endif () endif () # =================================== OpenMP and OpenMP/offload backend diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a243c94c21a..f89ec32ed83 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ The core of DBCSR is written in Fortran. All other languages must be supported t There is a single [API](./src/dbcsr_api.F) file for DBCSR, which is provided for external usage only. **Do not use the API for any internal DBCSR development!** Packages build on top of DBCSR, for example [DBCSR Tensors](./src/tensors), **must only use** the DBCSR API. Note that any change in the APIs will require a major release of the library. -We support Make and CMake for compilation, please keep the build system updated when adding/removing files. When adding new functions, it is extremely important to provide simple test programs, aka "unit tests", to check whether these functions are performing as they should. The directory [test](./tests) serves as infrastructure for that. If you do not feel comfortable integrating these tests with the build system, please notify the other developers. +We support CMake for compilation, please keep the build system updated when adding/removing files. When adding new functions, it is extremely important to provide simple test programs, aka "unit tests", to check whether these functions are performing as they should. The directory [test](./tests) serves as infrastructure for that. If you do not feel comfortable integrating these tests with the build system, please notify the other developers. Having examples (under the directory [examples](./examples)) is also appreciated. They must be independent of the DBCSR compilation and only use the DBCSR APIs. @@ -23,7 +23,7 @@ Please make sure that you follow the following code conventions (based on [CP2K 7. Use the routines from [MPI wrappers](./src/mpi) instead of calling MPI directly. 8. Don't use `UNIT=*` in `WRITE` or `PRINT` statements. Instead, request a unit from the logger: `iw=dbcsr_logger_get_default_unit_nr()` and write only if you actually received a unit: `IF(iw>0) WRITE (UNIT=iw, ,,,)`. 9. Avoid to use `STOP`. Prefer the DBCSR error handlers: `DBCSR_WARN`, `DBCSR_ABORT`, `DBCSR_ASSERT`. -10. Each preprocessor flag should start with two underscores and be documented in the [Makefile.inc](./Makefile.inc). +10. Each preprocessor flag should start with two underscores and be documented in the [documentation](./docs/guide/3-developer-guide/3-programming/1-overview/index.md#list-of-macros-used-in-the-code). 11. All routines in the API must start with the `dbcsr_` namespace. For submodules API (e.g. [DBCSR Tensors](./src/tensors)), each function has to start with the `dbcsr__` namespace. **Most important, please avoid committing dead code and useless comments!** diff --git a/DBCSR.md b/DBCSR.md index 8ebdb79048c..a42f2126269 100644 --- a/DBCSR.md +++ b/DBCSR.md @@ -69,7 +69,7 @@ To cite the DBCSR software library, use: ```latex @misc{dbcsr-software, author = {The CP2K Developers Group}, - title = {DBCSR: Distributed Block Compressed Sparse Row matrix library}, + title = {{DBCSR: Distributed Block Compressed Sparse Row matrix library}}, publisher = {GitHub}, journal = {GitHub repository}, year = {2020}, diff --git a/Makefile b/Makefile deleted file mode 100644 index db005a51d3c..00000000000 --- a/Makefile +++ /dev/null @@ -1,498 +0,0 @@ -# -SHELL = /bin/sh -# -# the home dir is taken from the current directory -# -DBCSRHOME ?= $(CURDIR) -MAKEFILE := $(DBCSRHOME)/Makefile -BINDIR := $(DBCSRHOME)/bin -LIBDIR ?= $(DBCSRHOME)/lib -OBJDIR ?= $(DBCSRHOME)/obj -PRETTYOBJDIR := $(OBJDIR)/prettified -TOOLSRC := $(DBCSRHOME)/tools -FYPPEXE ?= $(TOOLSRC)/build_utils/fypp/bin/fypp -SRCDIR := $(DBCSRHOME)/src -TESTSDIR := $(DBCSRHOME)/tests -EXAMPLESDIR := $(DBCSRHOME)/examples -PREFIX ?= $(DBCSRHOME)/install -INCLUDEMAKE ?= $(DBCSRHOME)/Makefile.inc -NPROCS ?= 1 - -# Default Target ============================================================ -LIBNAME := dbcsr -LIBRARY := lib$(LIBNAME) -default_target: $(LIBRARY) - -# Read the configuration ==================================================== -MODDEPS = "lower" -include $(INCLUDEMAKE) - -# Read the version ========================================================== -include $(DBCSRHOME)/VERSION -ifeq ($(DATE),) -DATE = "Development Version" -endif - -# Set the compute version and NVFLAGS ======================================= -ifneq ($(NVCC),) -ifeq ($(GPUVER),K20X) - ARCH_NUMBER = 35 -else ifeq ($(GPUVER),K40) - ARCH_NUMBER = 35 -else ifeq ($(GPUVER),K80) - ARCH_NUMBER = 37 -else ifeq ($(GPUVER),P100) - ARCH_NUMBER = 60 -else ifeq ($(GPUVER),V100) - ARCH_NUMBER = 70 -else ifeq ($(GPUVER),Mi50) - ARCH_NUMBER = gfx906 -else ifeq ($(GPUVER),) # Default to the P100 - ARCH_NUMBER = 60 -else - $(error GPUVER not recognized) -endif - -ifneq ($(ARCH_NUMBER),) -# If compiling with nvcc -ifneq (,$(findstring nvcc,$(NVCC))) -NVFLAGS += -D__CUDA # add the flag for compilation with CUDA -#if "-arch" has not yet been set in NVFLAGS -ifeq ($(findstring "-arch", $(NVFLAGS)), '') -NVFLAGS += -arch sm_$(ARCH_NUMBER) -endif -# If compiling with hipcc -else ifneq (,$(findstring hipcc,$(NVCC))) -#if "--amdgpu-target" has not yet been set in NVFLAGS -ifeq ($(findstring "--amdgpu-target", $(NVFLAGS)), '') -NVFLAGS += --amdgpu-target=$(ARCH_NUMBER) -endif -endif -endif -endif - -# Test programs ========================================================= -include $(TESTSDIR)/Makefile.inc -BIN_TESTS := $(sort $(addprefix $(TESTSDIR)/, $(SRC_TESTS))) - -# Set the configuration ============================================ -# the only binaries for the moment are the tests -BIN_FILES := $(BIN_TESTS) -BIN_NAMES := $(basename $(notdir $(BIN_FILES))) -# -ifneq ($(LD_SHARED),) - ARCHIVE_EXT := .so -else - ARCHIVE_EXT := .a -endif - -# Declare PHONY targets ===================================================== -.PHONY : $(BIN_NAMES) \ - dirs makedep \ - default_target $(LIBRARY) all \ - toolversions \ - toolflags \ - pretty prettyclean \ - install clean realclean help \ - version test - -# Discover files and directories ============================================ -ALL_SRC_DIRS := $(shell find $(SRCDIR) -type d | awk '{printf("%s:",$$1)}') -ALL_SRC_DIRS += $(TESTSDIR) -LIBSMM_ACC_DIR := $(shell cd $(SRCDIR) ; find . -type d -name "libsmm_acc") -LIBSMM_ACC_ABS_DIR := $(shell find $(SRCDIR) -type d -name "libsmm_acc") - -ALL_PKG_FILES := $(shell find $(SRCDIR) -name "PACKAGE") -OBJ_SRC_FILES = $(shell cd $(SRCDIR); find . ! -name "dbcsr_api_c.F" -name "*.F") -OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . -name "*.c") - -# if compiling with GPU acceleration -ifneq ($(NVCC),) - # All *.cpp files belong to the accelerator backend - OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . ! -name "acc_cuda.cpp" ! -name "acc_hip.cpp" -name "*.cpp") - # if compiling with nvcc - ifneq (,$(findstring nvcc,$(NVCC))) - OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../cuda/acc_cuda.cpp - # Exclude autotuning files - OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . ! -name "tune_*_exe*_part*.cu" ! -name "tune_*_exe*_main*.cu" -name "*.cu") - # if compiling with hipcc - else ifneq (,$(findstring hipcc,$(NVCC))) - OBJ_SRC_FILES += $(LIBSMM_ACC_DIR)/../hip/acc_hip.cpp - # Exclude autotuning files - OBJ_SRC_FILES += $(shell cd $(SRCDIR); find . ! -name "tune_*_exe*_part*.cpp" ! -name "tune_*_exe*_main*.cpp" -name "*.cpp") - endif -endif - -ifneq ($(CINT),) -OBJ_SRC_FILES += ./dbcsr_api_c.F -PUBLICHEADERS += $(SRCDIR)/dbcsr.h -endif - -# OBJECTS used for pretty -ALL_OBJECTS := $(addsuffix .o, $(basename $(notdir $(OBJ_SRC_FILES)))) -ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.F")))) -ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.c")))) -ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.cpp")))) -ALL_OBJECTS += $(addsuffix .o, $(basename $(notdir $(shell cd $(TESTSDIR); find . -name "*.cu")))) - -# Included files used by Fypp preprocessor and standard includes -INCLUDED_SRC_FILES := $(filter-out base_uses.f90, $(notdir $(shell find $(SRCDIR) -name "*.f90"))) -INCLUDED_SRC_FILES += $(notdir $(shell find $(TESTSDIR) -name "*.f90")) - -# Include also source files which won't compile into an object file -ALL_SRC_FILES = $(strip $(subst $(NULL) .,$(NULL) $(SRCDIR),$(NULL) $(OBJ_SRC_FILES))) -ALL_SRC_FILES += $(filter-out base_uses.f90, $(shell find $(SRCDIR) -name "*.f90")) -ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.h") -ALL_SRC_FILES += $(shell find $(SRCDIR) -name "*.hpp") - -# stage 1: create dirs and run makedep.py. -# Afterwards, call make recursively again with -C $(OBJDIR) and INCLUDE_DEPS=true -ifeq ($(INCLUDE_DEPS),) -$(LIBRARY): dirs makedep - @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME) - -$(BIN_NAMES): $(LIBRARY) - @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) $@ INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME) - -all: $(LIBRARY) - @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) all INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME) - -dirs: - @mkdir -p $(OBJDIR) - @mkdir -p $(LIBDIR) - -version: - @echo "DBCSR Version: "$(MAJOR)"."$(MINOR)"."$(PATCH)" ("$(DATE)")" -OTHER_HELP += "version : Print DBCSR version" - -toolversions: -ifneq ($(FC),) - @echo "=========== FC ===========" -ifeq (Cray,$(shell $(CC) -V 2>&1 | head -n1 | cut -d' ' -f1)) - $(FC) -V -else ifeq (IBM,$(shell $(CC) -qversion 2>&1 | head -n1 | cut -d' ' -f1)) - $(FC) -qversion -else - $(FC) --version -endif -endif -ifneq ($(CXX),) - @echo "========== CXX ==========" - $(CXX) --version - @echo "" -endif -ifneq ($(CC),) - @echo "=========== CC ===========" -ifeq (Cray,$(shell $(CC) -V 2>&1 | head -n1 | cut -d' ' -f1)) - $(CC) -V -else ifeq (IBM,$(shell $(CC) -qversion 2>&1 | head -n1 | cut -d' ' -f1)) - $(CC) -qversion -else - $(CC) --version -endif -endif -ifneq ($(LD),) - @echo "========== LD ==========" - $(LD) --version - @echo "" -endif -ifneq ($(NVCC),) - @echo "========== NVCC / HIPCC ==========" - $(NVCC) --version - @echo "" -endif -ifneq ($(AR),) - @echo "=========== AR ===========" - $(firstword $(AR)) V - @echo "" -endif - @echo "========== Make ==========" - $(MAKE) --version - @echo "" - @echo "========= Python =========" - /usr/bin/env python --version - -OTHER_HELP += "toolversions : Print versions of build tools" - -toolflags: -ifneq ($(FCFLAGS),) - @echo "========== FCFLAGS ==========" - @echo $(FCFLAGS) - @echo "" -endif -ifneq ($(CXXFLAGS),) - @echo "========== CXXFLAGS ==========" - @echo $(CXXFLAGS) - @echo "" -endif -ifneq ($(CFLAGS),) - @echo "========== CFLAGS ==========" - @echo $(CFLAGS) - @echo "" -endif -ifneq ($(LDFLAGS),) - @echo "========== LDFLAGS ==========" - @echo $(LDFLAGS) - @echo "" -endif -ifneq ($(NVFLAGS),) - @echo "========== NVFLAGS / HIPFLAGS ==========" - @echo $(NVFLAGS) - @echo "" -endif -ifneq ($(GPUVER),) - @echo "========== GPUVER ==========" - @echo $(GPUVER) - @echo "" -endif - -OTHER_HELP += "toolflags : Print flags used with build tools" - -else -# stage 2: Include $(OBJDIR)/all.dep, expand target all, and get list of dependencies. - -# Check if FYPP is available =============================================== -ifeq (, $(shell which $(FYPPEXE) 2>/dev/null )) -$(error "No FYPP submodule available, please read README.md on how to properly download DBCSR") -endif - -all: $(foreach e, $(BIN_NAMES), $(e)) - -ifeq ($(BIN_NAME),) -$(BIN_NAMES): - @mkdir -p $(BINDIR) - @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) $(BINDIR)/$@.x INCLUDE_DEPS=true BIN_NAME=$@ BIN_DEPS="$(BIN_DEPS)" DBCSRHOME=$(DBCSRHOME) -else -# stage 3: Perform actual build. -$(BIN_NAME).o: $(BIN_DEPS) $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) - -$(BINDIR)/%.x: %.o $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) - $(LD) $(LDFLAGS) -L$(LIBDIR) -o $@ $< $(BIN_DEPS) -l$(LIBNAME) $(LIBS) -endif - -endif - -help: - @echo "=================== Default ====================" - @printf "%s\n" "$(LIBRARY) Build DBCSR library" - @echo "" - @echo "=================== Binaries ====================" - @echo "all Builds all executables" - @for i in $(BIN_FILES); do \ - basename $$i | sed 's/^\(.*\)\..*/\1/' | awk '{printf "%-29s\n", $$1}'; \ - done - @echo "" - @echo "===================== Tools =====================" - @printf "%s\n" $(TOOL_HELP) | awk -F ':' '{printf "%-28s%s\n", $$1, $$2}' - @echo "" - @echo "================= Other Targets =================" - @printf "%s\n" $(OTHER_HELP) | awk -F ':' '{printf "%-28s%s\n", $$1, $$2}' - @echo "help Print this help text" - @echo "================= Variables =====================" - @echo "For convenience, some variables can be set during compilation," - @echo "e.g. make VARIABLE=value (multiple variables are possible):" - @echo "MPI=0 : disable MPI compilation" - @echo "GNU=0 : disable GNU compiler compilation and enable Intel compiler compilation" - @echo "CHECKS=1 : enable GNU compiler checks and DBCSR asserts" - @echo "CINT=1 : generate the C interface" - @echo "GPU=1 : enable GPU support" - -ifeq ($(INCLUDE_DEPS),) -install: $(LIBRARY) - @echo "Remove any previous installation directory" - @rm -rf $(PREFIX) - @echo "Copying files ..." - @mkdir -p $(PREFIX) - @mkdir -p $(PREFIX)/lib - @mkdir -p $(PREFIX)/include - @printf " ... library ..." - @cp $(LIBDIR)/$(LIBRARY)$(ARCHIVE_EXT) $(PREFIX)/lib - @echo " done." - @+$(MAKE) --no-print-directory -C $(OBJDIR) -f $(MAKEFILE) install INCLUDE_DEPS=true DBCSRHOME=$(DBCSRHOME) - @echo "... installation done at $(PREFIX)." -else -install: - @printf " ... modules ..." - @if [ -n "$(wildcard $(addprefix $(OBJDIR)/, $(PUBLICFILES:.F=.mod)))" ] ; then \ - cp $(addprefix $(OBJDIR)/, $(PUBLICFILES:.F=.mod)) $(PREFIX)/include ; \ - echo " done." ; \ - else echo " no modules were installed!" ; fi - @printf " ... headers ..." - @if [ -n "$(PUBLICHEADERS)" ] ; then \ - cp $(PUBLICHEADERS) $(PREFIX)/include ; \ - echo " done." ; \ - else echo " no headers were installed!" ; fi -endif - - -OTHER_HELP += "install : Install the library and modules under PREFIX= (default $(PREFIX))" - -test: - @export OMP_NUM_THREADS=2 ; \ - for test in $(UNITTESTS); do \ - mpirun -np $(NPROCS) $(BINDIR)/$$test.x || exit 1; \ - done - @export OMP_NUM_THREADS=2 ; \ - for input in $(PERFTESTS); do \ - mpirun -np $(NPROCS) $(BINDIR)/dbcsr_performance_driver.x $$input || exit 1; \ - done - -OTHER_HELP += "test : Run the unittests available in tests/" - -clean: - rm -f $(TESTSDIR)/libsmm_acc_unittest_multiply.cpp - rm -f $(TESTSDIR)/libsmm_acc_timer_multiply.cpp - rm -rf $(OBJDIR) - rm -f $(LIBSMM_ACC_ABS_DIR)/parameters.h $(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h $(LIBSMM_ACC_ABS_DIR)/*.so -OTHER_HELP += "clean : Remove intermediate object and mod files, but not the libraries and executables" - -# -# delete the intermediate files, the programs and libraries and anything that might be in the objdir or libdir directory -# Use this if you want to fully rebuild an executable (for a given compiler) -# -realclean: clean - rm -rf $(BINDIR) $(LIBDIR) $(PREFIX) - rm -rf `find $(DBCSRHOME) -name "*.pyc"` - rm -rf `find $(DBCSRHOME) -name "*.callgraph"` -OTHER_HELP += "realclean : Remove all files" - -# Prettyfier stuff ========================================================== -vpath %.pretty $(PRETTYOBJDIR) - -pretty: $(addprefix $(PRETTYOBJDIR)/, $(ALL_OBJECTS:.o=.pretty)) $(addprefix $(PRETTYOBJDIR)/, $(INCLUDED_SRC_FILES:.f90=.pretty_included)) -TOOL_HELP += "pretty : Reformat all source files in a pretty way" - -prettyclean: - -rm -rf $(PRETTYOBJDIR) -TOOL_HELP += "prettyclean : Remove prettify marker files" - -define pretty_func - @mkdir -p $(PRETTYOBJDIR) - @touch $2 - $(TOOLSRC)/fprettify/fprettify.py --disable-whitespace $1 -endef - -$(PRETTYOBJDIR)/%.pretty: %.F - $(call pretty_func, $<, $@) - -$(PRETTYOBJDIR)/%.pretty_included: %.f90 - $(call pretty_func, $<, $@) - -$(PRETTYOBJDIR)/%.pretty: %.c -# TODO: call indent here? - @mkdir -p $(PRETTYOBJDIR) - @touch $@ - -$(PRETTYOBJDIR)/%.pretty: %.cpp -# TODO: call indent here? - @mkdir -p $(PRETTYOBJDIR) - @touch $@ - -$(PRETTYOBJDIR)/%.pretty: %.cu -# TODO: call indent here? - @mkdir -p $(PRETTYOBJDIR) - @touch $@ - -# Libsmm_acc stuff ========================================================== -$(LIBSMM_ACC_ABS_DIR)/parameters.h: $(LIBSMM_ACC_ABS_DIR)/generate_parameters.py $(wildcard $(LIBSMM_ACC_ABS_DIR)/parameters_*.txt) - cd $(LIBSMM_ACC_ABS_DIR); ./generate_parameters.py --gpu_version=$(GPUVER) - -$(LIBSMM_ACC_ABS_DIR)/smm_acc_kernels.h: $(LIBSMM_ACC_ABS_DIR)/generate_kernels.py $(wildcard $(LIBSMM_ACC_ABS_DIR)/kernels/*.h) - cd $(LIBSMM_ACC_ABS_DIR); ./generate_kernels.py - - -# automatic dependency generation =========================================== -MAKEDEPMODE = "normal" -ifeq ($(HACKDEP),yes) -MAKEDEPMODE = "hackdep" -endif - -# this happens on stage 1 -makedep: $(ALL_SRC_FILES) $(ALL_PKG_FILES) dirs -ifeq ($(LD_SHARED),) - @echo "Removing stale archives ... " - @$(TOOLSRC)/build_utils/check_archives.py $(firstword $(AR)) $(SRCDIR) $(LIBDIR) -endif - @echo "Resolving dependencies ... " - @$(TOOLSRC)/build_utils/makedep.py $(OBJDIR)/all.dep dbcsr $(MODDEPS) $(MAKEDEPMODE) $(ARCHIVE_EXT) $(SRCDIR) $(OBJ_SRC_FILES) - -# on stage 2, load the rules generated by makedep.py -ifeq ($(INCLUDE_DEPS), true) -include $(OBJDIR)/all.dep -endif - - -# ================= Stuff need for compiling (stage 2) ====================== -# These rules are executed in a recursive call to make -C $(OBJDIR) -# The change of $(CURDIR) allows to find targets without abs paths and vpaths. - - -### Slave rules ### -vpath %.F $(ALL_SRC_DIRS) -vpath %.h $(ALL_SRC_DIRS) -vpath %.hpp $(ALL_SRC_DIRS) -vpath %.f90 $(ALL_SRC_DIRS) -vpath %.cu $(ALL_SRC_DIRS) -vpath %.c $(ALL_SRC_DIRS) -vpath %.cpp $(ALL_SRC_DIRS) - -# $(FCLOGPIPE) can be used to store compiler output, e.g. warnings, for each F-file separately. -# This is used e.g. by the convention checker. - -FYPPFLAGS ?= -n - -%.o: %.F - $(FYPPEXE) $(FYPPFLAGS) $< $*.F90 - $(FC) -c $(FCFLAGS) -D__SHORT_FILE__="\"$(notdir $<)\"" -I'$(dir $<)' -I'$(SRCDIR)' $*.F90 $(FCLOGPIPE) - -%.mod: %.o - @true - -%.o: %.c - $(CC) -c $(CFLAGS) $< - -# Compile the CUDA/HIP files -ifneq ($(NVCC),) -%.o: %.cpp - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< - -libsmm_acc.o: libsmm_acc.cpp parameters.h smm_acc_kernels.h - $(NVCC) -c $(NVFLAGS) -DARCH_NUMBER=$(ARCH_NUMBER) $< - -libsmm_acc_benchmark.o: libsmm_acc_benchmark.cpp parameters.h - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< - -libsmm_acc_init.o: libsmm_acc_init.cpp libsmm_acc_init.h parameters.h - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< -endif - -ifneq (,$(findstring nvcc,$(NVCC))) -%.o: %.cpp - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< - -acc_cuda.o: acc_cuda.cpp acc_cuda.h - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< - -%.o: %.cu - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< -else ifneq (,$(findstring hipcc,$(NVCC))) -%.o: %.cpp - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< - -acc_hip.o: acc_hip.cpp acc_hip.h - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< - -hipblas.o: hipblas.cpp - $(NVCC) -c $(NVFLAGS) -I'$(SRCDIR)' $< -endif - -$(LIBDIR)/%: -ifneq ($(LD_SHARED),) - @echo "Creating shared library $@" - @$(LD_SHARED) $(LDFLAGS) -o $(@:.a=.so) $^ $(LIBS) -else - @echo "Updating archive $@" - @$(AR) $@ $? -endif - -#EOF diff --git a/Makefile.inc b/Makefile.inc deleted file mode 100644 index 53d4e740b54..00000000000 --- a/Makefile.inc +++ /dev/null @@ -1,257 +0,0 @@ -####################################################################################### -# -# DBCSR can be compiled in 4 main variants: -# 1) Serial, i.e. no OpenMP and MPI -# 2) OpenMP -# 3) MPI -# 4) OpenMP+MPI -# -# Except the 1) variant (not useful for real production cases), all others are tested. -# Variants 2) and 4) requires to add the compiler flag to enable OpenMP, -# e.g. -fopenmp for GNU and Intel compilers. -# Variants 3) and 4) requires to use MPI wrappers for the compilation and to specify -# -D__parallel in the FCFLAGS variable. -# -# The 4 variants can be combined with the CUDA compilation. In this case, it further -# requires: -# a) set the accelerator compiler variable NVCC: -# - for CUDA: e.g. NVCC = nvcc -# - for HIP: e.g. HIPCC = hipcc -# b) specify -D__DBCSR_ACC in FCFLAGS variable -# c) set the GPUVER variable, e.g. GPUVER = P100 for P100 card -# - for CUDA, possible values correspond to NVIDIA GPUs: -# possible values are K20X, K40, K80, P100, V100 -# TODO check that an NVIDIA GPU is selected, not an AMD one -# - for HIP, possible values correspond to NVIDIA and AMD GPUs: -# possible values are K20X, K40, K80, P100, V100, Mi50 -# d) set the accelerator compiler flag variable NVFLAGS: -# - for CUDA: e.g. NVFLAGS = -O3 -g -w --std=c++11 -D__CUDA -# - for HIP: e.g. NVFLAGS = -O3 -g -w --std=c++11 -D__HIP -# in the Makefile, the -arch/--amdgpu-target will be appended with the correct -# compute version -# e) for CUDA: specify the CUDA include path in the CXXFLAGS variable, -# e.g. CXXFLAGS += -I${CUDA_PATH}/include -# f) specify the corresponding accelerator libraries in the LIBS variable, -# - e.g. for CUDA: LIBS += -lstdc++ -lcudart -lnvrtc -lcuda -# - e.g. for HIP: LIBS += -lstdc++ -lhiprtc -# -# Below we present an example for OpenMP+MPI compilation for the GNU compiler. -# Make sure that the env variable $LAPACK_PATH is set. -# -# GPU compilation example can be enabled with -# - `make GPU=1` (for CUDA): make sure that the env variable ${CUDA_PATH} is set. -# - or `make GPU=2` (for HIP) -# -# For convenience, some variables can be set during compilation, -# e.g. make `VARIABLE=value` (multiple variables are possible): -# 1) OMP=0 : disable OpenMP compilation -# 2) MPI=0 : disable MPI compilation -# 3) GNU=0 : disable GNU compiler compilation and enable Intel compiler compilation -# 4) CHECKS=1 : enable GNU compiler checks and DBCSR asserts -# 5) CINT=1 : generate the C interface -# 6) GPU=1,2 : enable GPU support (1: with CUDA, 2: with HIP) -# - -####################################################################################### -# -# Optional configuration: -# -# *** LIBXSMM *** -# -# Performance of the library can be improved on the CPU execution by using -# the libxsmm, a library for small matrix multiplications which -# is provided by Intel: https://github.com/hfp/libxsmm/. -# Provide LIBXSMM_DIR or adjust the FCFLAGS variable e.g., -# FCFLAGS += -I${LIBXSMM_DIR}/include -D__LIBXSMM -# and specify the library in the LIBS variable, e.g. -# LIBS += -L${LIBXSMM_DIR}/lib -lxsmmf -lxsmm -ldl - -ifneq (,$(LIBXSMM_DIR)) - FCFLAGS += -I${LIBXSMM_DIR}/include -D__LIBXSMM - LIBS += -L${LIBXSMM_DIR}/lib -lxsmmf -lxsmm -ldl -endif - -####################################################################################### -# -# Optional configuration: -# -# *** CUBLAS / HIPBLAS *** -# -# For multiplications of dense matrices, it can be beneficial to "densify" the matrices -# and then use DGEMM for the local multiplication. This is the default behavior for the -# CPU execution, while it requires -# - for CUDA: -# - to link CUBLAS for the GPU execution (LIBS += -lcublas) -# - and to specify the macro -D__DBCSR_ACC=2 in the FCFLAGS and NVFLAGS variables. -# - for HIP: -# - to link HIPBLAS for the GPU execution (LIBS += -lhipblas) -# - and to specify the macro -D__DBCSR_ACC=3 in the FCFLAGS and HIPFLAGS variables. -# - -####################################################################################### -# -# Variables for the commands: -# CC => C compiler, e.g. gcc or mpicc -# CXX => C++ compiler, e.g. g++ or mpicxx -# FC => Fortran compiler, e.g. gfortran or mpifort -# LD => Linker, e.g. gfortran or mpifort -# AR => Archive command, e.g. ar -r - -ifneq (0,$(MPI)) - ifneq (0,$(GNU)) # DEFAULT, just edit... - CXX = mpicxx - CC = mpicc - FC = mpif90 - LD = mpif90 - AR = ar -r - else - # Intel compiler - CXX = mpiicpc - CC = mpiicc - FC = mpiifort - LD = mpiifort - AR = xiar -r - endif -else # no MPI - # By default use GNU - GNU = 1 - CXX = g++ - CC = gcc - FC = gfortran - LD = gfortran - AR = ar -r -endif - -####################################################################################### -# -# Corresponding command flags. -# Note the -fopenmp flag to have OpenMP parallelization. -# A set of macros is available: -# 1) -D__parallel : enable MPI runs -# 2) -D__MPI_VERSION=N : DBCSR assumes that the MPI library implements MPI version 3. -# If you have an older version of MPI (e.g. MPI 2.0) available -# you must define `-D__MPI_VERSION=2`. -# 3) -D__MKL : link the MKL library -# 4) -D__NO_STATM_ACCESS, -# -D__STATM_RESIDENT or -# -D__STATM_TOTAL : toggle memory usage reporting between resident memory -# and total memory. In particular, MacOS users must use -# -D__NO_STATM_ACCESS -# 5) -D__NO_ABORT : avoid calling abort, but STOP instead (useful for -# coverage testing, and to avoid core dumps on some -# systems) -# 6) -D__LIBXSMM : enable LIBXSMM link (read LIBXSMM section above) -# 7) -D__DBCSR_ACC : compile with CUDA support. -D__DBCSR_ACC=2 -# enables CUBLAS link (read CUDA support description -# above) -# 8) -D__ACCELERATE : must be defined on MacOS when Apple's Accelerate -# framework is used for BLAS and LAPACK (this is due to -# some interface incompatibilities between Accelerate -# and reference BLAS/LAPACK) -# 9) -DNDEBUG : assertions are stripped ("compiled out"), -# regular release builds may carry assertions for safety. -# - -ifneq (0,$(GNU)) # DEFAULT, just edit... - CXXFLAGS += -std=c++11 - FCFLAGS += -std=f2008ts -ffree-form -fimplicit-none -ffree-line-length-512 - OPTFLAGS += -O3 -g -fno-omit-frame-pointer -funroll-loops -else - # Intel compiler flags - CXXFLAGS += -std=c++11 - FCFLAGS += -free - OPTFLAGS += -O2 -g -endif - -ifneq (0,$(OMP)) # using OpenMP - CXXFLAGS += -fopenmp - FCFLAGS += -fopenmp -endif - -OPTFLAGS += -CFLAGS += $(OPTFLAGS) -CXXFLAGS += $(OPTFLAGS) -FCFLAGS += $(OPTFLAGS) -LDFLAGS += $(FCFLAGS) - -####################################################################################### -# -# Macro for MPI parallelization. - -ifneq (0,$(MPI)) -FCFLAGS += -D__parallel -endif - -####################################################################################### -# -# Minimal external libraries, i.e. BLAS and LAPACK. - -LIBS = -L${LAPACK_PATH}/lib -llapack -lblas - -####################################################################################### -# -# GPU compilation. Use `make GPU=1` to enable GPU acceleration with CUDA, -# and `make GPU=2` to enable GPU acceleration with HIP - -ifneq ($(GPU),) -FCFLAGS += -D__DBCSR_ACC -GPUVER = K40 -ACCFLAGS = -O3 -g -w --std=c++11 -endif - -ifeq ($(GPU),1) -NVCC = nvcc -NVFLAGS = ${ACCFLAGS} -D__CUDA -CXXFLAGS += -I${CUDA_PATH}/include -D__CUDA -LIBS += -lstdc++ -lcudart -lnvrtc -lcuda -# CUBLAS (off by default) -#FCFLAGS := $(subst -D__DBCSR_ACC,-D__DBCSR_ACC=2,$(FCFLAGS)) -#NVFLAGS += -D__DBCSR_ACC=2 -#LIBS += -lcublas -else ifeq ($(GPU),2) -NVCC = hipcc -NVFLAGS = ${ACCFLAGS} -D__HIP -CXXFLAGS += -D__HIP -LIBS += -lstdc++ -# HIPBLAS (off by default) -#FCFLAGS := $(subst -D__DBCSR_ACC,-D__DBCSR_ACC=3,$(FCFLAGS)) -#NVFLAGS += -D__DBCSR_ACC=3 -#LIBS += -lhipblas -endif - -####################################################################################### -# -# Optional flags for warnings and checks. -# We do not simply use -Wall since some warnings for Fortran are misleading. -# For the checks use `make CHECKS=1` to enable them. - -ifneq (0,$(GNU)) -WFLAGS = -Werror=aliasing -Werror=ampersand -Werror=c-binding-type \ - -Werror=intrinsic-shadow -Werror=intrinsics-std \ - -Werror=line-truncation \ - -Werror=tabs -Werror=target-lifetime \ - -Werror=underflow \ - -Werror=unused-but-set-variable -Werror=unused-variable \ - -Werror=unused-dummy-argument -Werror=conversion \ - -Werror=uninitialized -Wno-maybe-uninitialized - -FCFLAGS += $(WFLAGS) - -ifeq ($(CHECKS),1) -ifneq ($(LEAKS),0) -FCFLAGS += -fsanitize=leak -endif -FCFLAGS += -fcheck=bounds,do,recursion,pointer -Wconversion -fbacktrace -endif - -endif - -ifneq ($(CHECKS),1) -FCFLAGS += -DNDEBUG -endif - -####################################################################################### -# -# Enable for MacOS compilation (-D__ACCELERATE is for Apple Accelerate library) -#FCFLAGS += -D__NO_STATM_ACCESS -D__ACCELERATE -####################################################################################### diff --git a/README.md b/README.md index 50bfc993667..78703a96d18 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It is MPI and OpenMP parallel and can exploit Nvidia and AMD GPUs via CUDA and H ## How to Install -Follow the [installation guide](https://cp2k.github.io/dbcsr/page/2-user-guide/1-installation/1-install.html). +Follow the [installation guide](docs/guide/2-user-guide/1-installation/1-install.md). ## Documentation @@ -40,7 +40,7 @@ To cite the DBCSR software library, use: ```latex @misc{dbcsr-software, author = {The CP2K Developers Group}, - title = {DBCSR: Distributed Block Compressed Sparse Row matrix library}, + title = {{DBCSR: Distributed Block Compressed Sparse Row matrix library}}, publisher = {GitHub}, journal = {GitHub repository}, year = {2020}, diff --git a/cmake/CompilerConfiguration.cmake b/cmake/CompilerConfiguration.cmake index 0d6f0d629d2..03b0b436b7a 100644 --- a/cmake/CompilerConfiguration.cmake +++ b/cmake/CompilerConfiguration.cmake @@ -1,8 +1,8 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") - set(CMAKE_Fortran_FLAGS "-ffree-form -std=f2008ts") + set(CMAKE_Fortran_FLAGS "-ffree-form -std=f2008ts -fimplicit-none -Werror=aliasing -Werror=ampersand -Werror=c-binding-type -Werror=intrinsic-shadow -Werror=intrinsics-std -Werror=line-truncation -Werror=tabs -Werror=target-lifetime -Werror=underflow -Werror=unused-but-set-variable -Werror=unused-variable -Werror=unused-dummy-argument -Werror=conversion -Werror=zerotrip -Werror=uninitialized -Wno-maybe-uninitialized") set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -funroll-loops") - set(CMAKE_Fortran_FLAGS_COVERAGE "-O0 -g --coverage") - set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ggdb") + set(CMAKE_Fortran_FLAGS_COVERAGE "-O0 -g --coverage -fno-omit-frame-pointer -fcheck=all -ffpe-trap=invalid,zero,overflow -fbacktrace -finit-real=snan -finit-integer=-42 -finit-derived -Werror=realloc-lhs -finline-matmul-limit=0") + set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ggdb -fno-omit-frame-pointer -fcheck=all -ffpe-trap=invalid,zero,overflow -fbacktrace -finit-real=snan -finit-integer=-42 -finit-derived -Werror=realloc-lhs -finline-matmul-limit=0") elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") set(CMAKE_Fortran_FLAGS "-free -stand f08 -fpp") set(CMAKE_Fortran_FLAGS_RELEASE "-O3") @@ -34,9 +34,9 @@ Please open an issue at https://github.com/cp2k/dbcsr/issues with the reported c endif () if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS_RELEASE "-O3 -funroll-loops") - set(CMAKE_CXX_FLAGS_COVERAGE "-O0 -g --coverage") - set(CMAKE_CXX_FLAGS_DEBUG "-O0 -ggdb") + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -funroll-loops -Wall -Werror") + set(CMAKE_CXX_FLAGS_COVERAGE "-O0 -g --coverage -Wall -Werror") + set(CMAKE_CXX_FLAGS_DEBUG "-O0 -ggdb -Wall -Werror") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(CMAKE_CXX_FLAGS_RELEASE "-O3 -funroll-loops") set(CMAKE_CXX_FLAGS_COVERAGE "-O0 -g --coverage") diff --git a/cmake/compiler-tests/f95-reshape-order-allocatable.f90 b/cmake/compiler-tests/f95-reshape-order-allocatable.f90 index 4a7b93048ed..787ec241a36 100644 --- a/cmake/compiler-tests/f95-reshape-order-allocatable.f90 +++ b/cmake/compiler-tests/f95-reshape-order-allocatable.f90 @@ -10,7 +10,8 @@ program test_reshape integer, dimension(4) :: x = [1,2,3,4] integer, dimension(:), allocatable :: order - order = [2,1] + allocate(order(2)) + order(:) = [2,1] ! PGI <= 19.10 does not accept allocatables for the order parameter print *, reshape(x, shape=[2,2], order=order) diff --git a/docs/guide/2-user-guide/1-installation/1-install.md b/docs/guide/2-user-guide/1-installation/1-install.md index 83c827bf983..e31be0838c7 100644 --- a/docs/guide/2-user-guide/1-installation/1-install.md +++ b/docs/guide/2-user-guide/1-installation/1-install.md @@ -6,7 +6,7 @@ title: Install You absolutely need: -* [CMake](https://cmake.org/) (3.10+) +* [CMake](https://cmake.org/) (3.12+) * GNU make or Ninja * a Fortran compiler which supports at least Fortran 2008 (including the TS 29113 when using the C-bindings) * a BLAS+LAPACK implementation (reference, OpenBLAS and MKL have been tested. Note: DBCSR linked to OpenBLAS 0.3.6 gives wrong results on Power9 architectures.) @@ -36,6 +36,13 @@ Download either a [release tarball](https://github.com/cp2k/dbcsr/releases) or c ## Build +DBCSR can be compiled in 4 main variants: +* Serial, i.e. no OpenMP and MPI +* OpenMP +* MPI +* OpenMP+MPI +The 4 variants can be combined with the accelerator support. + Run inside the `dbcsr` directory: ```bash diff --git a/docs/guide/2-user-guide/1-installation/2-cmake-build-recipes.md b/docs/guide/2-user-guide/1-installation/2-cmake-build-recipes.md index 82c1ea449d9..0fed1be9eab 100644 --- a/docs/guide/2-user-guide/1-installation/2-cmake-build-recipes.md +++ b/docs/guide/2-user-guide/1-installation/2-cmake-build-recipes.md @@ -29,7 +29,7 @@ To use the Intel MKL together with the GNU compiler and possibly a system-MPI, assuming that MKL is installed in `/sw/intel/mkl`. Verified with MKL provided as part of the Intel Parallel Studio XE 2019.5 installed in `/sw/intel` -with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.10.2. +with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.12.0. 1. Make sure the MKL environment is properly loaded: @@ -55,7 +55,7 @@ This can be worked around by setting `export I_MPI_FABRICS=shm`. ### Intel MPI, GNU Compiler and system-provided OpenBLAS Verified with Intel Parallel Studio XE 2019.5 installed in `/sw/intel` -with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.10.2. +with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.12.0. 1. Make sure that the Intel environment is properly loaded: @@ -73,7 +73,7 @@ with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.10.2. ### Intel MPI, GNU Compiler and Intel MKL Verified with Intel Parallel Studio XE 2019.5 installed in `/sw/intel` -with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.10.2. +with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.12.0. 1. Make sure that the Intel environment is properly loaded: @@ -90,7 +90,7 @@ with an OS-provided GCC 7.4.1 on Linux openSUSE Leap 15.1, using CMake 3.10.2. ### Intel MPI, Intel Compiler and Intel MKL Verified with Intel Parallel Studio XE 2019.5 installed in `/sw/intel` -on Linux openSUSE Leap 15.1, using CMake 3.10.2. +on Linux openSUSE Leap 15.1, using CMake 3.12.0. 1. Make sure that the Intel environment is properly loaded: diff --git a/docs/guide/2-user-guide/1-installation/3-supported-compilers.md b/docs/guide/2-user-guide/1-installation/3-supported-compilers.md index 1e4a379b215..089289b2ed4 100644 --- a/docs/guide/2-user-guide/1-installation/3-supported-compilers.md +++ b/docs/guide/2-user-guide/1-installation/3-supported-compilers.md @@ -5,7 +5,7 @@ title: Supported Compilers DBCSR uses the Fortran 2008+ standard, which requires up-to-date compilers. Currently direct testing is done with the following compilers: -* GNU 5.4.0, 8.3.0 +* GNU 7.4.0, 8.3.0 * Intel 19.0.1.144 Since DBCSR is a core library of CP2K, the code gets additional testing on a diff --git a/docs/guide/2-user-guide/1-installation/4-using-dbcsr-in-a-cmake-project.md b/docs/guide/2-user-guide/1-installation/4-using-dbcsr-in-a-cmake-project.md index 032b6167cfb..61390b7131a 100644 --- a/docs/guide/2-user-guide/1-installation/4-using-dbcsr-in-a-cmake-project.md +++ b/docs/guide/2-user-guide/1-installation/4-using-dbcsr-in-a-cmake-project.md @@ -20,7 +20,7 @@ if you can not run commands as root, use the following to add a custom prefix to In your project's CMake you can then easily search for the DBCSR library: ```cmake -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.12) enable_language(Fortran C CXX) # only request the required language diff --git a/docs/guide/3-developer-guide/1-tooling/index.md b/docs/guide/3-developer-guide/1-tooling/index.md index bcf24b470bd..d2727bc3752 100644 --- a/docs/guide/3-developer-guide/1-tooling/index.md +++ b/docs/guide/3-developer-guide/1-tooling/index.md @@ -2,24 +2,10 @@ title: Tooling # Build System -## Build with GNU Make +We support CMake for compilation. See [here](https://cp2k.github.io/dbcsr/page/2-user-guide/1-installation/1-install.html) on how to compile and +[here](https://cp2k.github.io/dbcsr/page/2-user-guide/1-installation/2-cmake-build-recipes.html) for more CMake details. -@note Building with GNU Make is supported and maintained for compatibility with CP2K. However, the recommended way to build DBCSR is with CMake. - -Run - -```bash - make help -``` - -to list all possible targets. - -Update the provided Makefile.inc to fit your needs -(read the documentation inside the file for further explanations) and then run - -```bash - make -``` +Compilations is based on [Fypp](https://github.com/aradi/fypp) meta-progamming package, which is available as submodule. # CI Setup diff --git a/docs/guide/3-developer-guide/3-programming/1-overview/dbcsr_mm_overview.png b/docs/guide/3-developer-guide/3-programming/1-overview/dbcsr_mm_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..baf11f1a20b79a9f59f4f867a3a353c4bd5bd080 GIT binary patch literal 49976 zcmd@6g;!MF`v(leI5g7T9Z~`!A&qnj(hVXY-3&dnbR&XG7EV~159*q zWQQCs2mZruFDIwvAg3tj;^^Y8<7#PbBlFC~%0^pJmS0p<1O+=)CRzO!k$HmQejrw>?gNdi9P$`JX50#D{O17}r zH}CK0IK*IlJ~q%%Asq#W2j7!b8%l!9Naw*Vf99-?y%qS#AE(xXiyMB9QUFsSeM0SU z4Mq;jRmso;1%;dl`3KcDZww5qG>Y8P; zygW|UHxJZp^4wse74uBOmpqsLY}QKUQ01{cm|W*H*W*XlPd#$#vpvnYHjtTV^))7< zD*JKmM0;>B7x#d=064+p+LaOkQW5iK=Q;lkxU~EFAg5d;ZnFPgtG-J5J=QP`{EYfV zKUIZzD%O>r_*93f&&q54>1*S~I?hJqye3J+>>5h>eYs8EJDZ=WzeU(yv8tpcS$>H` z(-z7X=uGG}es>20FRjXEz|WsF#rElvY-n=)f1~}3xM4Yxnnm~1Q#UrOEt7*^UyJtI7Wzev6 z+e?LS=>JWbCM*lCTJ0j0!yamvZ#(rrd8KZSQ;k-Q>YaDcJ+9B)3@a?)&%Pze)LR!2 zJnv6aHVeM-s-5I9YcUwe;L_I+upKTv(f@41$)W#F;jUqWYk~9f<%^%5(5?BgBGt9S zwE^u)EBU^3uFN(upWPg8<2p5sJV~B1quMd=sWM~stJ9s-xAL){BHZ>CTP1s~XMa}P zCf4OWbTXpV-<@w9;%f9gcxy3Oq0n_ND@_*1M0?C;?!U*32E``5S6FS*9@Ofb?L6PW zKS?RVpRHn&8tf3L2g?8Jif$r zx$ASS!&DRZh}u*-m%fSApX=(jdTzbX#?FmjO(%maIm3AeAc)J8DRWcgNropj97awL zMzyg`Gvr6i7D$PSSE@>3pBI{awmb&9-mlb%%Pp)?2|I67J+Mo*7Xvaywrc{5coSp6H@(rDCLuGiOR-UQENu$DSOM~pTU0uD@)x3?8jIZwBOvf zlr25Xu+gYjuSBzFQO)4~)RJ{&(M}k}<1kq&S+ZQ3!&jMz5tDC2M4YqrrBS@uLX{%4 zF7(Olw$t^=lueeH*SnkKkuB{a_doh@LDv3EK8}qaKgv7zmxTYVVvoh(x+SNEozGW< ze@Vj*YFyz%V4|%y@3s99G+&qKpGC8s@!pwbjgHr=ey(2Ivn)=-;(xmBM|Rf>k5-Ju z_2KkAb z729{GcFHrhO26FfG5|6BhROvGj{sBmaNc7CD~GkEww$yyxb(bm@8+s1a%39A-rcd9m$+C>-Avwe65r32w$#>3Z8k!V! zPLJg4wGLw1L&Tr*wR__~!fkN-ak~+$4un=E22dn5<*j77ipU($R!D)?W-NhF<)nIuWKDyyHZnn%9bNuxP>9 zJ=1F4n_xdpVtrAaDL6G(PhU+k5s|`Yt_M3?@LR8(+S^^QRZ<)LDPX6yFm3$NO!bKI zzt)I?#S)5=CX7i#0m?~%_Tp$=Wdm#rV}#&%syph-%tM!7F5#z}msY0??o8=)`quuZfY3`B^q46ss!jD*LUx993L!*niV4ryN= zA#!q*QQS%h3?7)b1-uTf0)^NL-+XD%&~2PkCSM`mit{_ql^P4UqnFTco5QRjn7^16 z;zq9nlze0vQQ3(@4krubFyf!P5CuJgNzyz5#UU8-bm{tNc@C4c!f{5x{`=bq8d3Mh zw$XO8wJ#=8x*~8Ku*nML3zDQoh(ppv*w_@3>TH+0BHYjSEh$Ca3j+WeIb4pSSIK_p z)CD`Ip$5ap1cpzQMOx2?4$jwhE}IKmr&RYdDO4gHsy*u_*0h@?MGQqUv%&umN$2*r z1zx>K=mT)1|8!^0W`C)(TMoiPPynUC;FTrM_;ILVvDgA@8VYX-C^N2~4DSY=zu<74 zmOM`^T6*G>t_}F3m(VTqdWx{G7g^LWhC<|6QHd}kQAiss>?%v&VO@jXGFECR*cf!v zLkfZ{73zhh*U8$FKxN!o?I6tj8!|3<)cub7U=` z=(rRN!8a$FPfHD+ydj5_eAQ-I5{HN|jaFJK-dGobqNoP0nh$*FfXNE3#NfKr zfhsW{KZ^4u3LlYS&%%8Vv6R^X7A-TfQ(hkq@BtJ-({6K-0#)jOL4S8IwFs?56H6~` zmA7XA1cfz>C9&Ue$IYk#V1ezK>fW2{_u0-V5H#$Dy+BiGesKUSuDl9lBcO}$Xm*6E z`|3Bw`kz7mIa41r^T-06TMiT?zJz;MEW{r9`7FY0a}L)#BXG!x#`B8vOnz1bIWM*t zYP=CEib4C8Arl~h|IFlNF}2c_25(rNpok#D(_zr~ikugkG;;BJ!RlX<2_X}~XK0C6 zgY1unJ+K>aiX-_*zKy@??u0fXjRm9;2tO&xiAJ#9`W?1MvC$46I+8C)|`n zm9xVzS-B=$oG3Nad9RuGSuZI|&=$L{X18gJ7Y0E6>iNOSE-&cGd^#UA(m@+gzP-Mv z#T^#tJm|_}sgVkHV*C1N(Cc)2#t-k{U81DB&+=Adb4VUeu-Dn{0$2Jy-4srp_viME z_U1*$6oVsI0EeY|zjb^ARbsfnG6-I4o~_B+e(~KZl#6X0;>^hc*Z}vytAWe%fs-Ga zQWtNsiyj;P;i(y|a%sP*`&pG2aQJed%GJ(rz>oaHHkg~ora#`Rue4N?DCo~r*_gr( zSKVkO{2omDI}E69Nt{kwSLXKmq>kSHau6QhD|uS{%kQv{edWQ{7Az;!jLx^@>gMbx zm!c};sW!7`B3Lo>Qx%q9S{}YKvEH9;JQ=O>zIW$(J5MTjQJ-Iu9hJxx6YjjwrW@$A zw@5+GMyBI#90Aq zzx*9Z{p;2?L&!l7wo#P%AZb6AuZ^PQQ?YHEb?dQ*bw9>jULITODrwz^)|cgo#eP4@ zBfYbOUi!?t)DORXeDY2_HtIMCVEHN!ghsg&+Z08Mx3k1xu0-?PkJ)SCr4nu=x)6z# zp8v@`OeLkL3PB~5Zqtc=;{K~7JneC_&mP$`$9=XE{qhWeR(<_R_XTq@!??}c#FxZg za=x0sVSQd&@X7zr&6#RX9Hrbu&Xl*_XOoSSM%XJF>-&MQtB1iRul}I7q%RACVu>Kk zhaixvBNdJkdiA_7MZPD2cB{5ti^H|&qBAcnbZVIRd{jT!HANc5K*Y_U8Pm*IF$=Vm z4N57JkfRPz6lgyrhTmKtTe|l&V#k^QE z+s~k@vU^;oR;Oh;S4S4*AJntoFRPgaU94G@uzhT?4c5Y2szL3=#;l`q82?z1q3Dls zgc!+;q7O2t&y%{^7KGn{^sf(OYQ}jeVaJfe~nyEmMaOYgiafoDjO z4)*wH7A%elQ?lHzfMF5e58i7%d!Qo&{*6YLz0dKG=vzPs@h}(QCdpr8$yH1A%8msA zz;o)EqEMs9A>$@+taP5OQJ(A2yW}cwd280o5hg|SNxz)8e!)ANA(FAQlbB6y)CF1? zS&akGJ3|cC(mdT83TZe|AC%kSHQjn$m;d=2HX7{3fFpGt>l?^~7=7@yTd9~HI!2}q z;dfipCeQBzdQ1lfHm-Mi(YJzmcvUZ44e81*;i=)IsRJk5)gw*VgGOPRaWaZ4cUAu@yKxEAs@vGKrCPS9;i`(;=QH4| zQ#OVZYC=<>V43PolbyZJChr{l;Ct|kci`Rh)A%DN`Y@P1jI)`0z@b!h$rL~`)1ho( z;jhPYhtIq_3)TQw^^g*Fh_8*-EIoo>9zBC~_vokX0esk%gYt&rt$`(C{0@5v0VC!a zKj3Bt=>(E20I62yUNpm5(cOOUR|a8~tUwMccBd=lO3epvKe7Eb&A(y%#?|F&u=sK} zh#>oJ)FI(*mkvQQ-lWLDiVlzQ^y_d|Od?0)TqpnL-Uf^0DserpM{{X#U2H@X!5p__ z<)bo-IdZb{2lltrw?sSE*A=`>SS_ZqqV!i3qZr)Y8E@I9-$x0?n&t-I!Yb3O;9kR+ z4a^iSH}iBp6({3 z^z&D2GHwnDzhlhzDsTJIXU)Gv-bJIwtivaKMn30F!0~7is^YXcUcAzxqGaqwb>O=X zT3J7!kGf9|JV_eXGK^OfrEd2>F8?7zO{wH@hHkuf$tVqnI`ou^j2TUsN%4{&ifyE7FZtlRg$5^L)>YL+*C%yLInJ(7i;d1Y>bk8(zb&4Hpre{e=#kuzGO^kK> zjYAq9ZmC5UMkkmZZEiy?MtZU%IgLtUr$S@@6O?&rrp zG_Pp1lbPg?X>%;a(dg>-6@<2yFLKY2GJplCcQH2k7z&LaGs$*Z4J#V`cXv>FyISuD!xs05(?G8~4 z)|b2cCY(PzH(0E$j9E6J9M)qTrr>@Yz3Kv8IhgdWa4Mj6@i+=dD0>`EAanaPXQX;y zfBB1-`k>%Q(<$QA*9(&qg1`~Mrm#g9A?b5X|A5Om{?{AZqfVh8G-O*+i4+f36WyA9r;v?EO~ag&$0kbIb1& zL}W6~0wDW&O3XLfqN?*0W39-b(#j8S8!)p??)!n4o+tf}*Fy=J6)?wiWU@MFfZSE4 zLNZ*$n9tjT@fD58&iES?ZeUxnDf3eHTF~H60*xpWsDd#^G`2q~7QngW1Mx=;D!RSv z(iQbu6+3&4O-CL+!HcoalMJlkgyK01 zLBsc*eLpe<(UZuzgu-OLZ!NYqKO4vxGA5K{{)=T`N8Sd1{@hH(5W+qb%6?0&bNXN~ z$1qiDZl-B~P!|w=Mn%Jh1=T?5$LvzD@E`aQL*~nsY)Ax1_v z6RZwD0u8%TRBru3l5-ba`|xL(WOO-~V1{#QY7#HsIVR?)4=2OstHbW+C6Z;KCPBR!ctnhFY8lnA4RoOJ*aeN!&(}Shi%=(YEjarKs zV)O=MKEw&~df3g)yZUK7um>pSjhP@D_|0fC1sD)tXgpnu80o3-73>2+_LJ8TZAq(#>nk^%h=)4-1BK#8kKC(U!KWg*UYWRNuaczt+5x% zH8H?T31b4MgF}h(A(_&yr3#P(!Vq@$lqny;UgVxGdM+uh5Uf79L zkrGAAp)?BT1)dt}G64CL=bQm=VYEPQ?R3uNEZeznae8;5S;+RojiHJ;Gq>u|`jCzv zMbOnq3ahF?96!y}2%ktB(7#-vs76aPm1;E43JL9gypKoMli#iYb=;o7xcj3V6+6@+ za5Y)cvmf(U49b`3_W=O46apNh-Q;N>7!}g6xOr|BVPK6JinE3@7c=x)=*LnErd{lg z8PrdH-vAmZuU6Gd=dYG6OE8T^%w}gIQUed^IrX)|l#YwTtGGX_BYi~x3T>F*A_jC>&p(qxg&w`_5j%)9*s!h)R`u;lTJ1t ztn@mt#Q>DQc>~DVJDFI*rN$@k6khw`cu+oce1*R5H1R2&+n_>PCs^Xo^-=$~pO>(N z_`kK7fCLqK==9@x)n9G%)hnEaq9o~V!irLU-YqHI%%XjktNF1HntN5Dqdgl_vqWBo zoNta{e{OF~+ix#R%#8ct1KJ$(Yh`hGpUKU9I|E&KuP69z2J4HG8cMg(c)2#)C9Bei zGmFwTYHM2&RRVV!00&y}JDJco1&XbwA*zSEu0CkQrh2Dnt$zThGBFRnsWltP6703% zOr88AY<1iQBrRUm8TY9p;Wp(|wnlEWVbe44PVXtF*_vVpAV%}5X1NXy{!idT>x)B)mBm!uDQK}J7}7)^=Z9`FIauqf!#w%O7DF!(mWNmwy~ z<^^LYD<)!6ugs`uBpHG|DTxUgW#Shv4FV1Vi9t#ngFgG=yDlSiem&76x)nuw0ufsS zmHc_XtuntohXCJ~G$j>>Sxw)}EH(m5z?H~3%y7K&Xjz;+krSBc@S2- z3o#GGaJ=HDB@#X}lX7)g-`2`o{{GPR14|78b1y)jxU_2lj2DqPHV5e$5~Qso~9s&tdJ zKxT-ky;~eZMZ%VPMT98X#~~{abw$TTR3e=t&~)-hxD7`8b;Pwv(KzlZpq}V{0JXx1 z{KS$cw4*D+-{Fzw(Mu~_5q*p`T-%>X+_Yvz=RttRj;Dl&A5iG@rLdk1Ql+8h_{e3u z5>NoJvi_$9*ccNkwoa^{GI>apI1<`=qAt!$aJHD&aYlxa4n{Tw4#nkbjNxxE?BV`` zJb>y;(g}}FgxFjoLG$zK=nhwuf%qECXpuYpqE_A*$`WtE&^N%z+OT(bW^g3ki?k7aZKJ8~K z&H!GAE3GJ7PSwiDmx6{Of7Uo}H+?gN&ep}V8u5?k#470Rp7=Cc^xBGgH6V4e%p#7) zQXccn+M(?#8i>gxS$VU%NAf7lmzsF^pg=RHxw_Q#eXPjO+e_r#n3%RDPq25JFEW5B_}F;36UoY&QEFNniXc- zH{J;t!^dTNFqb`2Nr*6oA-$y(nUBh<+mqQyjcbw~OwLRdk^%Kr{nbts8Y<1JUdsHQ zcF# zS78%gEED}o-weiju3!)bO-b$;(qq`OBIRPUPAu5N&RyRs8-LlyS<}JF?sS^|3%bN( zhxUM!%kFcBF)?CU%=34?Sth5X2wn)3NEszK%|L@;cbhDFk30b(K`g4Smv|py$bo42p5gP;7ASbkZAhUh_5@$5d`j z2&Y0v|8@Sx_)@;LO?9*lOpZo|xP4uWZ&)lNW_ZPhdhBk?X$c!pOjV##BMn*sMb@`a-}`%s?bQj!wde;L|(JJV{G26~x_v?u@W{ zUY?#|W2uB!P$E@erCVWD1$7g2@I7X+67HL3es0AnPG}IF#DrVi5JSS5()G^rpB|kV zl8&bMC%X4fc@C4Om21C>Th4!Wsyvi{2t&_5{EN)64VUQ~|I-GAQn~9V=izIeO)nIr5QP)J4F1J_3@VsQ;s}7+C%#a1PVZ34t4)$Yp8*&5^>b}y zjtux63>6?15*tZ;-FyO9zAPE6JTxxmytq~cY88@?HLYyv*3SC-rcT8%X z#*hiIl*11qB-8xQug1b;$P_wcsdt#txMyTY-Jiy}Qk@9|u0Bnvn?g51ICG z0s3ZOq`$1vM}Qxdz!NQm9j#YcbV4@f>Rn3hOaI?CFZ*1bQ>^au?di&~a&xI-Kxv+q z=&1ukf)y~ym%404E+0FJjk zT-!WOSl&mT@pc}*q_WA}2Y)^1078)p`_wiu`x{NSvM((y0YfiGz;F_2GW1*q%T_>& za5ZpT#gHu9T37M)V@IF$!7RbaqhDWRft`At&q*>>ZqA2naG>q7>QP=%HUJ&a;Smlw zuV-ebo&#`V7tU^ZCyld$#tc@XHL(pAYa2_;mcv!vk~GBHjFxExEU_6ezb`=466#`sDxCiH=dRmO)QX(^}+^h12@K4`%>8xGXy!; zwT3^}G*Vac0%5;^f4Ramwp60C-{nvB=~sQeR#GL&uOuRZ4K*2})eVVqA@6bj3KOXv zr(09<51rE-GwKcOxv1~_9S!NF!DMFl>6Ps)VS(O6`otrOVO-azuRsmIGCkRx5F)q& zlV##NDqj-aL8f^SVK_FNVTXo>Vd;B}hbuvvsN1h_lV)%Ay#f+44lU{DL$6uyQau>8 zjedl|j=f5mps8YHyRLmz*+E-1Jocv6YMxb($ObO>MlQB={(XGo+Ft{>0jspWj_%p! z)5am}hQ2?V4VC+$(6_lAdjlC4x=RG#$P=I@fke>7<1gg@mM9?j5afbG5r`(&r~f;)lnR>r|Igc$)t6RpZv_5)!B1_< zE_C|o9;xJ90Lh&1p83BZQAjpQ3S`@?x+icXwgVGD);MC-$@BZhrN)r)&^S;^%#b>nyURBb4=TiCqS)-f^pcuy#*!KSK@{;ZY5=_(+ zUklbi00CIbd{(TlPW5p6iWK8082{W}mDf)N{Nt_?;3G{llwU|3-GOD&>Zc2WLifD3 zCes0C?n`D?w9*9)s=Wv#SF}Hyh;GbOf7o7V7M}E9{Kv3`BEQ`t#~=pi)hH1AvBKmx zk9<@T=_HwrJ3G!{lSx-yr|cAC5&_2$Dxtfmhy~AUc3jR)zWZ7DL#*yO8uDE& z#Pac5*omkQk&-MVo?3WvGW&u3+n>lv9CUN33c3!#4jJgTuJ=E4o~d`q%aIB;bJfJU z3kL=lPFlU+$!6d2#t*q49LC9@Rw+l~QnCPJxQX@+2oq@|tqLZj7-3f_I?(|2201Ue zD7nSIZJm-kGf}0&7RR(S2>24+dgt1j%6P$*)$q zYabAb0L!rjJW7F}Yu`rio!4X|ulV1A9Gtmx>xrH+Xc+(AxksiMOK7k{UZ|J1^+c z{;pf6)5rTJ_woApP&wQ&vr)+u?c+29p91G>F>3br7+fz zj;iw_^tHQWF%u>F4h^wL^-Xq-$KW+LJsxyl5fj23e7+oY2p;tQW7KIjRi67ZHFRK5GOs ze{H(b`f%2vT+~MZC{W07&eNrUx683gOd&KmE^Qa@HSB>2PUXj#vfkU66!He8D|Sho zrcS>dxYi3swe1KY@S!6>I6ZEu+|f}9&OyCAHqio>d=XeQsN)udM6M6v0yh0hK;oYm zpjw`Fphu|GmB^lncL_C zF)p-?R_C^h=f!|)KmH}nhTAQY0|XUZN&yxo#5d(+=^7*kEaRfLIl2WAI?o;u(R{ zhtA)z

c_I!*WRuoHvcF8>#B?Nq>`8?=_d`8sXE1k~{&yDJ<3)T05a%jfW_4x5HXaa>!szS_5_#;4-4DWo zDkI~T0}m@hPXV@O2dmNxQ6S`mF2J^W^BDbZ4nlBEJ@CrsW3Q7-_7w8WNn>D;{sCM{ zP*le^OeK%+A1;OAzqnV4G#cFz!#qIe%L+LDuJ{^70#pg41m@rY(hKRJL=}@W(=o{S z#Zw7E8=u9;Q44F(qG*+C703!WPTxOL0kE#f7H|0h{M9pp;|NCz*H6V=A=@?>QC%DVA<35LMXpA)XLq0Jcl#{k4+kFxHGo5AXQoxYy6Y0 zD*nD4+!(|dkw&(d_cllLFBclgB$`0r2!CRbI3AVzl*8B*oXV!&ZSV_8l?-{1zB#Oh+~oHTYRCg&)9aS(4m_1V;@UNMnQ zY!T#3`$JIgjQ^NiDE1=!*08e%`&*ASSipt7Ea~A-m5#>j;3CZ6<1km^k8H_83Xzk` z`*2rk5bg3gi}bap7Iuy!jYP)dAo6&fDVBzU?jYs?+bsU@U<+;5cP(8X>UOTRs02`z zuKnXpcILJo=vShGCl8%wpT%s}+ghwk_tKUS{%u?#Uoh{-MSmfS`@#y>BIYn}Yw`6f zd5g~?+lv9Q!G*iHo#KWbxqHJ;@HqS5t7o%0&DBll19vvbHZM@G9K=!d;punD+CVyT zwxpk@ex4~9VCPcX#h$02Npc!g?V51lr>kO>7`_81ZaB~+U8gzh0kf>qU z#%$4wXUOBKQe*guKqfxiWP6-uSN^$;mnftn>}6!VDiBnn`%lh*GPzxkK3oNW2CqgI z00lUbK-Wk(!Lyw}25*K|0eVDR&=W#)P1xP|kVlpi$fcDZ((_tyuj1w<0TT$^>EXPA zCAh3I%To}6NfLf$_eD^TmWJB(IUdn=!}m-COwuX(4-(%rPd6vaxG;B6;aw3s zePDc5K-d_K@llcsDVZpTIA}=8iu8d`TUBbi{@lVL7!kN@;zdA|KLo8<1vuXmTcnhX zfC5arAQiiA;r9>h$9_|`Fv>;RQ4rR&17_IS0*L6l1vD{d5xEC0izeyHj#F>%4hl}c z^2_yDl|Vim7IOI0$z^l>xjLQF==G^*tKZm51@C1+cCQM(hHXF-R3;mUEj@`QM4hsh zAyRUNh&+nWA`{7uFkejT!0PVq{y2f7C&c`%=ryApx(D?ZHCgMqdmIxBCbzGA49QE~ zsh@XCW3=ilP`hCwj;(JccUzS!R!%1sSj5Aa5`|`(eabs#!a?Cps)CXQT~}^bwH$C6 z2LE2PzOkf^^W2k^v$Fcf5aMy|OtLm4ik5|79>Eh$12|nci1Q931ppB0($|7)y-t=U zX}9QgnqB{ALcjs~7PU+l0VV%G-=_k^RBXdKRsVOy6#_t|h5>5=KND7mHIdmHB!R?e ziqMgfT?00nU(epRWM(XV5byIj3|B!PAGtel0$#(+`<_7_?@Atxe zfB<4w2eWk2nOVFQWzWK~yBruAuYlNmEGx^fHNXDQMVEH6WKEeUO+spFomAfDMAhi&cRe&MTX&!WLJfQ zMDZLu98tmUmG~h`q^B4QMiE%ZyFR((KvijCP;Py)IOzZE`8PnOPk+`r=p&)rJ->?40`5Ba zGVA5}N}@wL5+$05jKhE%1wB|o#_IVu!G4A!{@+LdbEZ@sAquulC|yZki*mHs5ufa~ z9BDWaPbzdb2%ZpXuZH1ECUF~9eUvBm=xs?pUL;p^S-EH0RAWTFHw2_`?Pb3r5pM>aG39Ft)u^~q!L0a^b0p$xS^ z%*m_)Vo%uaZ2hH$z1;pB0Acnf%>dtJy{H1ZRNz#dlQBrg`mA-oB)-I}oI{rRP9H)$ z^0SJV3!)tVu(37u{rST3{x^^66D+UK4DPJAavv%lvPifo&b?X*!WgHpeFZIRL?*Jw9Z__CRfgn=bKs^e4C zQyl_>oP1B|PLQy5pe*%ogI=hDB;3ZyNdoO7=w5us>*yd$2{{W>mREnKvGq1IL9-_V zDF;S_`M9xYQR(Gw5liJ7gr&T^f8ZdoA#;+BS2*cY8cKQ#%kcAGANc~WtzD?+^SiUA zy(rjAohoo;UGXF4?~l9|D#>n4Tn88)@K_>sFm2xwR-v*J-2Z+_cq3JWrpLc}Ebg`W zw6vhZ9cqIL*`nZ<+?lHV09udc5<@7DCP&h($lEgMo(S=>Tr#1G-nu$M$FkY4_v@upUpm~! zz=^(aVPNg)lmx_5R+HiMkC*IENn;jx1p|tvCWqSe^ROZgRy4i$pk$bMgd#V%)Du-o zV54EqB@%EzD}Hw{qLR_&?0{TA+dUVh`Ghh}N8!m5ea(?5iSwl};OS6i3E0FqE;Q+u zIcl{KyoUyM!cdH3%8e1TNk_zUQ6e4Yt$tp497v#RJD2OfE4m;M&U6QjxSy$xwQ|BP z6Sy%0z)?QlrcEm@*7?w6{zXN^d)t84M}{jsOEfmhUF-C4eTdp&TVw@ei@pahb`8`` zG{jKS7U|qz?g>8`$9rIJXmimt)PiuISwikgSBS)1PX=GZ$=*-%-q3?CV)9{{1OBKY zUT~kP*Cw4Y_T1r|qI9PK01}8n1|z7%G8{SMu$E{@gD%<+dZ;gBMNA^GM|2E+SPWIV z%w-*PTD_7C^wWJ@v0z_^HssGiu(+9Z*49>lE5ncbG#`^8n6nO>pHA}vY$SQw81@$J z>vv;HgQERmb1L2uZ#l2MMCK2@nR5n| z0jyy~8CP#yL=FcFK`}H*uMAj0n>KM3>>@tONJh+!WW){j(#&yT-RhrJ1`vl=(>Mn5PXM0aGf8&cRybY@k+YyoY`i+ z!7Y>`>@U$~1$W^-l_7W}ha%&Y{Qd<9CHg5~a+VV0Pv(B_(DVXls9+AU7+PrZ`hn7O zeTQW`BkB$lJc$^LuX~~6s511$)MpUHkmf3N2Q-*ChdIPYCG-Mdb-)6RYzyIII z9y0=X^Q7T-j=+B^VMsMnmN>n1JO00qJy8PJMZzog;jd1HJiO$HWWcRKG=CA|UyMtF zloI~Gcu54_P|(2GK%n*D(PzLT1`6b&t}~YZ#gIS2L%4YUMHq~l3{e9JSC9a41*ADx zkqe-~qtqM3_0!uRCI^SdfpkTQlFZ*?X)7=CLta z5C$|tdBBa{r$2O@W1WI|5t!l3y zDDO)E-Ovd;#-vCk{TYb&rvTdC{a~aCIVT8CpbZgJCsGh7@Y3ZNfRqH{^La`Deb?LugukK~$y9g5tEl9wMzcufolAa{&|f$41vnO;wP`7#-7v;9O01Ls0&4)#CW zS_6)Tgdq{T0A}JLI9(mVIKE^6*G9N+Fad++w+MkIgY?sCwUDCiSiOt+?OTsUZTD_6 z)a!#MHFi&mpS+7~0sG4Qh6$ubKmkt&NTeTZUOky-v;?leV>;B?X=bFXDFp?iP}FY> zJeM^fB_)Ays}AhwOKf?L90SM@jRAaZSoBC5m7)HH+>d}P4f~2rwP7-X+m|Iv`LVBQ z0j$@mpc_B9?yuS@vm^6YF3JoL%o^tfBjk={q%6@=ua)jyWJx`|$a zJ#xc$E5Et!&-J+jGtLmmTex7K95Oo%L%i{M2DGPLP9O{o1P;aiW(~Q|gvkz(*){zC zYmgN{q7DQpOImu?oCAu=Eu?;!7kt|YtUld0_mNSccZ!LVJc8kg(F!;#i>>}bBb&fm z%Bf81d!XVmd<0xDOZ2#BR>RIp42@tARe8s@+2ygef3}-nwS&mEwkOTY4H}uKY#|3W zP}(!yR(krt29t8*uJa%u!?6NUatt8gZcoBVFYteJO38qRDSnVN)&<)vila_j$)d-i z))6;*?SCd;xjl)2|6qehvvi2K5;m6Rq_Lx3L&6x8%8AjGm!fbfnUL!ooXzizY%CcM zbnXK%gwHM)H}Hj>=Q+UOAB)BJ(g&M>ijY64T#okk*b?bNxEi?${r6h{I`7p&^MV8N zSCPw*DGW$L$(0E?^5Rf6cR%*`-&RSE=ySmmjEnb2K z_O6}j$;I$9QvN@8AG1O=UPGCZz*5FEd8)x#g~{O1{r#Py(6KUD`PUks11=6F?+{z# zxPwX;x}z|%^cg8|f~?%{UQgHUX@2;ZaH)eSw#9;YFmMjU$ZcIlaPc`gQD|NSb0i<@ z9czX6sOIiBmtG4@P}Kzs6u=a5+@t(b!-szw8~8Isprbtzs^1ej~!zy+i&?g0b zfaP-JsIhlbi)9=8pH9nh_Fa*_4^V8Rne$hs4dhS<^_qBvG44zWs}@9p4Q?2F5hdkq z_dxGJ?)4}kCCc~);(hj_jwOu64apH$c^k4{$9Y_H=-~8VFp_xg^$^>6dVbk zT>RBOIb}O8x5g*l4?WY5B_lTB=O=fiuE?2go)Y>bk296I(1d0Zg{I8FiV38%uQoRr z>mUZQ4(yzD7CI9$%58e!#*|vc)nE?+Ss&Z9FsJl;<;N@XS*zmN;-dK@S2KciNmU{W&p_M#T14cz!a9gM!UNb z+$VC(uISMPu{@q<)4#%X@vS(MtT??wPvej&h^5ky3WiSX(m(}^vT70`8QxK#iofo? zjGu-n6`^nCm_pvPx8jxmb^3}hM{$ITzEztOVJ4yAA>^>SD1}@E0ROpL0ROTn!W?zv z1VX2Y$2)WS$X|E>-9ndj@CVjXhOu)p*dW2Im#WnHOv*NDi`u2-$($fsWf8Snwn(Ne z-dOg-{e=okxtpa+ppI9;j-D?Jf0Y;>U{wgZ+7TA@{GIfiY2Oj?xp{Z&V9C)ER5j~X zyl1zS_x2-^bXPkXEHV^%93(9)`=_rCbQdLq{b6QEcqy#{$VR62>I@$iR4L!A)$fJ!Yk=tep7`kq zSa5==ms1yACSAQ_^!71UAiB+f7&Dng;%zM2-1~*Mwl0*M9q1s9I9CmVh?Dxcdb0ko z5eEahf)>A2Ci$4(D?o+ohTX+sZF!=e=Z3wHio`4;AxJ^bB3Y#SH(LmgTH$b49dftN z7BPBChJTFLeRFOM^m0nh~5~TL*E%gO$cvZ$y{#w*v1gQACLCCFPJ=pxeDAQ)>Z1MURL9q6deedZXzIZAs z*}RHtA?JAmi{J!NO}n|evL3xJFkzN*#Yr?rTPAs; z9Gd*q&e&t{n+u5(?x1X;VVo|fW4Vm_?=~t*NDzLwoAC+2nHLF9QbPx@D;XvHPsf+) zqiZ91=0L!D>*f`|H48`v5t(O$HbRsZk;%zz56l|U91jN&8Be&YvP5nkP>?_de!^t( z*y1-mP^q2-%q+2n4r8d7o% zaUP%dYK1|}COD<}m_t@6bBdcLqI^WI5O@TAFn4q5cu0Dwl)+x*Z3mu0lTV(d&6^8j z^t+$9YC6#55^R(i)G(Ja1*-6uh~s>nE75hP^k=&3Ub?cKPk+rmj6AET=nSPOWf*`W zce#LRMgkGPL(a+72SXzAzpux4k$kOsJcr}o10e8$+y_ut%7R!Hf$T?jU+^_|SfU_K zQe<6&{_aOv+o0?a=Df5cis9DZ;-bLSJ^_o9VU2j>IxFIziq7JPhvzSaFVV6p! zivUE+L=zAn;l|L?+(|O(f2O}LyaRr_qvThVi2W@zLn0jW0*Defq_O8dZN4YJzccI6 zO-EyNVu6j?A|OgAqR2;&L2nODqO}C&hl;E__Y#{DivWT~iHlT`PbE!T{~z++`-aNh5tVMVSo!?&y!)!Qt=9YcYUkM4Xw^ z=pp6+H)rzOKW}u?DA7HV=AqR5#$?b#z1kbn6^SvNG9Vq1LN6n}g|2r%9Y2w(M^Iyu zG8QABCo*2Dc^tcDu-;4h3!|2QFq^+Ynn%mmtgQaRRCFOO<+72M)4jR2BYVCbWeE1xozA|y3*wQZeJrIZ-Vx8nf^EN+`04{&-F4>*We-< zT2j{`z73?_g>aQrXf|DKY&x1Qm*NLEJ71v{#=_Sg7tk@EP*3BtQ=D%&L)71-k1m6ngim8%387r z@bS+wqc8okb$>v`Z>&vLMKc#3v66Pf>hsU36BQ<2tE#iAVqt9$E+TP1A9p{K^s#+< zNymWNv2!z+2}|kyVbd*W*PbVbQA7!8 znv}3sJyogVx=Ax9XxQLs>rS9UpDG=_j>&Ue-E}MDn+9GPqWhONY7v3 z4d}^^l`lV;P=W^FV|7B+(Dng6O&1;)`Lqp6Lz=B=fKG1S{`)J(xZ4`fuDD&XXQ2N$ z;~6GMcG7jW?1iDp{f$e5KcSWLHaFaOt%kskj0lt~>)M;S4+x_kl6iypFvETn-yC}v z<{c&DWnfTs*`@wP%MdDJ!2B#fFQYHfHCD>-SFWcKq(5;?@IRkFY>{aqERO_DZYofn zzxMNKXBioD$g6g4cD%m7CI4>CW|=GX@lV>Yr7y2<>rA(2qE`yWj^yeyhUOejIms@g zRnv&l^oc)jU#0Byk(5=N6`V1~c7AaCHl8LuKwzFbl`7ptbUi?g>Fs7@;G6-r4xt(e zJg&z7ue8Y&I%Va?aE;HxxvSr%58jeIw^9uV*k#en%e{N`Fc^C9i)i~xmzOpWVJ-F( zg3A5}Ubh5N6o`1Ur+uDg-Wnlr8rF z_WiP_Tsfqwfhci5WOUtvCXt^I)53?NgRknvI##wZsTcNgj1!T8<;ba%+4=K^thW;I zT|Dm#DH6#S0MX()ccYFz5;x|Z}qmP7uGAf?PqVtyS6d#+Vp;`cWEt45A{HHu&p!p0=C(MEB$ zzv zVK&7W99@E_Pi$IFF^~@8Dn;mEpC29>8gm>~W8D*g#3Fq0W~8rii$F6%mzl$n9`iu& z7D`(pf|P3;9ZAAe>hcQl>H#c8bf3WOAyTYAO-B6Zo}XnM|Brrl20;1e%Ff&`q|5mC z5B`X{f!E@We&+xDhjz<2+z{EoW0`yZ`!Otl$@s!&X`WpD4-14+!cz+BhCwT(#N7Y= z80%T&mb7X7O#Q#_&H-{P=k;jPv;WrzsfrZ3|9?EBw!{|LGV$oYfbt&2->v#z@#KYE zv;Oca$dar*v;4IzMcTuP&-g_$g5z14BM6#2!l!+?EZqxWS93aopCaOdQv?S7*O#xwg@;RlmB(L>KK z3s~RZNkf!g-w;l9;ECJ~(6vA0JgzoGeju6~c~o3#Wp7YuJE6dTo(xC^VQM3Y@eQD$ zdo;jd{B!Ru9Nq=3#gF;oTT^6RR=Df%DtE(yDoz z(%SSLX}mHZL+G>Kfaq@L*aA^l2n9*l&wpGE@=kA#W=L$S*nlS#jG8P+nqLwHfRnRV zY=Dh^6?EW@ctCT-80Gz|V3FbZOE5?dWhZAWyi4k{_Kw9A{zt3z3xG0S)inokP7zS` zo4=kV>3|?}0fcC2@FBUow;26W$^MH^Le?0hcmoh$rOs`f9Z@=i!ml41G=5Ml$5GJd za1*Mtz&rQ3e$h~A4m3HvNN2>j9`^?zd^`D9FHTE5r9$C%dO_cW9;x4(3cS)@3n4*T z)Ni53z8L6@a4Ueq-?3WHN`2*0r#>CJNHp}o9x&3xAkTmh=bBO{HgO|j!O+WFSPBqi|wZNpDh6O z|2Z;riwlyA0Lt$W5#s?24a6;f|89Mx_nrM9J3ffT5db4N z$zYEqEbEc&(6xcEDYG-+4(mmrH<>S^kN_2MyfexL0-e&HW^f^b7v_CvaUdX#3UXfy zFHwRcO0Q)nOengg7kYsqhdsawU8}x+Ur1t65=Q!aU&X^;9D6=if0b~zw%$P%eHzju z06Tj$Yl|p5d3Ah_dJx)X2Mg2>=ewtg_mbL|fdr+IxtW3a3*B})tRt@SRW{-}h!ynX zAO9;!U}@g(#hAgUJGCL7x|$zu~SUlYBxT6Kmvp)rWmp+KyZvRwCOWYY;r(M0vF{+S;r)Q3ju?EDs@QLK~Ff^H2Py zK4bnG__IRtPW)HyCS4Z)8C>k>BEppIug{9xwBT$wJyB5)acfsqaf4O#+q2J8tW@1+ z&e7jAhxpB8;22_l(p@wmWRSd7NOB&jDBN%XcB+wnHY#(|8dNont99QBL5%-r4~gY8 zp`2BahX9*BpAedj1frj3U~RDN1?dj|d%!sCHDKwwFd6^8?jR9a&)cxd4y!y7c!r5s zupHu0G4Lv`GXhCh5&TJE@UVGz!WCg;_HO2z0M4q`KM9eshl~dj<~29jqNh%KIFcIr z{{Z79bQMVrRTMCzH2X!3o?dxsBM*L?p> z@*yc-_v%$`#l`J9Fg{Bsk*Kryl>oW_ac)qaSpXDEdg3hyj;6Z=?bP_g;9rDyl zX!CDWSUx9yHyg98o4+5>2TRI|V*J{{U@*(C20bQT&#JCgaBl;Mmh0D$31{E!jWE&0E)}P6U&^#{1{! z&%cKkF;H>w9|;~V39G$rvpgds;XaQjC?&ot?g&P#>Yz6|&-0i2X!5mXRw|=oJ~hX9 zrHax~A(k+moaPnS$oU|N>>-n!@2|Oe;w=z0uh@4=3MT(UfIR8pOaIf}x}l-;{W;x4 zSSMZ3()wT9(O_nVYW8xx)-B#2Nv*vot7jXRSm!p~+Ij19 zKL-4h1mIj61c-5b6xNrqyW}7sRXo}R>GoAG-MKRwob$OL)OB< z*xE7VHwghQPwiiNFV=;0Z4JWS&WZgn?c(*z{pdydJFobLAfqHOrYjZx<<8i6f$RAH zDWlb(Qu6uL<~7ENgE9!8(V(ZKq$KD;tUDlrE^V6SQgv;)dKXZ>>b_v$iT{ZEWBqSJ z0!<&~kA_*i)^X{YD2u{uX^%RZub()(_P4hS|xP zil+Nb*u?(rm^bSkMaW|i`UT@j^wmY6Ya=Bz0f!30Le3D%>pk3~m`e3O6GtC5%O06U zKF<=-uJTye^1?P|FnBhHzD{`qn-bpOsW}c}kH+nl@vE?B7csxAiZ4)@ZfrS#kAud>*r&VB-1(LK)D=E>YU2!@l1uzB#+FSlPYY?Mdr6t zN^zhoCELZSX0v={+~ck>uSgOW4}HpY?}%5CKJEsel9zCG7@I5}&Pk9FI#^(9=H|sQ zttbJG@+`fI#rd@PF^-6qwj`x=%Mq3@Y2^)BdnO?m0*>!*KUVo#oYb9oR{kFXI)E8d z;PqN0&QnHbx4A-C2 zs$1@gh=GG<4`6(OR_o_GGhah~Z68pB_-kGXwC)XB3{bd!awGVobNV)v#~De^X!YM1 zizmRkdTA14eLnaP9PM|7d&kf{ZWewimZ$vjdBK|U0ys~@Ub_KRvD-(m%BndD3Xs%- z`(;weilV+9-R=8$-@`}>Ez^-o99xL&v`88>xeAW0G@0NYtN;&lWPM6+aHTYG&|O69 zjDZ^rKNhcv^M4>&Dt{C9RDyXx<3Q#~^+6$4{f}njL~~$0P^~&vK^q#^jB2V@Ss3TL zXYU7AnM&kBHM)cio+Jg?0+QwRlCCGiK&D`Bvpg=RjC(6Akc{n(^2)_|lL*Ri5v=zl zRLr>FZ19PzuG?g!hNarAm;ZZ>l(jJ}M75W*dtaUx;%>0B6P}T>k*)TtbTy7U|5D`b zU7`jOz}a=ks+wplM?yq}`-vo9`~maw%Bz&>w}PZ2COd+ieU7H@>=a(h^I4`B^&GFj zi3$I%de?upVm8=mVe*dY2=ktTaCtHdB+SL^3``;s5W>1xiF#oP=Ki@a$HX-0X?>0h zA_+Uh(%5*yg)k#4Ik98HyurJVVCgmq)K%)3T)b(efiuj^Y^s% zfhAWi<+rXL%8&OyTc=;3Hycl%ng}(~3-(!4;p8K5D!Y1`1)91QpOIi5KDUsuVjr_c zzZHeTk2tGClA4aF0v?>I;#yYOV3EYoxvvQ4G}mf6qF;S-GJFpJ{>R1TGMA!F2{ib~ zT4godo~e3uNlEXUU)QTxE>rSYJ><_;2=rC(iLNWKK%zr|>@N4((=$1bR)z|ul0N`* zC%)pgTdNv*z2J+|7W|>X*B=!H494OhfdK&VWM~nkMjCZI<`yAG68j60huj;FH z2asl(Awo;(NlzpM(;~kGE6qqT!1y4>G^)WrT8sd@Kt;l%iEVGLKWVz$Zf!b%r*i>C zrj}o&=G7rHJC7AhfEk8{t=?zr}W?}iWV0yPQ0f&V$Brf2_Fv?sLt63VDx;D?mF zgL5z5<-g81e?}v0#|Gc!d9qNeutAS}nxctca%taH&}OiM_{3LEW#&FqGhdDFLW3>S zm+gGtdJlHO8~2ZV9Ihzq2+gZSpzLU5*lpfY>7VyRA$p@lSII4FkW1a$Wp|CV{2ZYn zL00{^-B(Qh2HVNumYa0)t0}76H4iF$dA)q2h#Yd(x!z5v1jyXBA*q`9e7G3Ln)gU~ zn=W`H9nw7QS=`&Ch+Iych~Ut)stdbncUMg29{w+S`_RrB3gqj<};_cc!p ztoh=Sb*Zk4ho36oBm zAxZaIzp~-xSC(?$Dm&GOUH6cxTe06xj?%nstj`Lt9p#5no#z@Ys?W=mMGCo=i#TQ{9No>e)q_z?`m=y_V^Qps$4UvV-q!bD%7u4=^aOQ7PGSz~v%8 zOT28)AO<#}+{^yWQFaSL?{5Vik24TGFK^CmjO5Tz4biVkkoTU~#&%Ncwb+7!a9I$y zQ`7*9iV<@A>!$p0tBdnfoe6X-tQXK3M3NU6$8}Q4S^@-lJ;$EvzCaOxx;jaft*T}J zEWI449gaJhtd*+FT9+wxHl$y6Y|mW|Pz&^3v%0cyZmpn6CFtwuW2=*mq8A;bdCes@ zeG|Vke~0sXydd?s%4>=A(feUl%0DbP%L7_5!BcF2UlnYgCzeU)!X+S!ZuXc9#r28i zOeRA~-WvXN3t%H{?Hz{-Jfb+u0PT^%aZ?kr&`oKl0a~n98TIs)_g=f?--cT;b+6e@ z4?c#3m2|y=-h~9zb%VHZ9cNh7|?(BB-nU@Ns*7YJ?2sOVG?+zX2Gk-u+g zsZ|$Q*FCD(w2Xo^89{%cEQ-`)YH^MnkvL%+!e%wvct`I5)q3U7OEC`VOTp|;DLLB4 zRF zu^=K^YT5Gw#9KOIED;FfW1i6G0R1Yi`I`|*CJHF z0%b{Lbcc<`8T8|7)41lytOo=k=ZX9eMo)Q!Gk8m7z|VX4!@GZ=f)N7p4yP?NX#-tv z(dezA73Q$>uN^}|_uvQU69LxOZXv+U;S103VCmfdFBOIr1E&!d+W8c2XDg+0ym$`) zR9sLI5x>J7`G*2SQXkx9M$RCliRMT35LfxFS-F!q&q&>#q%D0u{x5J|9cd&DoB!s> z6LJdShC5+@M0(OB15U*49)*Nq15pr;Z~&$^^&de#R14IOHQzwzV*VavMNcT&ZSvsv zk#KCN047tsRXNF+(KZ3upo%1U4hX|xGr*yjPpOzW=6R9RZE1u*l=WMc@utAIq5#zV zG;kgSwQ0q<_jLNwg#<8MkQK0B#Vk$mZ{m$U@&XZ-b%>>e;t?W`l@^Q}G4D-gdw`^} zuPFf#1`BZrqYMYhKZc?$3XXN%lqfMA%^ukT(DXmqFeDo+jZ|b0mccJ&Rc?W_X%mvG zwFWF?$&q?rn-H#~QP{;JPArBybbtsB`zxIy0OH==On$%a0z&IJ(}KNs45MFe1`lJp zT1ue0OIA)n;s^@;$7KL|{fCf4hszD{YC6Q>bf7*xzs$e~_+<{@iB+}tK#FIw7Vyv3 z%t=NImmKS5#|KXa4hAt!gOmLwm)8@}znIj5X+Q{3Mu2d-7mDyAxXjg4TmSS`+N?-M z25PRu2(I}CSq20L$?r}2m@Ok))5W{X{uN#b9)dLaNZ}CAC~IVp{-24Qj1hjbb;HgD zPFMszV>$uy>_*Us*LX}T4vv&Kr@;nggb+nZUd$xUiw2MX=VSlxp~A-o^OQsoS{_n$ zeg)J)VpBf^6EvsBAH4v1Au@BKwgcSYI;Gbubh;xOGWNrf$EOJ{(@$z57C^u|v~0kt zZS>0F5@P>}BvP4_=Cvy^DvGTvJyNK%cv!&(YM)o$3p(x|9ns_<7?Sj(``4AiN4b)@ z|8OX>VduF*D_$c(Mu7St8SsaQJ0E-ZC#-JXFrew&-&k+$kLcC|z7E=U9@vZMdC#}n zNjMJzI_2>B<7MgZoINNTMHz#n&D0L^&@J31-2zV)kTG*x2W>>8idZYEoErtF5~5AD zDmfjU3TV;2ToiKJAk`5@Rm2+>7M&kzXR2Ohr*dOO;Mdvn<!m1UT9Yv2$pRv9_{D8ZEVcU9ME&(&J4)?$Rq7@RsZ10{JiXy`+x*r z25+>wIOGQ@sjbL#nehdpZ6a?}m~nlBBr{&-mX(0_D{^(2r*;q}*?1NxC`#N9pc6 zeSeRCs^`EhD!l@eN`&fbN$c09Dab4T^KgVLEiYlrruRoMo5t$_%0OCv&YPkZ+9M{% z9QaksvVGMlD8-vF34xjrrS~*VaDDa_1;LL*Hbdp7-I`LNtmsFmIUI@Z+=P9nsp*{J z`$g*0L0bt3eyws%=Q@6&1X)JT{oV1l2C?h5)gG>bDVyR=ZQmlE!(Y^XwwgP=8FN%)Cz|1-){%xgncmmF?4CmZ{F67l#JQ2Nd`_@=hL3t0nHhe6$*O$x3 zY@;_Xi~4ADChJw1UW(XLb5PO_D%|H4f+*5DeGYbfvm~aw4 zrkd}*-!dT=IOrqq{FVYF$gMt#|_ z{3w8mZL!Ai;uT*}<;B3Otlp-I>;`V`ZUqnz@Oo*uG+Ji z@KIsRieet|ol>di7rS4-9kmMVU3`}q@dAHWDgPd~KqK^ldXjmT!!u1d6s?r6GdY#(3Aw#`)`}c8iTK9m}%KmQM0f~unJ-ySSxJI1feb7x2lOhQ6;=c9uc&l1It zkM^V^ta#_krp;cmCwrh%vHj9*TxN>#ZJ_1m(OlH#%2y**Fpi(7_Niv+q*x73yKUM_ zSK4$Z{2b*#j=5R8hd{3_=>vk77X8diq{R3R?x)FaXs)gYQg4dnXmjjul05ow<{3Fb zSb)N>{09!NSMiAiE~`_=hYyMmtvpO@%WoZf<)|b2L6rSEdmnQQb2d|8{e`q?QTH4H zjpu?_|4xo_@F}b{Z8;o)`@3_5oFEe#$H68T#*|`}Ff+LPd0F#vhydyuxxty2{M`3A z&!K{ds~^xS1vgn-+Q)2Kv)3BxB!!-HMQx=LvN5%BlYSK*VBC!q<+$#xZYp|9lvq0} zS(Hs9e8WbK-ndl#bfV5(AlsV7)UySB3eiQ>`=%@9QP0yH%m{gi(fe%G{l+`Xqh}PZ zQ)7WGp&Jke9ri`TXFo)JfKJYaoa1OFcgR~Vn<~_TCqT1xW zc9eQXl=q_sN3@wX%`@&=iu~kQ!E8H!MIjDzsV9ksTGgf<2ZUYct~hY|jD1O9lB-K8 zl+?5wdd`mu&{F1bSY-M!RDv@X)YNS=Wzc9Fs4he3IHltpU&|A#EJn_%PF&4SD!;Bw zcr6U?0|dob4{*DDj|CT~0VH~=QC38fPcK1GF3npZ<09IJWz}9aUE1c)FsUe%jE_yO zioEjhCKWk@x3h_>B;T@hZOJj{%7 zm>+*4iNCu&LvbOjucvCtbPcDr1@D8%SoP`n67rEaQE#S0i|0|ubtK0y;o32^bDm*MG5@HDnRI(m+@bRHyL&f@9 zmaZmK;ig1CN{=BXDi1?me`RvsGpLSrxvc?-qFdrEyvQGldma)OD%*hT#XDWO`G3BS z5(`AO&JJYZq)_>Ey2XuDVAOGO(^@oaso2LNKPny#AnDW;&pEJH|6P{+3=E`obdYsL zr+P~zSc`poc2@w^iJYWelCt`5C{f|+lt}-2)^7Au2gI64f@2-}z z8{osal`20aP)2Gz6#4-3-|d? zWGMvJ{k;v6?iHwe9GQX1G%UMrQKK9H3Y!R3P<`Y;Zm#l1wdsAJ~j>sr-y2)j}T6-&!y(k z_FX_N7N<{7>Y?SeCYItxtanJ@vxNcx5HApxZjEe#4_IQ{5bU^@Y&&ph$4;fN^NZy2HLmIT0V&B|eU)JEj%*4^FW!9h7 z-i|^ZNLpO)0tmbxVhosnxClK~`b72pY*E`VaSwcEq6*=Qup84nsg73Htpg+HrM#W` z_s5rqpTmh}^~` zuipn}C#dw_k(<9tjukQ!D}LgeHSYHM!vI8fItZAlslM=kbXkhLggucK#)AZ7xNech zDqkSe=A>9e!lTZKc%O*eZOdE+?vT1dS$1CIyJ!2k&OaXTJ>n3z=u$#jxhsN)Y%Vy0 z?{&Kq7gjTia*hO7O1bv_~%7Rm$ zf_a?f#N?u9cp|;oIg>+9kEXfoa18NmR#N*I;K_K!cnS%5S>QUk7K=&r656!~%&6y; zakCH$2FhKdrduHp1Wp*EYLePcs$`)Us=bg(g>7K1f|}T8-0|_H$u5v0T+K>g3xU+& zf$Qe(Tx?Q5hG^WKMbTu$WYjjDvDBN*Y()5M*ARzPH6ii!SS$D`UcAShFB0h``||DP zJ5@jU0}-hE;@l5%J`GA-MPFnt6%sTx0DCAW8$tB)pvjMg=Y<$`CM-ac9gKH}4V^we z)o6Qh7v=*&kc3)cFN1rH;c#~4YCky^=p~&NJKB0A;vpBm?V@GiYHeS`&uDiGj~hIZ z#9E1@Czp7ha1NNm>N~IT2A(^vVa-U5r3HL-oZ@&Wqr1trZo_EO=g_NBV5lxJ=GN=P zDyNdSyi0dJzj)641qdswi4O^no(-6%Y6&;htyZ;@ET-rRlbgF1R*lKQ0M(tvSb>+A zN(v#q>&U6Ldfta1vR6MAIvYVu>?m0{D=Cg6%KkXTGVBvCc=seukJm!ZwWt)C!i1yoscOe7n)rw7xL+f$Y(5oY0&$2vnV#X9ab_O zvp{`yOd417zA6Aw!K-XW(q^+h(%DeQ!Lu0(tM!m+CX8qRq+?9$!p(xqm^yW^ zCO8sh*mW*{se@s4Zh@6hKe!MMnvO`wqAq{orow?|85N4;FJ=N$)1SuadxKvHy^;ZO zd+%y4K5vduKo!P@odCYs^-g30`p`Lc+N&_iYts2krlzFe8a3`*S%k;9;oFS zLeD5G3LdnZ!8Af>7?XS5dRtdpLte!)4{bwbja+GSKRxab9Eus5S08mpB9;=WRbd^+ zY1gUwOdPKkZnnCW^dl07d1y>ZWuEuC5jurU87DydXQcxsZPDx8 zJcErz$iV_l<5|KSmf6Lm{j4;y!E@ns2e9yxFRxc*ErG0!P1N3) zR%SxmN<-eg8Xv;;b7g;*zt)4Fju=-@&?7VaF$J+BN@c#3ml#L=ydP3LxO|D+0R2iU zpKQ(Td4<5DoE1hscKki7J=RmXB%aLkjQr>3qjmNA+wBznk1H3aTWZ{9+d5`$f2U9j z+#?CP!YFblg?sBdXO8v>U-nvfLGtgDjiiu$tDha-DO-sOacc)vBp3B*2zm4^uXo-* zTq+*Y?7sP??>MnTbXFbbij3C_3RuY9(zcdJOcn!`v`ueIqaT) z$vioaHuc2!WKyW#>&}}fPLCI+am$iN_{rz_s~3a>f0Dy4ESNSfX1V^=|6Lu;w%(V_ z)+TrVLU2%*(C_a@bhD%9g)MRo2rZ%qF7g}o{At$U2++Md6?nhy6bao-5$4Z6a%V8y zxff%Prqg~rB;ddZWRpI82gXO3glhLM!s;N2mq#8} zyZ&ykD7Xma1L%at>t={?j-h4v(PH~?t^xZ*=Vh+Zu-XXv@YWLzH>Q;@we9ySPOhW- zaIp?+<*R{j{Q9gO-)5O-f@yy5y!5DprY-wO%Y;fMmZP+i130u#jlQ~mELyft_8s%S zV?BkuW(6A|5FHpUmIXL#Yj*VC97vQe(z{ugS*RQEccYt&o2hgF8GMY`*Y%TNhe$Sy z2n0dxwO&!>bSxb9&=m6oGXZf=7ZU>BUj|}Q41>JMA~Hm~-hX9?bLq+kyVF zMKKx`X?EjXs0Oz?H}pr>mU8AkWc`|`Eo1cIPF4rMjlh$id61oP{o>+S2G^kM&m5iZ zu#?viOXu>eLSp4^NCZ#RF=15Zqu;9$!w;@T(A3{qt=jw6ERm@j zG91&&{`AO>gKRj4?XjO!j~w;(agF~$^BkySb!D7r2%SGWis*_~-hQ+6JO;GPz16Jl zB@MEjNfzrEaS#8a$ApS1N3?}Zw=wWd37Xp>Q41aU6bt7+U47|A%lhC>q9@njyhHM*#+Xlx5!^qiAMzXWHN@C-m$Sz(UKunQ zF5-D|$EcHQu)}!c+X?GRADwfKEog@XQX~WaLXjnUdN}%C{i0U6Lg7P_E$4=*AqcwQ z)#m^&LVc7&-b~YJFFPS21&2Ok1w4MY%s(8nqljkKIr^hm_Ta_wJng6V&m)SpcAT9x4Y7 ztP_AOi=Ye@g|OuGJ61wwHKK`}ACdC3?Jw0E1~hkM`QeDAl8fa{AMWAT&R2Y3m@(aO zOCqQKZ6?rT%O0(~Uf6@((x&RTy?yj|p&_I#Jg`i7{K_S&p7_|BoR=D&RyXc7<2QQ} z)SAO`PyiZMzNzGh_zv1Yo+yJ#r^Dd4{e6q|G^FO5Qd^p2LE^31bq-%hz7c$XOn~CY z#8oap!^Q|g-CB)wS*j~^fh!GZ#k>GdSmc8Negh_#;+ezUk^vM0mbcishsb-e}&uKYx;4&hX`|Bx!Ym_>%PN8x7&p6QU1}UG8Fvev_GN$ z(Gk$bpW|v0+|cB*$D?Gs6$Jroy^(C9B?>y9=HTit_87fq_LHRAc_g77s+WS-8 z&H*r)F%FruWvMC`tKBdz3?ye0g$f+*E5Etv0(3)qo~S3SDnWtUH}XQaW=-;FN>nf` zr)$Qga2d}hx>x4-x|u$*N_R0&IZw47&ceJq;`TG)&cEaPW_wg{G^F+K_~zh+zHDUN z43)t(vj?%ba<7qExy@-N;ufKZv74^G%q)6NouxkNT<`dI3RxqYh)dyo*lKkYEq80x ztXC{mA{m!|u^A}!({g->b5{#un;!+J4uxK0{KNtck^v#^dq1f!e*9mLOiRQ+DvkNb z4r)^~|Nir3_xpz$p+DlU4pyVIq!t?1E=64_IQn%v`$YIMGz~;v7EB(Td>>$c`Znk5 z@v+^LoZB9HZKsVNox4?{x96E6RD6bfdc7<@je_39>ztCOTZ>hj5x5Z5&74p7`b%v- z2JF?j>y=xnsz%qWDV-i9iJFX@o4(IfJ?%#KXKuG%S}I3jkom1FXT;b$q;Y7tk^e$> zNv_f1A-u1iXj;{`e3o*Pqstx|tb>@bJ)tlEa~?}wAJ>?Ee&=$UhgiP(cgE*kMg24D zij3HDLtb@0unQYt2tJ;hf-61L&AOVeU$m*IeC-8WywV12wtH4v5I8}{Qy9@|Lj6%U zWqcZ%CT~W(`>1X2Tm|N0Q9c=NWpvkf=);J+s>W&y_#IxUUDR@uj$R?q=cT;l?AW>C zUz|caMJU2Px=uhXu#th}m~KI)h^{3y&!fLBJsH_-h z4TQa8NM38sC$^_ncfCupfJn+lvRWK%!ncz*`J%S8*FBXzDTx`0>86UUR2}0a2$gny6B8BIH8m#=7eTBTncdxgFdgKJiz2-ajI+1w6O=xLYQLb+!y_e z2e%fwsMksji{E>-4P}>SeK_QZ=Nw}ilz*TgULv-Wa(SSV9!os`6{HL`X z89m{`P3ubfn$IK`pJ!iWNNd4YPISW|yZy5~e5<(+nZq&G5s)2;nKHWAh0picI+`>v zBz12oLwbUFv71PkK!mqTXYY0uIe2Ld)FdbWlH&yyrgBEuv zB>|?xxSf0Qx0?KRiBtt6xu-XO+}xx4`u*g1t3KXnh{5sm`F?k_7r)SQpg!QrX8Ny8 zJA^}19^P11KijQS9Q9~9Mh{nKLo?J*LIyodj(@BFAbk1mQ3fnxP7z!b8Zb%GCHC; z4aUo<<$Of`8+10;uRov?q%h-+BhGsIcrg@bnU~W;)B|l8ygTk(7DNU!qdgV}OS0%V zqHe|cx<2cA%a3>C!nRLk24k=MGePD6-%Hk7XBBEtItx)aE-k{%+eTXe)n5AP0rsF+ z*6Su63uXjV?+DRa!osH?xW;Q@=ygNDk{TytWk+kxEr>pI|3kjDV zq*jd1lj3sT_TQYX61_FdYnaU*FxUR;@yD-qg_;Q!dLPVoR^L}ofY)Qs>1*GR=|R`o zzsU@`v5hc>?+fB=3$vwMc?-NG1#k9+KtM(z$p{5Wh_t`;42SxB2dKy$ zlgLOg@lv1i_0IW?b3U7$n;$uNE*TSAoWG*@h(}gOq;a0Y7ALu_TE*e~x)Wc|)3eQ7 z>S&YdmoU8Usd^a4qb1y07YISme0^^xJ7dKC2j5bL+)pOQ8GcQL6D5q#RwU)$Eim9q zXZVoWzzgy9V=n?)B57HLkuk^WQSd zvG&fySI-F3$#}`>GaFo2>zHOs;`8NG*M)eWMmMkDS7d7*ms5p1wwTAI+5oUJNyf^| zwX`SPFoSy7frtW9QetV-+;ocKaB#yARQ}-`_Lyr)Uxt`|=PO(#v@j@4vCyIj$kxuK z5)<_*Y})L%%qWcRVd9=cTV?8zm9NKOSZ+Ke9c!E|D!TQ&A@OP5w>TWJ_h9N0VTk_* z!U|$1lvl1_(|x^NCE2WpFP`DHmOLAdF^;+EYw%3q%k|}FPfSk%awD!xy+Lq^{L!n> z=p4?dWu~ zp(hZ==s5a*0Fs{nUc9j((i&4?U$ENPds1{OC{1s}gl>R!dA!bcs8pYK`z4muq57r?tg|MC9K zW)lFY12#Ho1o}9x@P2-0E0|i!E()l(4-n^>$mB6vfR7>WnYI^I+MOXNyXs!g{NOnk1%UN;Uxs zqKp5V_9F?$XF8cpZ2c6bWXdy8yx)kJE!g}3c z?D#+=(v<&epH&nR^ZsF+Ha6T;(rk_#Qu(8<#F}G)>@rhK{55dS3Wex zx_BU4LuYRx42MVuljLoD>rbasqm4oA@tu!Fw-*!Mi9ZJG+p~A7;of<$lnHd!v-K;A zu~BeRsXF?>lGVRf_VP_18KFgqI7*w_Yt;i#!3E0GQzcX-vsVZCyPr)2hR;$O7@hQy zC!&-+^w15kXNnqH!VBTs=6oj@vQ(L`78B6>`V1Rt;s3eWI8Y22TQH~2scPY+j_Pj^ za(@}}+&Y7adWVou{Ywcq4z>s2nv0$SyeIb6Wect&U0AguP&7_pSbbMEkc$87 zl14r;8S6Vspalwxfxv%`z(U$iRrvQ;Ye)NVutR^?eLETli&%|sZ|_^fNuek5;aDQ5S2WWn4t^r5mni zrWcJ0pcX=LeDi|ldo@ETNl6rLVj$m0F!Nd!3ztp3kvw_JYnjYO?KjM6TS4@MXKr_H zRx*k-i+o9Vl>3nqCDOG1YAJBRI^`*_c;7F)cpsA~aU@hyarw7qBEcuI#iiozEYO2Y z5Yq*fjr&c%|5?KIna_72w#bD+jqLLs+v`^Mmbk=P?$XpjEuSs$vZ3vM&z>};*(;M9 znd*}~CCyA%reiG#vezaDck;wMKIYj4AGo_0*&imc8)&1QK4!TdS`6PPUGY+pmT>Nb zW@n(_`>y1b1V*-kz{HEyuNx0Xbcx0VD+E|k{lSOat7*N9Ly#B5z|ICJYL`t?f=kkk zGQ(d7cQ#%HI4u{S`fTftG~Qxxw>x<^hwasm@|LF{o&=+jRNjD*m_IK^mI@oE7W2`H ztJ8KyFU8BJUs?k*cUS2%lbrQm8uuwB-kkHlGAIE&aEa{C$$q`Pt8&?Dq@OXr;Dyaa zln2Oc!*Q45C;YZuO4`4k>pwAzz?OO2rr^!HGQl9uHB&!3*J&aPf)JCP3R@yoImy#N zE}l5PyhMqz5kXS#GAQ`M6I))1T+cE*=E(xT;r9#f-#)oDZJBKiX`Kf0Tzpa+MYoib zcj0!a>CW`p04~p(;nKmmW)(9VE3#i83=KD8PvjmC5iHoSRg;W;GuhuX{yS`?J?qNP zkfRb?T`fuJimS}TIrw&4A6#W+r|Bm`?&fKs-&q}2G!~uONZ5Djl@$HHWI3R5e^AeC z?z>-|l=S4Cq)buU(tz!1nt>mk0s}>lbM2RhUuJIS%H)|edY7npr^W70hlKcikQ~jR zydrPd7ESh}{j!X#jqWdQPG^-OXVB`?P=5xmih!(63zaw>e$fZ1vL$~ikH_1aN+*9S zO*b``2}?Sq-ez8wo)UDSyQfNyVZf43_eMvRkFM5Sw;y?A80XNV;wG1=z*9=XHwf*Logn>E zzWTX50H zYWMcR&uikUlFAqww_pS7HQ+%=+7;B?7KGpYdg@SrP1fq|V4c&)=0`loFgE9|3thtl zUxmI4<*PwLuvE))gi6b=bvN&kq;wAA5-_qRDo)|2xOWk*4qjOGz1WlEJK?gJ5VMw* zW_>JZ%X4zFHzd7NMXMfgI$zW3LMJov(c%pvJ(0ins~{>1B#b)siYNQ_KA?HI9=msA z;Go8_z3%y!ms&g#f6J)j`W|6*W219>=x{f)f8>O|t^06SCx09~Z{`nQ~M7n+h22 zLSavEmt}xK>P+FCFtMKSfY%uEz4!q<|Cds}c2C5{cbC3&L+dNmHb!psj4|dC{-W>O z)s!-U*RA+pap!6P7v$ecKNouMN+CHiOlbh{e*TX#Y^oVoFplbz(uE_?JbjbRr~{%+ zec-&!3DTV*jI`}?2El_Y;jzs-*79g@OZ}6S!K57OH@?NGi&q_IV)x<~i=0w$>{nt6 z<7c8?u-McXJZ8^@LBhq3DQaAgiNsV8FwzOY??sr%oZfVIey8I*%UApYv7RrN%&4-1 zE;wYyMa2I-4XAGI-{MrBqf#RpK?*#$+1V~Uno8w8w%L4 zpzvsbL8-)d?dyYdHRCZ76*|6dA7+N+iutE7b3431@pg0jxh@QZ$i zXe)5Hdcn`?B9MseM@|rr?TGpgsjwF!j4AYXs@TcyqCGFRoKRll&9p!gcx}_q3!1P> z6=*Lt8~L9PZaN*f>d(y1)K5_BHg3dNX)}=#*K@BeH^VL?xyqECfM?YHWvBuZeLRT2 zv_IJlaP+*-a<4KqkNQK66-AUUdKc7$BriQgd6oFJk#K1!+vpll9Bw82+pYXtuOkRVko32RztMq>P)92 zP>0$MjD+;Ikv9yfX1Ie6E$=_5x`xGPon%uh+V~dW^hjq+uz+$&RNzdZ{`wJ#)`bjrVa`K~g zw4As2IOCFz=aHML1(_swa2#`%jGdWepW;{(YqhA+(|TG2o^ARYI?bQGI(EfGY$#o% z7qYqMO;*q&M%l5CKl#zz zV9YJT5+SR6*n<^gtnYYb65DPrNmPP&GyItB1;G$&rN-+7sZXJ;ACI?5M`uCbLSczv z5tEj)+ufdXeIJ~k#T_s!DIoK-g+ROVE{5=Nt+-xY8v|L7F9mYO}|o-WAh3> zTKIV%kWP+@)U+Wc?kOJMu?W-QD=E$A2s1O!cU#TF{q7u&$&_A4@QujB8mc*$k zgpQTGs!}t3Sek&%DMyR<724Y9$hK!7B(^m5GHtB>;M`(qHe8#k;Jq7KJ3&Q%yHQZH z(ZT4Yq*oB<8cxO_Ux;g$UDxVmK2Vy#OBP3$&X*%?*0pMYo{Ih6tBEv-H5E&jHK{Rk z8H&`5#Y63>RK}?Iz4e%nFZK{S_5S?VDH<=`sjZ3ZX!&GV{o5!3^*wUoK~Nsd>rXCH zN{lq#M1(~X{JGv#Nc7lgs>9je;<$c6!{!Ij%np+rMPG1v|MYSj$zDS^=bgOs0`fd@ zc6O1>Zej9h_rX-|Op~y{JBx0eE|3aZ&Gz32(+KfRB{W@U@+;5p@6!c2qp9!l-*K3p zSlsE-)Dto*DE-wvqAy`UUR5ASKoB|IviTy<9^=I4F8|%kxVxchhAjk9hZVYsLUPnBH@4Fbfjl> zx#7OT*-8dtD-?*j`QGx$N}a3@afZm7q#2~%CSX&gOyYX5!F|wMW|DKAAV|_O2_{io zNHRZjsb0!HL7C)wsh2?C>fjx_9z@$7F}wG7r@Z3*Xnl;iibin@FuL z?$-d=y#0-OzS1yZBvjGR>Ch0|s2pyk#+(tyg@pkNNJ^m8~94 z4Iw26^d;E|UXjiZM9(rK;Wj8)>(x5>$BR8nM;HRZWSFE9B@C$yljiL~r>3Rgt2c1kS-E)a-)aOJ+S<_1*XnehZY2TwZ*?jML<~>+jlfhWp26uhTGj_{GvG?l++$ ze#sJm!X;ebUm;1%H^$I*n#In=M+#t4;_>&*_(e=e(K;XyQ|11#(=a~V5W`B*;}c=M zT>B9OQ>xcgwvC90@Y$`!JI;NKDsVS)I#yzTMG3iimJo*5fw%-C^>XA_8g50yd5pE* z-i_X=={t)OFHEe7AVFcg$r8BkrR7GJ4~v@+uQSqdLNCLFB@4B3i2SRxUd%EO^f1ul zzoAc&Ka6EjcFsz4AkNv(c$R@pqDr-a)$w+VwF`qxbJQv{E?+$6y9Wf;m zB@h)y%5lHu`ZWl7xVtUW=mN$~^xQ zXN}BT>ir4e#EM@@T2Cg9jK+&Qip}zv^(Ij>PPBuKT>}7qq4}{ai*b-MvQ=&c#5EV$ z9m~|VC>CW@!rF!y3tZVvmU19%nAWG)@$q0Gq}_xTozMv)X`Z+_GGhhMa?k-gfq20T zCd0O8@e|F-=4UM?&OW!4FB~{AXvCt4HzQ{mjtX5`gyC9$W5)PQxPePXc6y-$t@(q* z#F(fX1eF);bNCSWW87QA>4HuYb0$x^pwH+OChimk}X^zleixbyX6O>`keUn}J^>cX#j#Ch+l=$gU z8O5f?-UmSS@<5<(U`z20N{RFY znZ4&-n{f=D=tIzK>#K>zrmRMxAOxx!Nj0@8F~EMa$X}ISWmvkvGm^(kw(`r5IH@P%B z?Wb5*5Te7?ws>~f!P?>n`SFPd|AI4khkK}JSH8|~(-@?kU4mYKd~bwb-$sx|p)wFgi%){8 z!g%j1p*g8bNTf*9xcqC{f6_4miDw<#Sg|yW!~?VeW84i3QlqLFwIb&I&9ekaHG;&5 zzcGN0vL(xa9_&VIV#RhZDXqi8D*)(GqtqqhXLlQI<1Ha9#I1gaKOGB+VSrulRstY< z1?;nlFU(Z#hljG7SmJ?L)?CJQUzYXyMm8ZGo zf5)W=KMyrTke7n+FU?i>Cq|wJr+#sJo>o{09@n2QHRJqwzF%(#5VwnEeRXd;=tsQj zYNE-TkfHxhm2ZgC{9wCRYQ&tb1LYjmDEq`)AN~UGwMihk&Ep7|lkhR@%?j_~lL`QGeK{OqK zz3nhDwZ3}J8HgGjrW{lc^Ko|u@hA-sYiEF`T?2wxCMpUFL$D$MG4)zMH<^?4Rg_Vo zg`}5y9RGvA0AoU|W`gk=M8970=un3c@G=q^t5^P$ivaG-S@|EGgtj(!cVbAS52eOr zdO`qFlTn{LlHg9jGzC+}y>e$mmW!~b{9BjcA0LydZWkCbEc7x5SQleUVcH%=WZER- zu}LX|QkdsyK&i7WWxRnubo1f#*J#Db9zh#Xd#OQ zT-!jmT9so(i8g`WwUQ)oS1W>t%TepU!xQ`9boJ|<3&53ITF`W^02vE31G(yrc~BkD zc<5-m+H3PvnjNwb7KEPygrn(p*ud#j$7g_OP8Ymh-7uDJir{pwh9#&3F4;2UA8C3` z5K}Orn+P1R@e-_m{cRf}7Uk^#=8?Hh+AeeBXYz4}RDMZ0mC-zonYX0AVp)BkuV%O8j?Qdu21OLD%^1>%@(U4}DtapnE!A~P=Nuj+$=8zd*89sIG z7w`bY^AiBMSlRS3&HzVZbv36M=zrH5>(Fr+Y2;0Q>p$DOMTlTvNDH;w?g0M4u!VOI1GnkH~Yf+u`YGP$(%WXZ9LZgxBVD8{tBb;9^2kwGo>MKn&-c@1H`>-(6jV%G)e4Y?tCOgp(2Mj6H$kC~YXPQ12k z8V7Ug1p4S*N>QtiMGJQk+I43Cs4gG{KJ8z zr|*F0+*?vq(;OU%R+vbY+;mJN*p-=-iHSaRJp&mr@R`=C2V}ig8id83>@Ct6c?20OmhlJQ%PE<~vlR%B5R6^*~AeQ%ICu_@8At_jgCIhv&xQxJxvYK|-F z_C4t58SkWJ#joM+#RleO$#tPsO69w-g5R%tFR52Vv=srJa%b{0j*;46%t+(7nq6jjsJBN!t!xZUKtY4@dTn)&ki9-s)-Q9VMnteG=xGR7-v4B8^^uswKu zdI8?gtZP7(A2a4ngCyE5JrT)!Je(H*>4z-6)Y0l?3IU3+zN{oeHl)fzoc;_Zh9m3v zzO7HOrh_Of##YB^H-MLX+}XJESbYb&Fh2`Ex2MJ}JQM|F0*93B6}fOKjUQhdlE_(3 z1B}fdf}47f!vaK@lHO_iyo&g@vmHR9*nY+dd-cdh zp%m<@c{Y!u{E!er(#IOrYV!BIS;;dxIcH7lvpnNp zqHrJU2z+Fl0i`(`+4tw=mql%NrOXDgng!tvE&3S_7J_e0#b{#sx@|LRNc*!=SbN8V zxY;W3#suBa;Wm_Ku@>3{BFEPK8DYihkuTj~Ov9S0T?6h_ubSv`AaUIiuNTjO3|p9+ z7{pj+_;%1ZLVw!zQ96HUE}Wywq`CdjqCH&A!r`ne5T7E42(dmZq5pNPaNRu!J+P@& zw;3l;fQ*a}%LNral_UUn;B2-;yI=#CsX$`0;UkMoeP$QdiC|^{kIPd+=JV`Mk-_<7>%2K+w!n_*CJ?el-hc?BGd`N(XG5@8&J@juV0W}y=5&KZ-Qa%`hm?qaFJG=Pl^-H2g$7LBVa-uSnkZv@m|$_! zji-`*dIj>9IYxIZB0e6b4LiuEiOPeg-e?-ss#-Zs8s`s5wcee@ORh)fpHih6DkwR9 z(btL!=}5P1jTe^twW#@@n24Ay!J~ry!lU3eg3TApT$ibX`{-c9xG2*sF2ePDnnQm_ z)D!(i*mkbdZ!%=aa}nfpgwRqT2FIb`Z5H z0o$;;4;Dz<>snS=G>k|8jN1;RgpaJ=v3g{5n#_Rs|5c5re2;n_Dk&y$Sdm93g?(5z zd`}<}-Hk7}7_w2B@5|+tXuo=0ASvZf?fLyuQ8K5#Trd9i$fK@TJ_UA@i?De!S?1%g zqfclV`rF-U$GrJd?h~_jK*AoUyrmx_?U=as=2M+mns27TFCduR&iHwKi}HEHB8N|@ zrEv<+GaXB3)Gt{PKB3v6tp%(5;>(-mfcOkPphDxJ+JaW+c<_LDoA4>&4}x(`xc`LtP(=GPKNJf&xteM^=| zn{!?Gi|{&!-96cLd&P1e@xgGw*yHay+7D?iWiJ@8UqN5eCG+ykY^tZCrUQD?j`_r1 zLIv`P^N+t8slBWg-)Rw@4HEs_A~@Q@Gg@yPvFj_j5ip6}>V2*!6F%~=W7~R}Xii_| zTS;UF{Wq*}-S{dtUNxWV2US}QDF0^m+!eno=*#cxB>BK$5W4tGkyS}?hEF@?bnhbZ zhb!QN-cZL||nKj<2KaH-d)EMrrM6r}vP4Ty086Ifp8m|=H#m*{O zIii;1)Dw0X?RlGlja6gGVMMDfc|MEks!K}4xTBl109kF^hQVae`YT^%%S~xvy{hiI z3*Wm?N6S5*l4CYc^+|SAcyp^e`|<_1%DX&6eVN1 zHS_ht=DTb?ijLn$^xhNSr)~%i$a4#NI%fBDOan8YhF z;kmjjydv}^f5UNSc=e8_W#4&+J*?RezNA3B7{`p_4i)g0-a-zh5%InvXpIF;A4X2w zc%F__-Hh-olTgUi8U z{zR$&RIk3gscsgy-Gh1gk{rT1Nn6D!e{FW~<=A2=OgbZblZL(rxmZ z6_5d;PZD~=ZCYRr4|Z$Z6rLizFj)@USJpW5)G2=sIZ#T~0(@>lI}c@iqiLjyjb5Fu zAN`%&xqX2yue$@qWG_TnFwY@UH*U!s{_gIjpN7fl=@KctP(UVnbZb)f6AKWr;~^Jf zh7i_d<4mX17QuUeCNB8zGgRQu*pS~-YPz|gZeu9&O5fbz;}M!uo6Dgm6{m3UcnDl; zAqjY-1y(z;gB^POkAWTByv>P}ighJ8g>#fL-2TX-$=4{yS1ZSXScQ*%v$n*RNl3Pi zq#c!ueP`j^{JqS2?vo%Z&C|~qV(i!R53ZJ zenm@uZfVBz1qZUj>nx3tk7@LTurG!Pxb}hhG`$ON9X{ZA1Liq7h^#qFg??+w>`|PO z&)T(SSev7Dv{dPrqTfVLtK`&_@{|sOY#+MMp=Hj>2(2Fagq`qZ)wfuAOYUntUk@Tx zl)J+N=!*!9jb77{nov)hb?G&eRZyjO=n0P2OIAqlxDFMoLd`u{QKeo)fqd36c@wW* z%zyp%y^Did7}+YnElou3##Tlt$hG968uk#15e89l>PR*X^rK%AIcR`ZIK%%wP8 zPg#{@j*<j-bSIvN1mL|*+A%t#6P7#F==gdF)@)w^`wXvvzc-H0ptaI^gpvkVF) zS+mX?-sH8k#4aOc&`EQXBc?sm$eiFw{qWywQDHc2a^3Bj-g$nJ@l;QX^`5@@ z{i5+~cEWYi!q+YrhC(}a_Qv6Ex6+0J#|jjzc$KHGV@jgEuTz=|4{I!S#xpj~0bN8H z`9xLcq=OnJD^vdhGiUc%K(D;ep1mWP6>5Xk(fZQY{r(l!LKfXQ za@Fg~7*#Lk$T)S{bGk5S_oEF2^W@yN(A!lHW|a>>71MASWJLIo7A4HOm^`6s^((A4AP6T{l3zprVci@Z{P0x?&C;Z1&r&G!gensSZf4Nb}UZ>`Llz?4)S z3J-0Ur3(hjuY$|%{P+#c3*vFTVfB*_*I@Z_v}dc$*+E5D9+T2A$!VpJ9EC{q52<=L zvqfL$x|e-I)aM=c7djCCVMqtjltM(y50k;jiGD)uHK1Jc2PC5Z*su>(Q1IX#QG`O% zb-z($$t2p`j9MAh@2%x|LxBpB$+P+14^s}@#De~>hp#^}^#f5-MR7dev`>9t{NADN zZoss!Wqpun;{OI*ZR6^w3CtVzw$?}zooxD?ZP3f*j*+|gsTl)Z8W)>-DTk~y&(!df z?B3Tdv}KMBl_#Oj5!JwO(%I8f>gsO1Q_1x8p&7~}e!CLI_|r^yp>JGAeFvLF*l{!S zsfWBo{(4XR%789WsGo6p4$e#yLtz?~B$4%D8u_Wdg)CEOlRhv`7xJjTso9y^+JL)F0pU zi}+H#aVem^)mlI-1*wk4)Z(1@s%2B{e{Su2x_vm$CvN+Cc5!3Y1}4RAyWQ3Mq8Oy6 zcsB3V!Y3(EzR1wP{j`S!nXIEWLg^2cE=o&gMja}Kb+4s3T>qcNnCVCa|3UjCXiWg} zN^F?-Z4~!T8%2}P- z2PM>1_C4be=d7=~wSBKUCTu86aI5;(Jc_T{MY|8P>$_8#MML!Zgk2z%p*Nnx)^voh zM~9M4gkMs=Z>KWb)ncmN*uTa_BA<6PCfGE=Rm$j-Y_u4ZOBD9rZ^bDlW3Rw%%xWiN zDw`@)y+>=eFOc?oLJC1T@lIrF(~4w4xh4*`e~@TL(5uXLFV6H)1b*5tqj?F`8{#V> z{rE%H{C1CH;N$q|RQ$m?1pLsQZRl6}0O}eg-QLmSn#h>K@4b`7cSIb>szd{xHF^BJ zPq~}(ZhZA{HYR+#%>z%!(0R?hc1 z-+YAv56|bVeqUGrguGk7y3;~Q_NeJQUoGT1v&KWtVy>T7JDhfFk0e*@|8O)q&{jWN zm_S2@CL;<-288sWkbs_3#&r%Q5X1%q{bYGNp7X~V`$&%7vovES18)wWTSaxt7#^ci zu@T*nt1m<8)Sbq_m!A^cci^YQo+sYLn-M&bxfy^p646#z*stcj5ErVj!zAi6;Ct`1 z2={+FaH*c?wN9UryMK?-LGHeT>G0TBe5a`bFS8$K!q~Ug>piJnOg`_|j|-Iul8+4v zXarI#FNuk-Z%vo!$mMI;hP#Zu$~AtL;G@QOGM{|?+M-Mn-vUd%Ur?OhT8Z_Z{~O&y zH`)7B6P>F34JmCz)Slvl;|)3ny*oEvSKXRKnXSx$-n4B$ImsaD!jpIj9KrP_LHtMG zQVAFfwx=1&pR$iB+uH3u?n)Rz@}QI#Ol&!d1haB?&tAeS*nEwiUe#7POEJy&CZ__n@(Ip z)tV>4RYuXS_IRXZWwQ_jD$wnu(>SMaDoD18 z>J0N1Fcu!_^1^ec+-(7ayOxlp9yf)HSemG>smAh9wi3?-st>EHNve}kzK+zQk_zO#Jqyudl#uFASuuJ zs5f6t*bGhJYkY(DZVZi7-DjI}_wzi}CXH>KBs{xA;1f^iW#y;V7P}V|t5|TOIAm)E zWbid~5K8gPQIvlQ;;g-ynEO)PQov&LhxK7zV4@0ru@;59yS@Yz zenjzIDIykF-fpk8sm?6@K9N4esLv&7{#L8KZBmK68p3O_d!%GE5Z7;AUixM4e$-js z6Ge3e0YChoy2te0JyA9tNeXEfJ=JoimgPeb+mLQDfx)fZ&Ny{%0doXNek9of>$N?D zDBEA_UdHfh%GWyvl)RR6sVMU9K=%IGyaS$dvvX5K++5uG_@6;_EgL%mhY|yx5(4^X ziackk9Y0nPaqwRsne{ZI~ ztm1CRdl~uCE7~=J{zS;xB^D>e9{OkdlIkxwB1JUD(W)ygd-E{@@BG=aEZ&D)yCiwx zK*7vU60n^G`_>w;5PU9vew=YHF73wrlbnOSgwU732#yDqPW5i>A|Cu>c6r_V`^Zu< zff)M;n`)!N?~g7O7VV#)WhOe)`{eh=GkR+I_Q0clWCl0lBTt4d{B8$%`PVZvKf9y; z^EA4GytaL^qdJ}`9hnyE3(kZ=4qw=Kq+Q9&ac35V608&I^CicSPj?-0Kw#Ob_H!qy zmF>dZcnj{d-w;p4Y=uX{2iZ17?e?f%+HoP1mqtZYsOe4{`WP*q7CNI% z1YhsAUPT}Teu3a*YavYIx4CjlRCP5uH=eySNA5?W&0b+;rEL=W@0P~0O0E=dbzn8~ zaEovvXHjEU!*>_C)OzS!Y5__kxy3-5}9v0xXjw0))^Ol)0|a6S%Ow2 zC2s6Ik1Foc*#E*Y2?*j}cqN&bJKFafkb5jQiq;cC+j=)BW5K!2vXBA6xjw7xoh;`7f0D z0n-w~_BVy)3DCF6cddH8cq_wEcgnr-ZqAnD zkXkrG)<0w|Abxla6?EnJzjtsSM8@K191#^7I2hLdYzXYW5S%H9X!sV6W%QptaN~qU zu{JJ7|J@LfjtGEsfIPhZOVNKfj8~3@s0$|!f)?IaKfg?(t_!~P1nj6(Z2<_$&;fBl z2ajLsHzr+Gzj3SZm;k4?btS@?7`lFQ36Xg0=5kTxrGxF&DZ-;h)- z!_i1DD%Kt6O4Hfu(HPa%8VCayzNB1p#gT_V?%hY1eGk!qDEH6lm4XgqR8?gyh||w$ zHVaMFmS1aeU@+x3Q0IB)edlX9c-lWBgG0*cA1gxsJ*Hlq=EW9e^UeC7A-870xntNPc_!EFAf{Lhquy+WnHHxp*U?)m=%BGKvu%5v1$HGzPyd{7VB~Kzt1sVnU4rHW1FCJ>Gb=C0p;9pHV@~f>bpvxub({l z_Xrun*`bC4(d?a{t^Pd|0LeDo$w=t`>gKocB z^5CRken;@-+W|=MKW~Ts<2{uy{CfT0hZ@8O*VPB_hUdR0uMh??i6=bmp;;^q5Y*Iq zWAN`I>ac_>UFwQ{Ap_#sJ~XViTHo|K19y| literal 0 HcmV?d00001 diff --git a/docs/guide/3-developer-guide/3-programming/1-overview/index.md b/docs/guide/3-developer-guide/3-programming/1-overview/index.md index c3f89b7788a..171905f53e1 100644 --- a/docs/guide/3-developer-guide/3-programming/1-overview/index.md +++ b/docs/guide/3-developer-guide/3-programming/1-overview/index.md @@ -29,3 +29,32 @@ Assumed square matrix with 20x20 matrix with 5x5 blocks and a 2x2 processor grid ![DBCSR distribution over processors](./dbcsr_dist.png) ![DBCSR block scheme](./dbcsr_blocks.png) + +# List of standard compiler flags + +* OpenMP flag to enable multi-threaded parallelization, e.g. `-fopenmp` for GNU and Intel compilers. +* Warnings, e.g. `-Werror=aliasing -Werror=ampersand -Werror=c-binding-type -Werror=intrinsic-shadow -Werror=intrinsics-std -Werror=line-truncation -Werror=tabs -Werror=target-lifetime -Werror=underflow -Werror=unused-but-set-variable -Werror=unused-variable -Werror=unused-dummy-argument -Werror=conversion -Werror=zerotrip -Werror=uninitialized -Wno-maybe-uninitialized` for GNU compiler. +* Error checkings (only `Coverage` and `Debug` builds), e.g. `-fcheck=all -ffpe-trap=invalid,zero,overflow -fbacktrace -finit-real=snan -finit-integer=-42 -finit-derived -Werror=realloc-lhs -finline-matmul-limit=0` for GNU compiler. + +# List of Macros used in the code + +| Macro | Explanation | Language | +|-|-|-| +| `__parallel` | Enable MPI runs | Fortran | +| `__MPI_VERSION=N` | DBCSR assumes that the MPI library implements MPI version 3. If you have an older version of MPI (e.g. MPI 2.0) available you must define `-D__MPI_VERSION=2` | Fortran | +| `__NO_MPI_THREAD_SUPPORT_CHECK` | Workaround for MPI libraries that do not declare they are thread safe (funneled) but you want to use them with OpenMP code anyways | Fortran | +| `__MKL` | Enable use of optimized Intel MKL functions | Fortran +| `__NO_STATM_ACCESS`, `__STATM_RESIDENT` or `__STATM_TOTAL` | Toggle memory usage reporting between resident memory and total memory. In particular, macOS users must use `-D__NO_STATM_ACCESS` | Fortran | +| `__NO_ABORT` | Avoid calling abort, but STOP instead (useful for coverage testing, and to avoid core dumps on some systems) | Fortran | +| `__LIBXSMM` | Enable [LIBXSMM](https://github.com/hfp/libxsmm/) link for optimized small matrix multiplications on CPU | Fortran | +| `__ACCELERATE` | Must be defined on macOS when Apple's Accelerate framework is used for BLAS and LAPACK (this is due to some interface incompatibilities between Accelerate and reference BLAS/LAPACK) | Fortran | +| `NDEBUG` | Assertions are stripped ("compiled out"), `NDEBUG` is the ANSI-conforming symbol name (not `__NDEBUG`). Regular release builds may carry assertions for safety | Fortran, C, C++ | +| `__CRAY_PM_ACCEL_ENERGY` or `__CRAY_PM_ENERGY` | Switch on collectin energy profiling on Cray systems | Fortran | +| `__DBCSR_ACC` | Enable Accelerator compilation | Fortran, C, C++ | +| `__CUDA_PROFILING` | To turn on Nvidia Tools Extensions. It requires to link `-lnvToolsExt` | Fortran, C, C++ | +| `__CUDA` | Enable CUDA acceleration | C, C++ | +| `__HIP` | Enable HIP acceleration | C, C++ | + + + + diff --git a/examples/Makefile b/examples/Makefile deleted file mode 100644 index 946305a59dc..00000000000 --- a/examples/Makefile +++ /dev/null @@ -1,33 +0,0 @@ - -DBCSRDIR ?= ../install - -FC = mpifort -CXX = mpicxx -FLAGS = -O3 -fopenmp -INCLUDES = -I$(DBCSRDIR)/include -FCFLAGS = $(FLAGS) $(INCLUDES) -fno-omit-frame-pointer \ - -funroll-loops -ffree-form -std=f2008ts -fimplicit-none -CXXFLAGS = $(FLAGS) $(INCLUDES) -std=c++11 -LIBS = -L$(DBCSRDIR)/lib -ldbcsr -LIBS += -L${LAPACK_PATH}/lib -llapack -lblas -# Add the MPI Fortran library (check with `mpifort -show`): -# OpenMPI : -lmpi_mpifh -# MPICH : -lmpifort -CLIBS = $(LIBS) -lgfortran -lmpi_mpifh - -# Fortran examples -all: dbcsr_example_1.x dbcsr_example_2.x dbcsr_example_3.x - -# Include C/C++ examples if the dbcsr.h is installed -ifneq (,$(wildcard $(DBCSRDIR)/include/dbcsr.h)) -all: dbcsr_example_3_cpp.x -endif - -%.x: %.F - $(FC) $(FCFLAGS) $< $(LIBS) -o $@ - -%_cpp.x: %.cpp - $(CXX) $(CXXFLAGS) $< $(CLIBS) -o $@ - -clean: - rm -rf *.x diff --git a/src/acc/acc_event.cpp b/src/acc/acc_event.cpp index 473b30da871..9bf389ba208 100644 --- a/src/acc/acc_event.cpp +++ b/src/acc/acc_event.cpp @@ -26,7 +26,7 @@ extern "C" int acc_event_create(void** event_p){ ACC(Event_t)* acc_event = (ACC(Event_t)*) *event_p; ACC(Error_t) cErr = ACC(EventCreate)(acc_event); - if(verbose_print) printf("EventCreate) : %p -> %d\n", *event_p, *acc_event); + if(verbose_print) printf("EventCreate) : %p -> %ld\n", *event_p, (long int)*acc_event); if (acc_error_check(cErr)) return -1; if (acc_error_check(ACC(GetLastError)())) return -1; return 0; @@ -52,7 +52,7 @@ extern "C" int acc_event_record(void* event, void* stream){ ACC(Event_t)* acc_event = (ACC(Event_t)*) event; ACC(Stream_t)* acc_stream = (ACC(Stream_t)*) stream; - if(verbose_print) printf("EventRecord): %p -> %d, %p -> %d\n", acc_event, *acc_event, acc_stream, *acc_stream); + if(verbose_print) printf("EventRecord): %p -> %ld, %p -> %ld\n", acc_event, (long int)*acc_event, acc_stream, (long int)*acc_stream); ACC_API_CALL(EventRecord, (*acc_event, *acc_stream)); return 0; } diff --git a/src/acc/libsmm_acc/README.md b/src/acc/libsmm_acc/README.md index 2c2f4538398..dca8c37a04a 100644 --- a/src/acc/libsmm_acc/README.md +++ b/src/acc/libsmm_acc/README.md @@ -76,7 +76,7 @@ Follow the [predictive modeling procedure](predict/README.md) 2. Add the GPU to the `arch_number` data structure in [`kernels/smm_acc_predict.py`](kernels/smm_acc_predict.py) -3. Add the necessary code for setting `ARCH_NUMBER` correctly in the [`Makefile`](../../../../Makefile) and in the [`CMakeLists`](CMakeLists.txt). Also add this GPU to the list of `SUPPORTED_CUDA_ARCHITECTURES` or `SUPPORTED_HIP_ARCHITECTURES` in the [`CMakeLists`](CMakeLists.txt). +3. Add the necessary code for setting `ARCH_NUMBER` correctly in the [`CMakeLists`](CMakeLists.txt). Also add this GPU to the list of `SUPPORTED_CUDA_ARCHITECTURES` or `SUPPORTED_HIP_ARCHITECTURES` in the [`CMakeLists`](CMakeLists.txt). 4. Add a minimal JSON file `parameters_GPU.json`, containing: diff --git a/src/acc/libsmm_acc/libsmm_acc.cpp b/src/acc/libsmm_acc/libsmm_acc.cpp index fc8d4b739c2..4b6db01012d 100644 --- a/src/acc/libsmm_acc/libsmm_acc.cpp +++ b/src/acc/libsmm_acc/libsmm_acc.cpp @@ -82,7 +82,7 @@ inline void validate_kernel(ACC_DRV(function)& kern_func, ACC_DRV(stream) stream ACC_API_CALL(Memset, (h->d_mat_c, 0, h->n_c * m * n * sizeof(double))); void *args[] = { &h->d_stack, &h->n_stack, &h->d_mat_a, &h->d_mat_b, &h->d_mat_c }; - int res = launch_kernel_from_handle(kern_func, ((h->n_stack + grouping - 1) / grouping), threads, stream, args); + launch_kernel_from_handle(kern_func, ((h->n_stack + grouping - 1) / grouping), threads, stream, args); ACC_API_CALL(Memcpy, (h->mat_c, h->d_mat_c, h->n_c * m * n * sizeof(double), ACC(MemcpyDeviceToHost))); // Validate the kernel based on results @@ -366,4 +366,3 @@ extern "C" int libsmm_acc_transpose (const int *trs_stack, int offset, int nblks return 0; // maximum size over any dimension return libsmm_acc_transpose_d(trs_stack, offset, nblks, (double *) buffer, m, n, *((ACC_DRV(stream) *) stream)); } - diff --git a/src/acc/libsmm_acc/libsmm_acc_benchmark.cpp b/src/acc/libsmm_acc/libsmm_acc_benchmark.cpp index e36229f5d2c..f62eae448b2 100644 --- a/src/acc/libsmm_acc/libsmm_acc_benchmark.cpp +++ b/src/acc/libsmm_acc/libsmm_acc_benchmark.cpp @@ -264,7 +264,7 @@ int libsmm_acc_benchmark(libsmm_acc_benchmark_t* h, } - int n_iter, n_warm; + int n_iter{0}, n_warm{0}; switch(h->mode){ case tune: case timing: // for larger matrices few iteration give enough statistics diff --git a/src/acc/libsmm_acc/notebooks/libsmm_acc_predictive_modeling_features.png b/src/acc/libsmm_acc/notebooks/libsmm_acc_predictive_modeling_features.png index 54ef004abeb..b77db205b4d 120000 --- a/src/acc/libsmm_acc/notebooks/libsmm_acc_predictive_modeling_features.png +++ b/src/acc/libsmm_acc/notebooks/libsmm_acc_predictive_modeling_features.png @@ -1 +1 @@ -../../../../docs/images/libsmm_acc_predictive_modeling_features.png \ No newline at end of file +../../../../docs/media/images/libsmm_acc_predictive_modeling_features.png \ No newline at end of file diff --git a/src/mm/dbcsr_mm.F b/src/mm/dbcsr_mm.F index d767ec080e1..e9770c55d0c 100644 --- a/src/mm/dbcsr_mm.F +++ b/src/mm/dbcsr_mm.F @@ -978,7 +978,8 @@ SUBROUTINE dbcsr_multiply_generic(transa, transb, & END SUBROUTINE dbcsr_multiply_generic SUBROUTINE check_openmpi_rma() - !! Check if RMA is used with buggy OpenMPI (version<1.10.4 or 2.0.1) + !! Check if RMA is used with buggy OpenMPI + !! Currently tracked buggy versions: <1.10.4, 2.0.0, 2.1.x, 3.1.x CHARACTER(LEN=mp_max_library_version_string) :: mpi_library_version INTEGER :: ipos, jpos, major, minor, patch, & resultlen, jpos_next @@ -1003,8 +1004,8 @@ SUBROUTINE check_openmpi_rma() jpos = ipos + jpos - 2 READ (mpi_library_version(ipos:jpos), *) major - ! Open MPI 3+ is fine - IF (major .GT. 2) & + ! Open MPI 4+ is fine + IF (major .GT. 3) & RETURN ! Read minor version @@ -1028,11 +1029,13 @@ SUBROUTINE check_openmpi_rma() READ (mpi_library_version(ipos:jpos), *) patch IF ((major .LT. 2 .AND. (minor .LT. 10 .OR. (minor .EQ. 10 .AND. patch .LT. 4))) .OR. & - (major .EQ. 2 .AND. minor .EQ. 0 .AND. patch .EQ. 0)) THEN + (major .EQ. 2 .AND. minor .EQ. 0 .AND. patch .EQ. 0) .OR. & + (major .EQ. 2 .AND. minor .EQ. 1) .OR. & + (major .EQ. 3 .AND. minor .EQ. 1)) THEN CALL dbcsr_warn(__LOCATION__, "You are using an OpenMPI version with buggy RMA: --- "// & mpi_library_version(1:resultlen)// & " --- We disable RMA to prevent errors. "// & - "Please install a new OpenMPI version (>=1.10.4 or 2.0.1). ") + "Please install a new OpenMPI version (>=4.0.1). ") CALL dbcsr_set_config(use_mpi_rma=.FALSE.) ENDIF diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ddc1cbe4467..26c06832874 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,6 +31,10 @@ file(GLOB DBCSR_PERF_TESTS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "inputs/*.perf" ) +if ("${MPI_Fortran_LIBRARY_VERSION_STRING}" MATCHES "Open MPI v2.1" OR "${MPI_Fortran_LIBRARY_VERSION_STRING}" MATCHES "Open MPI v3.1") + list(FILTER DBCSR_PERF_TESTS EXCLUDE REGEX "_rma") +endif () + foreach (dbcsr_perf_test ${DBCSR_PERF_TESTS}) if (USE_MPI) separate_arguments(MPIEXEC_PREFLAGS) diff --git a/tests/Makefile.inc b/tests/Makefile.inc deleted file mode 100644 index 9516c9c42e9..00000000000 --- a/tests/Makefile.inc +++ /dev/null @@ -1,45 +0,0 @@ -# Add unit tests to be executed by test target in Makefile -UNITTESTS = dbcsr_unittest1 dbcsr_unittest2 dbcsr_unittest3 dbcsr_tensor_unittest dbcsr_test_csr_conversions dbcsr_tas_unittest -# Input tests for the dbcsr_performance_driver -PERFTESTS = $(wildcard $(TESTSDIR)/inputs/*.perf) - -SRC_TESTS += dbcsr_performance_driver.F -dbcsr_performance_driver: BIN_DEPS = dbcsr_performance_multiply.o - -SRC_TESTS += dbcsr_test_csr_conversions.F -dbcsr_test_csr_conversions: BIN_DEPS = - -SRC_TESTS += dbcsr_unittest1.F dbcsr_unittest2.F dbcsr_unittest3 -dbcsr_unittest1: BIN_DEPS = dbcsr_test_add.o dbcsr_test_multiply.o -dbcsr_unittest2: BIN_DEPS = dbcsr_test_multiply.o -dbcsr_unittest3: BIN_DEPS = dbcsr_test_multiply.o -.NOTPARALLEL: dbcsr_unittest1 dbcsr_unittest2 dbcsr_unittest3 # Dependency because both tests share an object file - -SRC_TESTS += dbcsr_tensor_unittest.F -dbcsr_tensor_unittest: BIN_DEPS = - -SRC_TESTS += dbcsr_tas_unittest.F -dbcsr_tas_unittest: BIN_DEPS = - -ifneq ($(NVCC),) -UNITTESTS += libsmm_acc_unittest_multiply libsmm_acc_unittest_transpose libsmm_acc_timer_multiply - -libsmm_acc_unittest_multiply.cpp: $(DBCSRHOME)/tests/generate_libsmm_acc_unittest_multiply.py $(DBCSRHOME)/tests/libsmm_acc_unittest_multiply.cpp.template - $(DBCSRHOME)/tests/generate_libsmm_acc_unittest_multiply.py --base_dir $(DBCSRHOME) --gpu_version=$(GPUVER) - -libsmm_acc_unittest_multiply: libsmm_acc_unittest_multiply.cpp - -SRC_TESTS += libsmm_acc_unittest_multiply.cpp -libsmm_acc_unittest_multiply: BIN_DEPS = - -SRC_TESTS += libsmm_acc_unittest_transpose.cpp -libsmm_acc_unittest_transpose: BIN_DEPS = - -libsmm_acc_timer_multiply.cpp: $(DBCSRHOME)/tests/generate_libsmm_acc_timer_multiply.py $(DBCSRHOME)/tests/libsmm_acc_timer_multiply.cpp.template - $(DBCSRHOME)/tests/generate_libsmm_acc_timer_multiply.py --base_dir $(DBCSRHOME) --gpu_version=$(GPUVER) - -libsmm_acc_timer_multiply: libsmm_acc_timer_multiply.cpp - -SRC_TESTS += libsmm_acc_timer_multiply.cpp -libsmm_acc_timer_multiply: BIN_DEPS = -endif diff --git a/tests/libsmm_acc_timer_multiply.cpp.template b/tests/libsmm_acc_timer_multiply.cpp.template index ae6e4078c04..27fdc8b727d 100644 --- a/tests/libsmm_acc_timer_multiply.cpp.template +++ b/tests/libsmm_acc_timer_multiply.cpp.template @@ -19,7 +19,7 @@ std::vector combinations(std::vector to_combine){ std::vector v; - int len = to_combine.size(); + size_t len = to_combine.size(); for(size_t i=0; i Date: Sun, 8 Mar 2020 18:38:01 -0500 Subject: [PATCH 2/6] Fix coverage --- cmake/CustomTargets.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/CustomTargets.cmake b/cmake/CustomTargets.cmake index 10ba04a2478..f8d134aa19e 100644 --- a/cmake/CustomTargets.cmake +++ b/cmake/CustomTargets.cmake @@ -32,7 +32,7 @@ find_program(GENHTML_EXE genhtml ) add_custom_target(cov-info - COMMAND "${LCOV_EXE}" --directory "${CMAKE_BINARY_DIR}" --base-dir "${CMAKE_SOURCE_DIR}" --exclude "${CMAKE_SOURCE_DIR}/tests/*" --no-external --capture --output-file coverage.info + COMMAND "${LCOV_EXE}" --directory "${CMAKE_BINARY_DIR}" --base-dir "${CMAKE_SOURCE_DIR}" --no-external --capture --output-file coverage.info COMMAND "${LCOV_EXE}" --list coverage.info VERBATIM BYPRODUCTS coverage.info From 9b1b3721db6a82599022e342aea76c1c270e63c4 Mon Sep 17 00:00:00 2001 From: Alfio Lazzaro Date: Sun, 8 Mar 2020 18:50:54 -0500 Subject: [PATCH 3/6] Fix lcov path --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a4cef5fc1af..dd9db1a42fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -161,6 +161,7 @@ after_success: # macOS needs `--gcov-tool /usr/local/bin/gcov-?`, but even then I couldn't make the Coverage run work - | if [ "$TRAVIS_OS_NAME" = "linux" ] ; then + cd build cmake --build . -- cov-info bash <(curl -s https://codecov.io/bash) -f coverage.info || echo "Codecov did not collect coverage reports" fi From a93586afbe247f8e0539824b32484db2b9a1f07d Mon Sep 17 00:00:00 2001 From: Alfio Lazzaro Date: Sun, 8 Mar 2020 19:17:40 -0500 Subject: [PATCH 4/6] Update fypp to v3.0 --- tools/build_utils/fypp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_utils/fypp b/tools/build_utils/fypp index 679566c6a4b..7895a7efb7d 160000 --- a/tools/build_utils/fypp +++ b/tools/build_utils/fypp @@ -1 +1 @@ -Subproject commit 679566c6a4b67d5484aafe2accd513b8bd9eacd8 +Subproject commit 7895a7efb7d2f07dc284cece6cc9474297b8dc55 From cc384f7e11049131f0ab76341f254b5109ccf57a Mon Sep 17 00:00:00 2001 From: Alfio Lazzaro Date: Sun, 8 Mar 2020 19:27:13 -0500 Subject: [PATCH 5/6] Update fprettify to current master --- tools/fprettify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/fprettify b/tools/fprettify index 63b74ea0403..c27fff438bb 160000 --- a/tools/fprettify +++ b/tools/fprettify @@ -1 +1 @@ -Subproject commit 63b74ea040356b22c545a7e76e4c29f7079a0f9a +Subproject commit c27fff438bb0124c854962b9fa4eb4b3b1b40030 From cf129f2c40844eda4146e411a18fe9f34d1d4b00 Mon Sep 17 00:00:00 2001 From: Alfio Lazzaro Date: Mon, 9 Mar 2020 02:55:01 -0500 Subject: [PATCH 6/6] Bump version to 2.1.0-rc6 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4d70fc6fee7..685d7686221 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,6 @@ MAJOR = 2 MINOR = 1 -PATCH = 0-rc5 +PATCH = 0-rc6 # A specific DATE (YYYY-MM-DD) fixes an official release, otherwise # it is considered Development version. DATE =