diff --git a/.github/workflows/docker-bases.yml b/.github/workflows/docker-bases.yml index 7eb281d89d..ba1582ac2a 100644 --- a/.github/workflows/docker-bases.yml +++ b/.github/workflows/docker-bases.yml @@ -1,5 +1,9 @@ name: Build base compilers docker images +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: push: paths: @@ -16,11 +20,11 @@ on: jobs: ####################################################### -################### CPU ############################### +############## Basic gcc CPU ########################## ####################################################### deploy-cpu-bases: - name: ${{ matrix.tag }} - runs-on: ${{ matrix.runner }} + name: "cpu-base" + runs-on: ubuntu-latest env: DOCKER_BUILDKIT: "1" @@ -46,34 +50,65 @@ jobs: - name: cleanup run: docker system prune -a -f - - name: Docker image + - name: GCC image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.cpu' push: true - build-args: | - 'arch=gcc' + target: 'gcc' + build-args: 'arch=gcc' tags: 'devitocodes/bases:cpu-gcc' - - name: Docker image +####################################################### +############## Intel OneApi CPU ####################### +####################################################### + deploy-oneapi-bases: + name: "oneapi-base" + runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: "1" + + steps: + - name: Checkout devito + uses: actions/checkout@v3 + + - name: Check event name + run: echo ${{ github.event_name }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: cleanup + run: docker system prune -a -f + + - name: ICX image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.cpu' push: true - build-args: | - 'arch=icx' + target: 'icx' + build-args: 'arch=icx' tags: 'devitocodes/bases:cpu-icx' - - name: Docker image + - name: ICC image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.cpu' push: true - build-args: | - 'arch=icc' + target: 'icc' + build-args: 'arch=icc' tags: 'devitocodes/bases:cpu-icc' ####################################################### @@ -107,42 +142,38 @@ jobs: - name: cleanup run: docker system prune -a -f - - name: Docker image + - name: NVC image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.nvidia' push: true - build-args: | - 'arch=nvc' + target: 'nvc' + build-args: 'arch=nvc' tags: 'devitocodes/bases:nvidia-nvc' - - name: Docker image + - name: NVCC image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.nvidia' push: true - build-args: | - 'arch=nvcc' + target: 'nvcc' + build-args: 'arch=nvcc' tags: 'devitocodes/bases:nvidia-nvcc' - - name: Docker image + - name: NVC host image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.nvidia' push: true - build-args: | - 'arch=nvc-host' + target: 'nvc-host' + build-args: 'arch=nvc-host' tags: 'devitocodes/bases:cpu-nvc' ####################################################### ################### Nvidia clang ###################### -# # clang is only officialy supporting cuda<11.5. The latest nvidia sdk with cuda 11.5 is 21.11 (last of 2021) -# # We cannot use newer ones such as 22-1 because the package manager automatically installs the latest 2022 -# # that re-install the latest (11.7) version on top of it. -# # For more info check https://llvm.org/docs/CompileCudaWithLLVM.html and the Prerequisites ####################################################### deploy-nvidia-clang-base: name: "nvidia-clang-base" @@ -172,15 +203,14 @@ jobs: - name: cleanup run: docker system prune -a -f - - name: Docker image + - name: Nvidia clang image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.nvidia' push: true - build-args: | - 'arch=clang' - 'ver=nvhpc-21-11' + target: 'clang' + build-args: 'arch=clang' tags: 'devitocodes/bases:nvidia-clang' ####################################################### @@ -214,20 +244,22 @@ jobs: - name: cleanup run: docker system prune -a -f - - name: Docker image + - name: AMD image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.amd' push: true + target: 'aomp' tags: devitocodes/bases:amd - - name: Docker image + - name: AMD HIP image uses: docker/build-push-action@v3 with: context: . file: './docker/Dockerfile.amd' push: true + target: 'hip' build-args: | arch=hip tags: devitocodes/bases:amd-hip \ No newline at end of file diff --git a/.github/workflows/docker-devito.yml b/.github/workflows/docker-devito.yml index a5f127b8d4..59aeae8b79 100644 --- a/.github/workflows/docker-devito.yml +++ b/.github/workflows/docker-devito.yml @@ -18,38 +18,38 @@ jobs: fail-fast: false matrix: include: - - base: 'bases:nvidia-nvc' + - base: 'test-bases:nvidia-nvc' tag: 'nvidia-nvc' flag: '--gpus all' test: 'tests/test_gpu_openacc.py tests/test_gpu_common.py' runner: ["self-hosted", "nvidiagpu"] - - base: 'bases:nvidia-clang' + - base: 'test-bases:nvidia-clang' tag: 'nvidia-clang' flag: '--gpus all' test: 'tests/test_gpu_openmp.py tests/test_gpu_common.py' runner: ["self-hosted", "nvidiagpu"] # Runtime gpu flags from https://hub.docker.com/r/rocm/tensorflow/ - - base: 'bases:amd' + - base: 'test-bases:amd' tag: 'amd' flag: '--network=host --device=/dev/kfd --device=/dev/dri --ipc=host --group-add video --group-add 109 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined' test: 'tests/test_gpu_openmp.py' runner: ["self-hosted", "amdgpu"] - - base: 'bases:cpu-gcc' + - base: 'test-bases:cpu-gcc' tag: "gcc" flag: '' test: 'tests/test_operator.py' runner: ubuntu-latest - - base: 'bases:cpu-icc' + - base: 'test-bases:cpu-icc' tag: "icc" flag: '' test: 'tests/test_operator.py' runner: ubuntu-latest - - base: 'bases:cpu-icx' + - base: 'test-bases:cpu-icx' tag: "icx" flag: '' test: 'tests/test_operator.py' @@ -90,16 +90,16 @@ jobs: tags: | type=raw,value=${{ matrix.tag }}-dev type=raw,value=${{ matrix.tag }}-latest,enable=${{ github.event_name == 'release' }} - type=raw,value=latest,enable=${{ matrix.base == 'bases:cpu-gcc' }} + type=raw,value=latest,enable=${{ matrix.base == 'test-bases:cpu-gcc' }} type=semver,pattern={{raw}},prefix=${{ matrix.tag }}-,enable=${{ github.event_name == 'release' }} # Legacy "gpu" tag - type=raw,value=gpu-dev,enable=${{ matrix.base == 'bases:nvidia-nvc' }} - type=semver,pattern={{raw}},prefix=gpu-,enable=${{ github.event_name == 'release' && matrix.base == 'bases:nvidia-nvc' }} - type=semver,pattern={{raw}},value=gpu-latest,enable=${{ github.event_name == 'release' && matrix.base == 'bases:nvidia-nvc' }} + type=raw,value=gpu-dev,enable=${{ matrix.base == 'test-bases:nvidia-nvc' }} + type=semver,pattern={{raw}},prefix=gpu-,enable=${{ github.event_name == 'release' && matrix.base == 'test-bases:nvidia-nvc' }} + type=semver,pattern={{raw}},value=gpu-latest,enable=${{ github.event_name == 'release' && matrix.base == 'test-bases:nvidia-nvc' }} # Legacy "cpu" tag - type=raw,value=cpu-dev,enable=${{ matrix.base == 'bases:cpu-gcc' }} - type=semver,pattern={{raw}},prefix=cpu-,enable=${{ github.event_name == 'release' && matrix.base == 'bases:cpu-gcc' }} - type=semver,pattern={{raw}},value=cpu-latest,enable=${{ github.event_name == 'release' && matrix.base == 'bases:cpu-gcc' }} + type=raw,value=cpu-dev,enable=${{ matrix.base == 'test-bases:cpu-gcc' }} + type=semver,pattern={{raw}},prefix=cpu-,enable=${{ github.event_name == 'release' && matrix.base == 'test-bases:cpu-gcc' }} + type=semver,pattern={{raw}},value=cpu-latest,enable=${{ github.event_name == 'release' && matrix.base == 'test-bases:cpu-gcc' }} - name: Check tags run: echo "${{ steps.meta.outputs.tags }}" diff --git a/.github/workflows/pytest-core-mpi.yml b/.github/workflows/pytest-core-mpi.yml index 57d2c5b2fc..d6ca8e80e8 100644 --- a/.github/workflows/pytest-core-mpi.yml +++ b/.github/workflows/pytest-core-mpi.yml @@ -15,7 +15,7 @@ on: - master jobs: - build: + test-mpi-basic: name: pytest-mpi runs-on: ubuntu-20.04 strategy: @@ -25,6 +25,7 @@ jobs: env: DEVITO_LANGUAGE: "openmp" DEVITO_ARCH: "gcc-9" + OMP_NUM_THREADS: "1" CC: "gcc-9" CXX: "g++-9" @@ -52,3 +53,22 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} name: pytest-mpi + + test-mpi-docker: + name: pytest-mpi + runs-on: ubuntu-latest + strategy: + matrix: + arch: [gcc, icc, icx] + + steps: + - name: Checkout devito + uses: actions/checkout@v3 + + - name: Build docker image + run: | + docker build . --file docker/Dockerfile.devito --tag devito_img --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} + + - name: Test with pytest + run: | + docker run --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} -e OMP_NUM_THREADS=1 --name testrun devito_img pytest tests/test_mpi.py \ No newline at end of file diff --git a/.github/workflows/pytest-core-nompi.yml b/.github/workflows/pytest-core-nompi.yml index 738daf79ed..11063b6b36 100644 --- a/.github/workflows/pytest-core-nompi.yml +++ b/.github/workflows/pytest-core-nompi.yml @@ -141,7 +141,7 @@ jobs: - name: Set run prefix run: | if [[ "${{ matrix.name }}" =~ "docker" ]]; then - echo "RUN_CMD=docker run --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} -e DEVITO_ARCH=${{ matrix.arch }} --name testrun devito_img" >> $GITHUB_ENV + echo "RUN_CMD=docker run --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} --name testrun devito_img" >> $GITHUB_ENV else echo "RUN_CMD=" >> $GITHUB_ENV fi diff --git a/devito/arch/compiler.py b/devito/arch/compiler.py index 01a0c71525..1e7a91a167 100644 --- a/devito/arch/compiler.py +++ b/devito/arch/compiler.py @@ -10,7 +10,7 @@ import numpy.ctypeslib as npct from codepy.jit import compile_from_string -from codepy.toolchain import GCCToolchain +from codepy.toolchain import GCCToolchain, call_capture_output from devito.arch import (AMDGPUX, Cpu64, M1, NVIDIAX, POWER8, POWER9, GRAVITON, INTELGPUX, IntelSkylake, get_nvidia_cc, check_cuda_runtime, @@ -54,6 +54,8 @@ def sniff_compiler_version(cc): compiler = "clang" elif ver.startswith("icc"): compiler = "icc" + elif ver.startswith("icx"): + compiler = "icx" elif ver.startswith("pgcc"): compiler = "pgcc" elif ver.startswith("cray"): @@ -62,7 +64,7 @@ def sniff_compiler_version(cc): compiler = "unknown" ver = Version("0") - if compiler in ["gcc", "icc"]: + if compiler in ["gcc", "icc", "icx"]: try: # gcc-7 series only spits out patch level on dumpfullversion. res = run([cc, "-dumpfullversion"], stdout=PIPE, stderr=DEVNULL) @@ -707,7 +709,17 @@ def __init__(self, *args, **kwargs): if mpi_distro != 'IntelMPI': warning("Expected Intel MPI distribution with `%s`, but found `%s`" % (self.__class__.__name__, mpi_distro)) - self.cflags.append("-cc=%s" % self.CC) + self.cflags.insert(0, '-cc=%s' % self.CC) + + def get_version(self): + if configuration['mpi']: + cmd = [self.cc, "-cc=%s" % self.CC, "--version"] + else: + cmd = [self.cc, "--version"] + result, stdout, stderr = call_capture_output(cmd) + if result != 0: + raise RuntimeError(f"version query failed: {stderr}") + return stdout def __lookup_cmds__(self): self.CC = 'icc' @@ -720,9 +732,9 @@ def __lookup_cmds__(self): # we try to use `mpiicc` first, while `mpicc` is our fallback, which may # or may not be an Intel distribution try: - check_output(["mpiicc", "--version"]).decode("utf-8") + check_output(["mpiicc", "-cc=%s" % self.CC, "--version"]).decode("utf-8") self.MPICC = 'mpiicc' - self.MPICXX = 'mpiicpc' + self.MPICXX = 'mpicxx' except FileNotFoundError: self.MPICC = 'mpicc' self.MPICXX = 'mpicxx' @@ -777,7 +789,7 @@ def __lookup_cmds__(self): self.CC = 'icx' self.CXX = 'icpx' self.MPICC = 'mpicc' - self.MPICX = 'mpicx' + self.MPICXX = 'mpicxx' class CustomCompiler(Compiler): diff --git a/devito/core/autotuning.py b/devito/core/autotuning.py index 23dac69d61..603a78efd6 100644 --- a/devito/core/autotuning.py +++ b/devito/core/autotuning.py @@ -122,7 +122,8 @@ def autotune(operator, args, level, mode): at_args.update(dict(run)) # Drop run if not at least one block per thread - if not configuration['develop-mode'] and nblocks_per_thread.subs(at_args) < 1: + if not configuration['develop-mode'] and \ + nblocks_per_thread.subs(normalize_args(at_args)) < 1: continue # Run the Operator diff --git a/devito/types/dense.py b/devito/types/dense.py index 5fe38ebc09..5d50e22d9d 100644 --- a/devito/types/dense.py +++ b/devito/types/dense.py @@ -16,7 +16,7 @@ from devito.logger import debug, warning from devito.mpi import MPI from devito.parameters import configuration -from devito.symbolics import FieldFromPointer +from devito.symbolics import FieldFromPointer, normalize_args from devito.finite_differences import Differentiable, generate_fd_shortcuts from devito.tools import (ReducerMap, as_tuple, c_restrict_void_p, flatten, is_integer, memoized_meth, dtype_to_ctype, humanbytes) @@ -1580,8 +1580,9 @@ def make(self, shape=None, initializer=None, allocator=None, **kwargs): raise ValueError("Either `shape` or `kwargs` (Operator overrides) " "must be provided.") shape = [] + args = normalize_args(kwargs) for n, i in enumerate(self.shape): - v = i.subs(kwargs) + v = i.subs(args) if not v.is_Integer: raise ValueError("Couldn't resolve `shape[%d]=%s` with the given " "kwargs (obtained: `%s`)" % (n, i, v)) diff --git a/devito/types/dimension.py b/devito/types/dimension.py index 42f6d26b08..19b0026628 100644 --- a/devito/types/dimension.py +++ b/devito/types/dimension.py @@ -321,7 +321,8 @@ def _arg_check(self, args, size, interval): upper = interval.upper else: # Autopadding causes non-integer upper limit - upper = interval.upper.subs(args) + from devito.symbolics import normalize_args + upper = interval.upper.subs(normalize_args(args)) if args[self.max_name] + upper >= size: raise InvalidArgument("OOB detected due to %s=%d" % (self.max_name, args[self.max_name])) diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd index 4b392a09c1..507bccab75 100644 --- a/docker/Dockerfile.amd +++ b/docker/Dockerfile.amd @@ -3,7 +3,7 @@ # Based on https://github.com/amd/InfinityHub-CI/tree/main/base-gpu-mpi-rocm-docker ############################################################## -ARG ROCM_VERSION=5.4.2 +ARG ROCM_VERSION=5.5.1 ARG arch="aomp" FROM rocm/dev-ubuntu-22.04:${ROCM_VERSION}-complete as sdk-base @@ -82,26 +82,8 @@ RUN cd /tmp/ \ && make -j ${nproc} \ && make install -# Install OpenMPI -RUN cd /tmp \ - && git clone --recursive https://github.com/open-mpi/ompi.git -b ${OMPI_BRANCH} \ - && cd ompi \ - && ./autogen.pl \ - && mkdir build \ - && cd build \ - && ../configure --prefix=$OMPI_HOME --with-ucx=$UCX_HOME \ - --enable-mca-no-build=btl-uct \ - --without-verbs \ - --with-pmix \ - --enable-mpi \ - --enable-mpi-fortran=yes \ - --disable-debug \ - && make -j ${nproc} \ - && make install - # Cleanup -RUN rm -rf /tmp/ompi && rm -rf /tmp/ucx - +RUN rm -rf /tmp/ucx # Adding OpenMPI and UCX to Environment ENV PATH=$OMPI_HOME/bin:$UCX_HOME/bin:$PATH \ @@ -113,7 +95,6 @@ ENV PATH=$OMPI_HOME/bin:$UCX_HOME/bin:$PATH \ INCLUDE=$OMPI_HOME/include:$UCX_HOME/include:$INCLUDE \ PKG_CONFIG_PATH=$OMPI_HOME/lib/pkgconfig:$UCX_HOME/lib/pkgconfig:$PKG_CONFIG_PATH - # Adding environment variable for Running as ROOT ENV OMPI_ALLOW_RUN_AS_ROOT=1 ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 @@ -132,12 +113,37 @@ RUN python3 -m venv /venv && \ /venv/bin/pip install --no-cache-dir mpi4py && \ rm -rf ~/.cache/pip +RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + +EXPOSE 8888 +CMD ["/bin/bash"] + ######################################################################## # AOMP for GPUs (OpenMP offloading) ######################################################################## # This will only trigger if arch is aomp since the final stage depends on it FROM sdk-base as aomp +# Install OpenMPI +RUN cd /tmp \ + && git clone --recursive https://github.com/open-mpi/ompi.git -b ${OMPI_BRANCH} \ + && cd ompi \ + && ./autogen.pl \ + && mkdir build \ + && cd build \ + && ../configure --prefix=$OMPI_HOME --with-ucx=$UCX_HOME \ + CC=amdclang CXX=amdclang++ FC=amdflang F90=amdflang \ + --enable-mca-no-build=btl-uct \ + --without-verbs \ + --with-pmix \ + --enable-mpi \ + --enable-mpi-fortran=yes \ + --disable-debug \ + && make -j ${nproc} \ + && make install +RUN RUN rm -rf /tmp/ompi + # MPI env ENV OMPI_CC="amdclang" ENV OMPI_CXX="amdclang++" @@ -147,13 +153,31 @@ ENV DEVITO_ARCH="aomp" ENV DEVITO_PLATFORM="amdgpuX" ENV DEVITO_LANGUAGE="openmp" - ######################################################################## # HIPCC for GPUs (HIP) ######################################################################## # This will only trigger if arch is hip since the final stage depends on it FROM sdk-base as hip +# Install OpenMPI +RUN cd /tmp \ + && git clone --recursive https://github.com/open-mpi/ompi.git -b ${OMPI_BRANCH} \ + && cd ompi \ + && ./autogen.pl \ + && mkdir build \ + && cd build \ + && ../configure --prefix=$OMPI_HOME --with-ucx=$UCX_HOME \ + CC=hipcc CXX=hipcc FC=hipfort F90=hipfort \ + --enable-mca-no-build=btl-uct \ + --without-verbs \ + --with-pmix \ + --enable-mpi \ + --enable-mpi-fortran=yes \ + --disable-debug \ + && make -j ${nproc} \ + && make install +RUN RUN rm -rf /tmp/ompi + # MPI env ENV OMPI_CC="hipcc" ENV OMPI_CXX="hipcc" @@ -162,15 +186,3 @@ ENV OMPI_CXX="hipcc" ENV DEVITO_ARCH="hip" ENV DEVITO_PLATFORM="amdgpuX" ENV DEVITO_LANGUAGE="hip" - - -######################################################################## -# Final image -######################################################################## -FROM ${arch} as final - -RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ - rm -rf /var/lib/apt/lists/* - -EXPOSE 8888 -CMD ["/bin/bash"] diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 7dfc00423d..02d7e91552 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -1,17 +1,21 @@ +# syntax=docker/dockerfile:1 ############################################################## # This Dockerfile contains the Devito codes and can be built using different base images. ############################################################## -ARG pyversion=python:3.7 +ARG pyversion=python:3.9 ARG arch=gcc # Base image -FROM ${pyversion} as base +FROM ${pyversion}-slim-bullseye as base ENV DEBIAN_FRONTEND noninteractive # Install for basic base not containing it -RUN apt-get update && apt-get install -y vim wget git flex libnuma-dev tmux numactl hwloc +RUN apt-get update && \ + apt-get install -y vim wget git flex libnuma-dev tmux \ + numactl hwloc curl \ + autoconf libtool build-essential # Install tmpi RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi @@ -19,6 +23,12 @@ RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/l # Install OpenGL library, necessary for the installation of GemPy RUN apt-get install -y libgl1-mesa-glx +RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + +EXPOSE 8888 +CMD ["/bin/bash"] + ############################################################## # GCC standard image ############################################################## @@ -30,7 +40,7 @@ RUN mkdir -p /deps && mkdir -p /opt/openmpi && cd /deps && \ mkdir build && cd build && \ ../configure --prefix=/opt/openmpi/ \ --enable-mca-no-build=btl-uct --enable-mpi1-compatibility && \ - make -j $(( $(lscpu | awk '/^Socket\(s\)/{ print $2 }') * $(lscpu | awk '/^Core\(s\) per socket/{ print $4 }') )) && \ + make -j ${nproc} && \ make install && \ rm -rf /deps/openmpi @@ -43,46 +53,50 @@ ENV DEVITO_ARCH="gcc" ENV DEVITO_LANGUAGE="openmp" ############################################################## -# ICC image +# Intel Oneapi base ############################################################## -FROM base as icc +FROM base as oneapi - # Download the key to system keyring - # https://www.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/apt.html#apt +# Download the key to system keyring +# https://www.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/apt.html#apt RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor > /usr/share/keyrings/oneapi-archive-keyring.gpg RUN echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" > /etc/apt/sources.list.d/oneAPI.list -# Install wanted components only (icc and mpiicc) -RUN apt-get update -y && apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi-devel - # Set en vars by hand since we can't use entrypoint for intermediate image ENV I_ICC_ROOT=/opt/intel/oneapi/compiler/latest/ ENV I_MPI_ROOT=/opt/intel/oneapi/mpi/latest/ -ENV I_MPI_CFLAGS="-cc=icc" ENV PATH $I_MPI_ROOT/libfabric/bin:$I_MPI_ROOT/bin:$I_ICC_ROOT/linux/bin/intel64:$I_ICC_ROOT/linux/bin:${PATH} ENV LD_LIBRARY_PATH $I_MPI_ROOT/libfabric/lib:$I_MPI_ROOT/lib/release:$I_MPI_ROOT/lib:$I_ICC_ROOT/linux/lib:$I_ICC_ROOT/linux/lib/x64:$I_ICC_ROOT/linux/compiler/lib/intel64_lin:${LD_LIBRARY_PATH} ENV FI_PROVIDER_PATH $I_MPI_ROOT/libfabric/lib/prov:/usr/lib64/libfabric:${LD_LIBRARY_PATH} +############################################################## +# ICC image +############################################################## +FROM oneapi as icc + +RUN apt-get update -y && apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-mpi-devel && \ + apt-get clean && apt-get autoclean && apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + # Devito config ENV DEVITO_ARCH="icc" ENV DEVITO_LANGUAGE="openmp" # MPICC compiler for mpi4py ENV MPICC=$I_MPI_ROOT/bin/mpiicc - -FROM icc as icx -# Devito config -ENV DEVITO_ARCH="icx" -ENV DEVITO_LANGUAGE="openmp" -# MPICC compiler for mpi4py -ENV I_MPI_CFLAGS="-cc=icx" +ENV MPI4PY_FLAGS='CFLAGS="-cc=icc"' ############################################################## -# Published image +# ICX image ############################################################## -FROM ${arch} as final +FROM oneapi as icx -RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ +RUN apt-get update -y && apt-get install -y intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mpi-devel && \ + apt-get clean && apt-get autoclean && apt-get autoremove && \ rm -rf /var/lib/apt/lists/* -EXPOSE 8888 -CMD ["/bin/bash"] +# Devito config +ENV DEVITO_ARCH="icx" +ENV DEVITO_LANGUAGE="openmp" +# MPICC compiler for mpi4py +ENV MPICC=$I_MPI_ROOT/bin/mpiicc +ENV MPI4PY_FLAGS='CFLAGS="-cc=icx"' diff --git a/docker/Dockerfile.devito b/docker/Dockerfile.devito index 08bea3a0c3..089ec5b2f8 100644 --- a/docker/Dockerfile.devito +++ b/docker/Dockerfile.devito @@ -7,9 +7,14 @@ ARG base=devitocodes/bases:cpu-gcc FROM $base as builder +# User/Group Ids +ARG USER_ID=1000 +ARG GROUP_ID=1000 + ################## Install devito ############################################ # Copy Devito ADD . /app/devito + # Remove git files RUN rm -rf /app/devito/.git @@ -18,6 +23,7 @@ RUN python3 -m venv /venv && \ /venv/bin/pip install --no-cache-dir --upgrade pip && \ /venv/bin/pip install --no-cache-dir jupyter && \ /venv/bin/pip install --no-cache-dir wheel && \ + eval "$MPI4PY_FLAGS /venv/bin/pip install --no-cache-dir mpi4py" && \ /venv/bin/pip install --no-cache-dir -e /app/devito[extras,mpi,tests] && \ rm -rf ~/.cache/pip @@ -29,6 +35,10 @@ FROM $base as user # COPY is much faster than RUN chown by order of magnitude so we have a final step that # just copies the built image into the user. +# User/Group Ids +ARG USER_ID=1000 +ARG GROUP_ID=1000 + ## Create App user # Set the home directory to our app user's home. ENV HOME=/app diff --git a/docker/Dockerfile.nvidia b/docker/Dockerfile.nvidia index a8811d5519..b2bfb62fde 100644 --- a/docker/Dockerfile.nvidia +++ b/docker/Dockerfile.nvidia @@ -31,8 +31,8 @@ RUN if [ "$ver" = "nvhpc" ]; then \ apt-get install -y -q --allow-unauthenticated ${ver}; \ else \ export year=$(echo $ver | cut -d "-" -f 2) && export minor=$(echo $ver | cut -d "-" -f 3) && \ - wget https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64/nvhpc-20${year}_${year}.${minor}_amd64.deb && \ - apt-get install --allow-unauthenticated -y -q ./nvhpc-20${year}_${year}.${minor}_amd64.deb; \ + wget https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64/nvhpc_${year}.${minor}_amd64.deb && \ + apt-get install --allow-unauthenticated -y -q ./nvhpc_${year}.${minor}_amd64.deb; \ fi; RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - && \ @@ -47,7 +47,6 @@ RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - && \ ARG MPI_VER=HPCX ENV MPIVER=${MPI_VER} - # nvidia-container-runtime ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility @@ -74,11 +73,21 @@ ENV UCX_TLS=cuda,cuda_copy,cuda_ipc,sm,shm,self #ENV UCX_TLS=cuda,cuda_copy,cuda_ipc,sm,shm,self,rc_x,gdr_copy # Make simlink for path setup since ENV doesn't accept shell commands. -RUN export NVARCH=$(ls -1 opt/nvidia/hpc_sdk/Linux_x86_64/ | grep '\.' | head -n 1) && \ - export CUDA_V=$(ls opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH}/cuda/ | grep '\.') && \ +RUN export NVARCH=$(ls -1 /opt/nvidia/hpc_sdk/Linux_x86_64/ | grep '\.' | head -n 1) && \ + export CUDA_V=$(ls /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH}/cuda/ | grep '\.') && \ ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH} /opt/nvhpc && \ - ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH}/cuda/${CUDA_V}/extras/CUPTI /opt/CUPTI - + ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH}/cuda/${CUDA_V}/extras/CUPTI /opt/CUPTI && \ + ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/comm_libs/${CUDA_V}/nvshmem /opt/nvhpc/comm_libs/nvshmem && \ + ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/comm_libs/${CUDA_V}/nccl /opt/nvhpc/comm_libs/nccl + +# Starting nvhpc 23.5 and cuda 12.1, hpcx and openmpi are inside the cuda version folder, only the bin is in the comm_libs path +RUN export CUDA_V=$(ls /opt/nvhpc/${NVARCH}/cuda/ | grep '\.') && \ + ls /opt/nvhpc/comm_libs/${CUDA_V}/hpcx/ &&\ + if [ -d /opt/nvhpc/comm_libs/${CUDA_V}/hpcx ]; then \ + rm -rf /opt/nvhpc/comm_libs/hpcx && rm -rf /opt/nvhpc/comm_libs/openmpi4 && \ + ln -sf /opt/nvhpc/comm_libs/${CUDA_V}/hpcx /opt/nvhpc/comm_libs/hpcx && \ + ln -sf /opt/nvhpc/comm_libs/${CUDA_V}/openmpi4 /opt/nvhpc/comm_libs/openmpi4;\ + fi; # Set base path based on version ENV HPCSDK_HOME=/opt/nvhpc ENV HPCSDK_CUPTI=/opt/CUPTI @@ -98,15 +107,14 @@ ENV NVHPC_CUDA_HOME $HPCSDK_HOME/cuda ENV CUDA_ROOT $HPCSDK_HOME/cuda/bin ENV PATH $HPCSDK_HOME/compilers/bin:$HPCSDK_HOME/cuda/bin:$HPCSDK_HOME/comm_libs/mpi/bin:${PATH} ENV LD_LIBRARY_PATH $HPCSDK_HOME/cuda/lib:$HPCSDK_HOME/cuda/lib64:$HPCSDK_HOME/compilers/lib:$HPCSDK_HOME/math_libs/lib64:$HPCSDK_HOME/comm_libs/mpi/lib:$HPCSDK_CUPTI/lib64:bitcomp_DIR:${LD_LIBRARY_PATH} -ENV CPATH $HPCSDK_HOME/comm_libs/mpi/include:${CPATH} +ENV CPATH $HPCSDK_HOME/comm_libs/mpi/include:$HPCSDK_HOME/comm_libs/nvshmem/include:$HPCSDK_HOME/comm_libs/nccl/include:${CPATH} # MPI RUN if [ "x$MPI_VER" = "x4" ]; then \ rm -f $HPCSDK_HOME/comm_libs/mpi && \ - ln -sf $HPCSDK_HOME/comm_libs/openmpi4/openmpi-4.0.5 \ + ln -sf $HPCSDK_HOME/comm_libs/openmpi4/latest \ $HPCSDK_HOME/comm_libs/mpi ; \ - fi; \ - if [ "$MPI_VER" = "HPCX" ]; then \ + elif [ "$MPI_VER" = "HPCX" ]; then \ rm -f $HPCSDK_HOME/comm_libs/mpi && \ ln -sf $HPCSDK_HOME/comm_libs/hpcx/latest/ompi \ $HPCSDK_HOME/comm_libs/mpi ; \ @@ -121,6 +129,12 @@ RUN python3 -m venv /venv && \ /venv/bin/jupyter serverextension enable dask_labextension && \ rm -rf ~/.cache/pip +RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + +EXPOSE 8888 +CMD ["/bin/bash"] + ######################################################################## # NVC for GPUs via OpenACC config ######################################################################## @@ -130,7 +144,7 @@ FROM sdk-base as nvc ADD docker/nvdashboard.json /app/nvdashboard.json # mpi4py -RUN CC=nvc CFLAGS="-noswitcherror -tp=px" /venv/bin/pip install --no-cache-dir mpi4py && rm -rf ~/.cache/pip +ENV MPI4PY_FLAGS='CC=nvc CFLAGS="-noswitcherror -tp=px"' ENV DEVITO_ARCH="nvc" ENV DEVITO_PLATFORM="nvidiaX" @@ -139,13 +153,7 @@ ENV DEVITO_LANGUAGE="openacc" ######################################################################## # NVC for GPUs via CUDA config ######################################################################## -FROM sdk-base as nvcc - -# Make devito env vars file and extras -ADD docker/nvdashboard.json /app/nvdashboard.json - -# mpi4py -RUN CC=nvc CFLAGS="-noswitcherror -tp=px" /venv/bin/pip install --no-cache-dir mpi4py && rm -rf ~/.cache/pip +FROM nvc as nvcc ENV DEVITO_ARCH="cuda" ENV DEVITO_PLATFORM="nvidiaX" @@ -154,10 +162,7 @@ ENV DEVITO_LANGUAGE="cuda" ######################################################################## # NVC for CPUs config ######################################################################## -FROM sdk-base as nvc-host - -# mpi4py -RUN CC=nvc CFLAGS="-noswitcherror -tp=px" /venv/bin/pip install --no-cache-dir mpi4py && rm -rf ~/.cache/pip +FROM nvc as nvc-host ENV DEVITO_ARCH="nvc" ENV DEVITO_PLATFORM="cpu64" @@ -193,8 +198,11 @@ RUN cd /llvm-project/build && \ # Set path ENV PATH /llvm/bin:${PATH} ENV LD_LIBRARY_PATH /llvm/lib:${LD_LIBRARY_PATH} +ENV CPATH /llvm/include:${CPATH} RUN rm -rf llvm-project +RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* # Recompile mpi4py with clang ENV OMPI_CC="clang" @@ -204,14 +212,3 @@ RUN /venv/bin/pip install --no-cache-dir mpi4py && rm -rf ~/.cache/pip ENV DEVITO_ARCH="clang" ENV DEVITO_PLATFORM="nvidiaX" ENV DEVITO_LANGUAGE="openmp" - -######################################################################## -# Final image -######################################################################## -FROM ${arch} as final - -RUN apt-get clean && apt-get autoclean && apt-get autoremove && \ - rm -rf /var/lib/apt/lists/* - -EXPOSE 8888 -CMD ["/bin/bash"]