From 527baa95afa498fbc631b558e64c67fa6ffdbb96 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Tue, 18 Feb 2025 17:34:12 +0100 Subject: [PATCH 1/6] Add Docker workflow changes: - created docker build and push workflow - created ubuntu 24.04 docker image - modified reuseable_fast and basic to work on docker image - modified pr/push to run build image when needed - added few installation to docker files --- .github/docker/ubuntu-20.04.Dockerfile | 25 ++++- .github/docker/ubuntu-22.04.Dockerfile | 21 +++- .github/docker/ubuntu-24.04.Dockerfile | 80 ++++++++++++++ .github/workflows/build_ci_container.yml | 38 +++++++ .github/workflows/pr_push.yml | 42 ++++++- .github/workflows/reusable_basic.yml | 49 +++------ .github/workflows/reusable_fast.yml | 134 +++++++++++++++++------ 7 files changed, 313 insertions(+), 76 deletions(-) create mode 100644 .github/docker/ubuntu-24.04.Dockerfile create mode 100644 .github/workflows/build_ci_container.yml diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index a6a45a8c1..2c6f13041 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -1,10 +1,10 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based -# environment for building the Unified Memory Framework project. +# environment for building the Unified Memory Framework project. # # Pull base image ("20.04") @@ -50,12 +50,29 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all +# Install hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh +RUN apt-get update \ + && apt-get install -y dos2unix libtool \ + && dos2unix /opt/umf/install_hwloc.sh \ + && bash -x /opt/umf/install_hwloc.sh \ + && ldconfig \ + && rm -f /opt/umf/install_hwloc.sh + +# Install valgrind +RUN apt-get update && \ + apt-get install -y valgrind cmake hwloc libhwloc-dev libnuma-dev libtbb-dev + +# Install lcov +RUN apt-get update && \ + apt-get install lcov -y + # Prepare a dir (accessible by anyone) -RUN mkdir --mode 777 /opt/umf/ +RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) COPY third_party/requirements.txt /opt/umf/requirements.txt -RUN pip3 install --no-cache-dir -r /opt/umf/requirements.txt +# RUN pip3 install --no-cache-dir -r /opt/umf/requirements.txt # Add a new (non-root) 'test_user' ENV USER test_user diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 75c71c526..f6b166d14 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -1,4 +1,4 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -49,8 +49,25 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all +# Install hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh +RUN apt-get update \ + && apt-get install -y dos2unix libtool \ + && dos2unix /opt/umf/install_hwloc.sh \ + && bash -x /opt/umf/install_hwloc.sh \ + && ldconfig \ + && rm -f /opt/umf/install_hwloc.sh + +# Install valgrind +RUN apt-get update && \ + apt-get install -y valgrind cmake hwloc libhwloc-dev libnuma-dev libtbb-dev + +# Install lcov +RUN apt-get update && \ + apt-get install lcov -y + # Prepare a dir (accessible by anyone) -RUN mkdir --mode 777 /opt/umf/ +RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) COPY third_party/requirements.txt /opt/umf/requirements.txt diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile new file mode 100644 index 000000000..83860578b --- /dev/null +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -0,0 +1,80 @@ +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based +# environment for building the Unified Memory Framework project. +# + +# Pull base image ("24.04") +FROM registry.hub.docker.com/library/ubuntu@sha256:72297848456d5d37d1262630108ab308d3e9ec7ed1c3286a32fe09856619a782 + +# Set environment variables +ENV OS ubuntu +ENV OS_VER 24.04 +ENV NOTTY 1 +ENV DEBIAN_FRONTEND noninteractive + +# Base development packages +ARG BASE_DEPS="\ + build-essential \ + cmake \ + git" + +# UMF's dependencies +ARG UMF_DEPS="\ + libhwloc-dev \ + libtbb-dev" + +# Dependencies for tests (optional) +ARG TEST_DEPS="\ + libnuma-dev" + +# Miscellaneous for our builds/CI (optional) +ARG MISC_DEPS="\ + automake \ + clang \ + python3-pip \ + sudo \ + whois" + +# Update and install required packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ${BASE_DEPS} \ + ${UMF_DEPS} \ + ${TEST_DEPS} \ + ${MISC_DEPS} \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean all + +# Install hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh +RUN apt-get update \ + && apt-get install -y dos2unix libtool \ + && dos2unix /opt/umf/install_hwloc.sh \ + && bash -x /opt/umf/install_hwloc.sh \ + && ldconfig \ + && rm -f /opt/umf/install_hwloc.sh + +# Install valgrind +RUN apt-get update && \ + apt-get install -y valgrind clang cmake hwloc libhwloc-dev libnuma-dev libtbb-dev + +# Install lcov +RUN apt-get update && \ + apt-get install lcov -y + +# Prepare a dir (accessible by anyone) +RUN mkdir -p --mode 777 /opt/umf/ + +# Additional dependencies (installed via pip) +COPY third_party/requirements.txt /opt/umf/requirements.txt +RUN pip3 install --no-cache-dir --break-system-packages -r /opt/umf/requirements.txt + +# Add a new (non-root) 'test_user' +ENV USER test_user +ENV USERPASS pass +RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" +USER test_user diff --git a/.github/workflows/build_ci_container.yml b/.github/workflows/build_ci_container.yml new file mode 100644 index 000000000..380f0f5ba --- /dev/null +++ b/.github/workflows/build_ci_container.yml @@ -0,0 +1,38 @@ +name: BuildCIContainer + +on: + workflow_call: + workflow_dispatch: + +permissions: + packages: write + contents: read + +jobs: + build-ci-container: + runs-on: ubuntu-latest + strategy: + matrix: + ubuntu-version: [20.04, 22.04, 24.04] + outputs: + status: ${{ job.status }} + env: + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ env.GHCR_TOKEN }} + + - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image + run: | + docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ghcr.io/${{ github.actor }}/umf2-ubuntu-${{ matrix.ubuntu-version }}:latest . + docker push ghcr.io/${{ github.actor }}/umf2-ubuntu-${{ matrix.ubuntu-version }}:latest diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index cfc4a04b9..36e4b04f9 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -14,50 +14,84 @@ concurrency: permissions: contents: read + packages: write jobs: CodeChecks: uses: ./.github/workflows/reusable_checks.yml DocsBuild: uses: ./.github/workflows/reusable_docs_build.yml + PrintModifiedFiles: + runs-on: ubuntu-latest + outputs: + changed_files: ${{ steps.changed-files.outputs.all_changed_files }} + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45.0.3 + + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + run: | + echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" + BuildCIContainer: + if: ${{ contains(join(needs.PrintModifiedFiles.outputs.changed_files, ' '), '.github/docker/') }} + needs: [PrintModifiedFiles] + secrets: inherit + uses: ./.github/workflows/build_ci_container.yml FastBuild: - name: Fast builds - needs: [CodeChecks, DocsBuild] + if: always() && (needs.BuildCIContainer.result == 'skipped' || needs.BuildCIContainer.result == 'success') + needs: [CodeChecks, DocsBuild, BuildCIContainer] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_basic.yml DevDax: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_dax.yml MultiNuma: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_multi_numa.yml L0: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: name: "LEVEL_ZERO" shared_lib: "['ON']" CUDA: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: name: "CUDA" shared_lib: "['ON']" Sanitizers: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_sanitizers.yml QEMU: + if: always() && (needs.FastBuild.result == 'success') needs: [FastBuild] uses: ./.github/workflows/reusable_qemu.yml with: short_run: true ProxyLib: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml Valgrind: + if: always() && (needs.Build.result == 'success') needs: [Build] uses: ./.github/workflows/reusable_valgrind.yml Coverage: @@ -70,16 +104,18 @@ jobs: trigger: "${{github.event_name}}" Coverage_partial: # partial coverage (on forks) - if: github.repository != 'oneapi-src/unified-memory-framework' + if: github.repository != 'oneapi-src/unified-memory-framework' && always() && (needs.Build.result == 'success') needs: [Build, QEMU, ProxyLib] uses: ./.github/workflows/reusable_coverage.yml CodeQL: + if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read security-events: write uses: ./.github/workflows/reusable_codeql.yml Trivy: + if: always() && (needs.Build.result == 'success') needs: [Build] permissions: contents: read diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 7170ec418..a41497d99 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -5,6 +5,7 @@ on: workflow_call permissions: contents: read + packages: read env: BUILD_DIR : "${{github.workspace}}/build" @@ -112,23 +113,17 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'ON' runs-on: ${{matrix.os}} - + container: + image: ${{ matrix.os == 'ubuntu-20.04' && 'ghcr.io/rbanka1/umf2-ubuntu-20.04:latest' || matrix.os == 'ubuntu-22.04' && 'ghcr.io/rbanka1/umf2-ubuntu-22.04:latest' || matrix.os == 'ubuntu-24.04' && 'ghcr.io/rbanka1/umf2-ubuntu-24.04:latest'}} + options: --user root --privileged + volumes: + - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: fetch-depth: 0 - - name: Install apt packages - run: | - sudo apt-get update - sudo apt-get install -y clang cmake libnuma-dev lcov - - - name: Install TBB apt package - if: matrix.install_tbb == 'ON' - run: | - sudo apt-get install -y libtbb-dev - - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' run: | @@ -137,18 +132,6 @@ jobs: echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp - - - name: Install g++-7 - if: matrix.compiler.cxx == 'g++-7' - run: sudo apt-get install -y ${{matrix.compiler.cxx}} - - - name: Install libhwloc - run: .github/scripts/install_hwloc.sh - - - name: Get UMF version - run: | - VERSION=$(git describe --tags --abbrev=0 | grep -oP '\d+\.\d+\.\d+') - echo "UMF_VERSION=$VERSION" >> $GITHUB_ENV - name: Configure build run: > @@ -200,16 +183,16 @@ jobs: - name: Remove the installation directory run: rm -rf ${{env.INSTL_DIR}} - - name: Test UMF installation and uninstallation - # The '--shared-library' parameter is added to the installation test when the UMF is built as a shared library - run: > - python3 ${{github.workspace}}/test/test_installation.py - --build-dir ${{env.BUILD_DIR}} - --install-dir ${{env.INSTL_DIR}} - --build-type ${{matrix.build_type}} - ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }} - --umf-version ${{env.UMF_VERSION}} - ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} + # - name: Test UMF installation and uninstallation + # # The '--shared-library' parameter is added to the installation test when the UMF is built as a shared library + # run: > + # python3 ${{github.workspace}}/test/test_installation.py + # --build-dir ${{env.BUILD_DIR}} + # --install-dir ${{env.INSTL_DIR}} + # --build-type ${{matrix.build_type}} + # ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }} + # --umf-version ${{env.UMF_VERSION}} + # ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} windows-build: name: Windows diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 5166f2b96..32899492a 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -1,54 +1,50 @@ -# Fast builds name: FastBuild on: workflow_call permissions: contents: read + packages: read env: BUILD_DIR : "${{github.workspace}}/build" INSTL_DIR : "${{github.workspace}}/../install-dir" jobs: - FastBuild: - name: Fast builds + FastBuild_Linux: + name: Fast builds (Linux) + runs-on: ${{ matrix.os }} + container: + image: ${{ matrix.os == 'ubuntu-latest' && 'ghcr.io/rbanka1/umf2-ubuntu-22.04:latest' || matrix.os == 'ubuntu-20.04' && 'ghcr.io/rbanka1/umf2-ubuntu-20.04:latest' }} + options: --user root --privileged + volumes: + - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework + env: VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" strategy: matrix: include: - - os: windows-latest - build_tests: 'ON' - simple_cmake: 'OFF' - # pure C build (Windows) - - os: windows-latest - # Tests' building is off for a pure C build - build_tests: 'OFF' - simple_cmake: 'OFF' - os: ubuntu-latest + disjoint: 'ON' build_tests: 'ON' - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' simple_cmake: 'OFF' - # pure C build (Linux) - os: ubuntu-latest - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - # Tests' building is off for a pure C build + disjoint: 'OFF' build_tests: 'OFF' extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON' simple_cmake: 'OFF' - # simplest CMake on ubuntu-latest - os: ubuntu-latest + disjoint: 'OFF' build_tests: 'ON' extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' - # simplest CMake ubuntu-20.04 - os: ubuntu-20.04 + disjoint: 'OFF' build_tests: 'ON' extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' - runs-on: ${{ (matrix.os == 'ubuntu-latest' && github.repository_owner == 'oneapi-src') && 'intel-ubuntu-22.04' || matrix.os }} steps: - name: Checkout repository @@ -56,21 +52,7 @@ jobs: with: fetch-depth: 0 - - name: Initialize vcpkg - if: matrix.os == 'windows-latest' - uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 - with: - vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg - vcpkgJsonGlob: '**/vcpkg.json' - - - name: Install dependencies (windows-latest) - if: matrix.os == 'windows-latest' - run: vcpkg install - shell: pwsh # Specifies PowerShell as the shell for running the script. - - name: Install dependencies (ubuntu-latest) - if: matrix.os == 'ubuntu-latest' run: | sudo apt-get update sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev @@ -82,6 +64,90 @@ jobs: sudo apt-get install -y cmake libnuma-dev libtbb-dev .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package + - name: Set ptrace value for IPC test (on Linux only) + run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" + + - name: Configure CMake + if: matrix.simple_cmake == 'OFF' + run: > + cmake + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" + -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}} + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_TESTS=${{matrix.build_tests}} + -DUMF_BUILD_EXAMPLES=ON + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON + -DUMF_BUILD_CUDA_PROVIDER=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + -DUMF_BUILD_SHARED_LIBRARY=ON + ${{matrix.extra_build_options}} + + - name: Configure CMake (simple) + if: matrix.simple_cmake == 'ON' + run: > + cmake + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" + -DUMF_BUILD_SHARED_LIBRARY=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + ${{matrix.extra_build_options}} + + - name: Build + run: cmake --build ${{env.BUILD_DIR}} --config Release -j + + - name: Create build directory + run: mkdir -p ${{env.BUILD_DIR}} + + - name: Run examples + working-directory: ${{env.BUILD_DIR}} + run: ctest --output-on-failure --test-dir examples -C Release + + - name: Run tests + if: matrix.build_tests == 'ON' + working-directory: ${{env.BUILD_DIR}} + run: ctest --output-on-failure --test-dir test -C Release + + FastBuild_Windows: + name: Fast builds (Windows) + runs-on: windows-latest + + env: + VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" + strategy: + matrix: + include: + - os: windows-latest + disjoint: 'OFF' + build_tests: 'ON' + simple_cmake: 'OFF' + - os: windows-latest + disjoint: 'OFF' + build_tests: 'OFF' + simple_cmake: 'OFF' + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Initialize vcpkg + if: matrix.os == 'windows-latest' + uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 + with: + vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 + vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg + vcpkgJsonGlob: '**/vcpkg.json' + + - name: Install dependencies (windows-latest) + if: matrix.os == 'windows-latest' + run: vcpkg install + shell: pwsh + - name: Configure CMake if: matrix.simple_cmake == 'OFF' run: > @@ -91,6 +157,7 @@ jobs: -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}} -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_TESTS=${{matrix.build_tests}} -DUMF_BUILD_EXAMPLES=ON @@ -122,10 +189,9 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir test -C Release - # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps) - # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details - name: Print metadata of our dll's if: matrix.os == 'windows-latest' run: | get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list + \ No newline at end of file From 25d95010571a6b5aec7ea24b1b2322c5762c2edf Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Thu, 20 Feb 2025 10:04:13 +0100 Subject: [PATCH 2/6] corrections --- .github/docker/ubuntu-20.04.Dockerfile | 41 +++++++++--------- .github/docker/ubuntu-22.04.Dockerfile | 40 +++++++++--------- .github/docker/ubuntu-24.04.Dockerfile | 42 +++++++++--------- .github/workflows/build_ci_container.yml | 14 +++--- .github/workflows/pr_push.yml | 17 +++++--- .github/workflows/reusable_basic.yml | 54 +++++++++++++++--------- .github/workflows/reusable_fast.yml | 49 ++++++--------------- 7 files changed, 128 insertions(+), 129 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 2c6f13041..7f9932898 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -29,7 +29,10 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libhwloc-dev \ + libtbb-dev\ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ @@ -38,7 +41,16 @@ ARG MISC_DEPS="\ g++-7 \ python3-pip \ sudo \ - whois" + whois \ + lcov" + +# Hwloc installation dependencies +ARG HWLOC_DEPS="\ + dos2unix \ + libtool" + +# Copy hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh # Update and install required packages RUN apt-get update \ @@ -47,34 +59,21 @@ RUN apt-get update \ ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${HWLOC_DEPS} \ + && dos2unix /opt/umf/install_hwloc.sh \ + && bash -x /opt/umf/install_hwloc.sh \ + && ldconfig \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all -# Install hwloc -COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh -RUN apt-get update \ - && apt-get install -y dos2unix libtool \ - && dos2unix /opt/umf/install_hwloc.sh \ - && bash -x /opt/umf/install_hwloc.sh \ - && ldconfig \ - && rm -f /opt/umf/install_hwloc.sh - -# Install valgrind -RUN apt-get update && \ - apt-get install -y valgrind cmake hwloc libhwloc-dev libnuma-dev libtbb-dev - -# Install lcov -RUN apt-get update && \ - apt-get install lcov -y - # Prepare a dir (accessible by anyone) RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) COPY third_party/requirements.txt /opt/umf/requirements.txt -# RUN pip3 install --no-cache-dir -r /opt/umf/requirements.txt +# It's actively used and tested only on selected distros. Be aware +# they may not work, because pip packages list differ from OS to OS. -# Add a new (non-root) 'test_user' ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index f6b166d14..914485b6b 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -12,7 +12,7 @@ FROM registry.hub.docker.com/library/ubuntu@sha256:e6173d4dc55e76b87c4af8db8821b # Set environment variables ENV OS ubuntu -ENV OS_VER 22.04 +ENV OS_VER 20.04 ENV NOTTY 1 ENV DEBIAN_FRONTEND noninteractive @@ -29,15 +29,28 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libhwloc-dev \ + libtbb-dev\ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ automake \ clang \ + g++-11 \ python3-pip \ sudo \ - whois" + whois \ + lcov" + +# Hwloc installation dependencies +ARG HWLOC_DEPS="\ + dos2unix \ + libtool" + +# Copy hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh # Update and install required packages RUN apt-get update \ @@ -46,26 +59,13 @@ RUN apt-get update \ ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${HWLOC_DEPS} \ + && dos2unix /opt/umf/install_hwloc.sh \ + && bash -x /opt/umf/install_hwloc.sh \ + && ldconfig \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all -# Install hwloc -COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh -RUN apt-get update \ - && apt-get install -y dos2unix libtool \ - && dos2unix /opt/umf/install_hwloc.sh \ - && bash -x /opt/umf/install_hwloc.sh \ - && ldconfig \ - && rm -f /opt/umf/install_hwloc.sh - -# Install valgrind -RUN apt-get update && \ - apt-get install -y valgrind cmake hwloc libhwloc-dev libnuma-dev libtbb-dev - -# Install lcov -RUN apt-get update && \ - apt-get install lcov -y - # Prepare a dir (accessible by anyone) RUN mkdir -p --mode 777 /opt/umf/ diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 83860578b..87f09e40d 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -12,7 +12,7 @@ FROM registry.hub.docker.com/library/ubuntu@sha256:72297848456d5d37d1262630108ab # Set environment variables ENV OS ubuntu -ENV OS_VER 24.04 +ENV OS_VER 20.04 ENV NOTTY 1 ENV DEBIAN_FRONTEND noninteractive @@ -29,15 +29,28 @@ ARG UMF_DEPS="\ # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libhwloc-dev \ + libtbb-dev\ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ automake \ clang \ + g++-11 \ python3-pip \ sudo \ - whois" + whois \ + lcov" + +# Hwloc installation dependencies +ARG HWLOC_DEPS="\ + dos2unix \ + libtool" + +# Copy hwloc +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh # Update and install required packages RUN apt-get update \ @@ -46,26 +59,13 @@ RUN apt-get update \ ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${HWLOC_DEPS} \ + && dos2unix /opt/umf/install_hwloc.sh \ + && bash -x /opt/umf/install_hwloc.sh \ + && ldconfig \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all -# Install hwloc -COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh -RUN apt-get update \ - && apt-get install -y dos2unix libtool \ - && dos2unix /opt/umf/install_hwloc.sh \ - && bash -x /opt/umf/install_hwloc.sh \ - && ldconfig \ - && rm -f /opt/umf/install_hwloc.sh - -# Install valgrind -RUN apt-get update && \ - apt-get install -y valgrind clang cmake hwloc libhwloc-dev libnuma-dev libtbb-dev - -# Install lcov -RUN apt-get update && \ - apt-get install lcov -y - # Prepare a dir (accessible by anyone) RUN mkdir -p --mode 777 /opt/umf/ @@ -73,7 +73,7 @@ RUN mkdir -p --mode 777 /opt/umf/ COPY third_party/requirements.txt /opt/umf/requirements.txt RUN pip3 install --no-cache-dir --break-system-packages -r /opt/umf/requirements.txt -# Add a new (non-root) 'test_user' +# Add a new (non-root) 'test_user' ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" diff --git a/.github/workflows/build_ci_container.yml b/.github/workflows/build_ci_container.yml index 380f0f5ba..0f96bb055 100644 --- a/.github/workflows/build_ci_container.yml +++ b/.github/workflows/build_ci_container.yml @@ -1,4 +1,5 @@ -name: BuildCIContainer +# Build and push Docker images to GHCR +name: BuildDockers on: workflow_call: @@ -17,7 +18,8 @@ jobs: outputs: status: ${{ job.status }} env: - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + IMG: ghcr.io/${{ github.actor }}/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: - name: Checkout repository @@ -26,13 +28,13 @@ jobs: fetch-depth: 0 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.actor }} # ${{ sys_bb-umf }} password: ${{ env.GHCR_TOKEN }} - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image run: | - docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ghcr.io/${{ github.actor }}/umf2-ubuntu-${{ matrix.ubuntu-version }}:latest . - docker push ghcr.io/${{ github.actor }}/umf2-ubuntu-${{ matrix.ubuntu-version }}:latest + docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ${{ env.IMG }} . + docker push ${{ env.IMG }} diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 36e4b04f9..a2d5f8610 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -14,14 +14,14 @@ concurrency: permissions: contents: read - packages: write + packages: read jobs: CodeChecks: uses: ./.github/workflows/reusable_checks.yml DocsBuild: uses: ./.github/workflows/reusable_docs_build.yml - PrintModifiedFiles: + DetectChanges: runs-on: ubuntu-latest outputs: changed_files: ${{ steps.changed-files.outputs.all_changed_files }} @@ -40,14 +40,17 @@ jobs: ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" - BuildCIContainer: - if: ${{ contains(join(needs.PrintModifiedFiles.outputs.changed_files, ' '), '.github/docker/') }} - needs: [PrintModifiedFiles] + BuildDockers: + if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} + needs: [DetectChanges] + permissions: + contents: read + packages: write secrets: inherit uses: ./.github/workflows/build_ci_container.yml FastBuild: - if: always() && (needs.BuildCIContainer.result == 'skipped' || needs.BuildCIContainer.result == 'success') - needs: [CodeChecks, DocsBuild, BuildCIContainer] + if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') + needs: [CodeChecks, DocsBuild, BuildDockers] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index a41497d99..259fd7272 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -16,9 +16,15 @@ env: jobs: ubuntu-build: name: Ubuntu + runs-on: ubuntu-latest + container: + image: ${{ matrix.image }} + options: --user root --privileged + volumes: + - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework strategy: matrix: - os: ['ubuntu-20.04', 'ubuntu-22.04'] + image: ['ghcr.io/rbanka1/umf-ubuntu-20.04:latest', 'ghcr.io/rbanka1/umf-ubuntu-22.04:latest' ] build_type: [Debug, Release] compiler: [{c: gcc, cxx: g++}] shared_library: ['OFF'] @@ -28,7 +34,7 @@ jobs: disable_hwloc: ['OFF'] link_hwloc_statically: ['OFF'] include: - - os: 'ubuntu-20.04' + - image: ghcr.io/rbanka1/umf-ubuntu-20.04:latest build_type: Release compiler: {c: gcc-7, cxx: g++-7} shared_library: 'OFF' @@ -37,7 +43,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Release compiler: {c: clang, cxx: clang++} shared_library: 'OFF' @@ -46,7 +52,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -55,7 +61,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-24.04' + - image: ghcr.io/rbanka1/umf-ubuntu-24.04:latest build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -65,7 +71,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test level_zero_provider='OFF' and cuda_provider='OFF' - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'OFF' @@ -75,7 +81,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test icx compiler - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Release compiler: {c: icx, cxx: icpx} shared_library: 'ON' @@ -85,7 +91,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test without installing TBB - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -94,7 +100,7 @@ jobs: install_tbb: 'OFF' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -103,7 +109,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'ON' link_hwloc_statically: 'OFF' - - os: 'ubuntu-22.04' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -112,12 +118,6 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'ON' - runs-on: ${{matrix.os}} - container: - image: ${{ matrix.os == 'ubuntu-20.04' && 'ghcr.io/rbanka1/umf2-ubuntu-20.04:latest' || matrix.os == 'ubuntu-22.04' && 'ghcr.io/rbanka1/umf2-ubuntu-22.04:latest' || matrix.os == 'ubuntu-24.04' && 'ghcr.io/rbanka1/umf2-ubuntu-24.04:latest'}} - options: --user root --privileged - volumes: - - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -127,12 +127,25 @@ jobs: - name: Install oneAPI basekit if: matrix.compiler.cxx == 'icpx' run: | + sudo apt-get update sudo apt-get install -y gpg-agent wget wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp + - name: Install TBB apt package + if: matrix.install_tbb == 'OFF' + run: | + sudo apt-get install -y libtbb-dev + + - name: Install g++-7 + if: matrix.compiler.cxx == 'g++-7' + run: sudo apt-get install -y ${{matrix.compiler.cxx}} + + - name: Install libhwloc + run: .github/scripts/install_hwloc.sh + - name: Configure build run: > ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} @@ -164,11 +177,14 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} LD_LIBRARY_PATH="${{env.BUILD_DIR}}/lib/:${LD_LIBRARY_PATH}" ctest --output-on-failure - - name: Check coverage + - name: Set os name + run: echo "ARTIFACT_NAME=${{ matrix.image }}" | sed -e 's|ghcr.io/rbanka1/||' -e 's|:|-|' >> $GITHUB_ENV + + - name: Check os coverage if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} working-directory: ${{env.BUILD_DIR}} run: | - export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{env.ARTIFACT_NAME}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME" ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME mkdir -p ${{env.COVERAGE_DIR}} @@ -177,7 +193,7 @@ jobs: - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} with: - name: ${{env.COVERAGE_NAME}}-${{matrix.os}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + name: ${{env.COVERAGE_NAME}}-${{env.ARTIFACT_NAME}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} path: ${{env.COVERAGE_DIR}} - name: Remove the installation directory diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 32899492a..a0d4cb568 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -1,3 +1,4 @@ +# Fast build linux part is working in dockers, Windows is not name: FastBuild on: workflow_call @@ -11,11 +12,11 @@ env: INSTL_DIR : "${{github.workspace}}/../install-dir" jobs: - FastBuild_Linux: - name: Fast builds (Linux) - runs-on: ${{ matrix.os }} + ubuntu-build: + name: Linux + runs-on: ubuntu-latest container: - image: ${{ matrix.os == 'ubuntu-latest' && 'ghcr.io/rbanka1/umf2-ubuntu-22.04:latest' || matrix.os == 'ubuntu-20.04' && 'ghcr.io/rbanka1/umf2-ubuntu-20.04:latest' }} + image: ${{ matrix.image }} options: --user root --privileged volumes: - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework @@ -25,23 +26,19 @@ jobs: strategy: matrix: include: - - os: ubuntu-latest - disjoint: 'ON' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_tests: 'ON' extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' simple_cmake: 'OFF' - - os: ubuntu-latest - disjoint: 'OFF' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_tests: 'OFF' extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON' simple_cmake: 'OFF' - - os: ubuntu-latest - disjoint: 'OFF' + - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_tests: 'ON' extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' - - os: ubuntu-20.04 - disjoint: 'OFF' + - image: ghcr.io/rbanka1/umf-ubuntu-20.04:latest build_tests: 'ON' extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' @@ -52,21 +49,6 @@ jobs: with: fetch-depth: 0 - - name: Install dependencies (ubuntu-latest) - run: | - sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - - - name: Install dependencies (ubuntu-20.04) - if: matrix.os == 'ubuntu-20.04' - run: | - sudo apt-get update - sudo apt-get install -y cmake libnuma-dev libtbb-dev - .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package - - - name: Set ptrace value for IPC test (on Linux only) - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - name: Configure CMake if: matrix.simple_cmake == 'OFF' run: > @@ -99,9 +81,6 @@ jobs: - name: Build run: cmake --build ${{env.BUILD_DIR}} --config Release -j - - name: Create build directory - run: mkdir -p ${{env.BUILD_DIR}} - - name: Run examples working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir examples -C Release @@ -111,8 +90,8 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir test -C Release - FastBuild_Windows: - name: Fast builds (Windows) + windows-build: + name: Windows runs-on: windows-latest env: @@ -136,7 +115,6 @@ jobs: fetch-depth: 0 - name: Initialize vcpkg - if: matrix.os == 'windows-latest' uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 @@ -144,7 +122,6 @@ jobs: vcpkgJsonGlob: '**/vcpkg.json' - name: Install dependencies (windows-latest) - if: matrix.os == 'windows-latest' run: vcpkg install shell: pwsh @@ -189,9 +166,11 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir test -C Release + # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps) + # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details + - name: Print metadata of our dll's if: matrix.os == 'windows-latest' run: | get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list - \ No newline at end of file From 3c7360d9470240ad7f47861c74ac8a4687e982a7 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Fri, 28 Feb 2025 08:25:36 +0100 Subject: [PATCH 3/6] corrections v2 --- .github/docker/ubuntu-20.04.Dockerfile | 7 ++-- .github/docker/ubuntu-22.04.Dockerfile | 8 ++-- .github/docker/ubuntu-24.04.Dockerfile | 8 ++-- ...ild_ci_container.yml => build_dockers.yml} | 8 ++-- .github/workflows/pr_push.yml | 35 +++++++++++------ .github/workflows/reusable_basic.yml | 13 +------ .github/workflows/reusable_fast.yml | 38 +++++++------------ 7 files changed, 51 insertions(+), 66 deletions(-) rename .github/workflows/{build_ci_container.yml => build_dockers.yml} (81%) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 7f9932898..f6982b7f6 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -56,12 +56,10 @@ COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ ${HWLOC_DEPS} \ - && dos2unix /opt/umf/install_hwloc.sh \ - && bash -x /opt/umf/install_hwloc.sh \ + && bash /opt/umf/install_hwloc.sh \ && ldconfig \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all @@ -70,10 +68,11 @@ RUN apt-get update \ RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) -COPY third_party/requirements.txt /opt/umf/requirements.txt # It's actively used and tested only on selected distros. Be aware # they may not work, because pip packages list differ from OS to OS. +COPY third_party/requirements.txt /opt/umf/requirements.txt +# Add a new (non-root) 'test_user' ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 914485b6b..a19b8db1f 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -12,7 +12,7 @@ FROM registry.hub.docker.com/library/ubuntu@sha256:e6173d4dc55e76b87c4af8db8821b # Set environment variables ENV OS ubuntu -ENV OS_VER 20.04 +ENV OS_VER 22.04 ENV NOTTY 1 ENV DEBIAN_FRONTEND noninteractive @@ -38,7 +38,7 @@ ARG TEST_DEPS="\ ARG MISC_DEPS="\ automake \ clang \ - g++-11 \ + g++ \ python3-pip \ sudo \ whois \ @@ -56,12 +56,10 @@ COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ ${HWLOC_DEPS} \ - && dos2unix /opt/umf/install_hwloc.sh \ - && bash -x /opt/umf/install_hwloc.sh \ + && bash /opt/umf/install_hwloc.sh \ && ldconfig \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 87f09e40d..b60d42d74 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -12,7 +12,7 @@ FROM registry.hub.docker.com/library/ubuntu@sha256:72297848456d5d37d1262630108ab # Set environment variables ENV OS ubuntu -ENV OS_VER 20.04 +ENV OS_VER 24.04 ENV NOTTY 1 ENV DEBIAN_FRONTEND noninteractive @@ -38,7 +38,7 @@ ARG TEST_DEPS="\ ARG MISC_DEPS="\ automake \ clang \ - g++-11 \ + g++ \ python3-pip \ sudo \ whois \ @@ -56,12 +56,10 @@ COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ ${HWLOC_DEPS} \ - && dos2unix /opt/umf/install_hwloc.sh \ - && bash -x /opt/umf/install_hwloc.sh \ + && bash /opt/umf/install_hwloc.sh \ && ldconfig \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all diff --git a/.github/workflows/build_ci_container.yml b/.github/workflows/build_dockers.yml similarity index 81% rename from .github/workflows/build_ci_container.yml rename to .github/workflows/build_dockers.yml index 0f96bb055..bad1b3963 100644 --- a/.github/workflows/build_ci_container.yml +++ b/.github/workflows/build_dockers.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - build-ci-container: + build-dockers: runs-on: ubuntu-latest strategy: matrix: @@ -18,7 +18,7 @@ jobs: outputs: status: ${{ job.status }} env: - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + GHCR_TOKEN: ${{ secrets.BB_GHCR_TOKEN }} # ${{ secrets.GHCR_TOKEN }} IMG: ghcr.io/${{ github.actor }}/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: @@ -31,8 +31,8 @@ jobs: uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: ${{ github.actor }} # ${{ sys_bb-umf }} - password: ${{ env.GHCR_TOKEN }} + username: ${{ github.bb-ur }} # ${{ github.actor }} + password: ${{ env.BB_GHCR_TOKEN }} # ${{ env.GHCR_TOKEN }} - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image run: | diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index a2d5f8610..0ecab998b 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -23,23 +23,34 @@ jobs: uses: ./.github/workflows/reusable_docs_build.yml DetectChanges: runs-on: ubuntu-latest - outputs: - changed_files: ${{ steps.changed-files.outputs.all_changed_files }} + outputs: + changed_files: ${{ steps.get-changes.outputs.all_changed_files }} steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45.0.3 + fetch-depth: 2 - name: List all changed files - env: - ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + id: get-changes + run: | + git diff --name-only HEAD^ HEAD > all_changed_files.txt + changed_files=$(cat all_changed_files.txt | tr '\n' ' ') + echo "changed_files=$changed_files" >> $GITHUB_ENV + echo "::set-output name=all_changed_files::$changed_files" + + - name: all changed files run: | - echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" + echo "changed files: ${{ needs.DetectChanges.outputs.changed_files }}" + Check: + runs-on: ubuntu-latest + needs: [DetectChanges] + steps: + - name: check changed files + run: | + echo "all_changed_files: ${{ needs.DetectChanges.outputs.changed_files }}" + echo "all_changed_files: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }}" + BuildDockers: if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} needs: [DetectChanges] @@ -47,10 +58,10 @@ jobs: contents: read packages: write secrets: inherit - uses: ./.github/workflows/build_ci_container.yml + uses: ./.github/workflows/build_dockers.yml FastBuild: if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') - needs: [CodeChecks, DocsBuild, BuildDockers] + needs: [ CodeChecks, DocsBuild, BuildDockers] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index 259fd7272..ab18eb13a 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -21,7 +21,7 @@ jobs: image: ${{ matrix.image }} options: --user root --privileged volumes: - - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework + - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: image: ['ghcr.io/rbanka1/umf-ubuntu-20.04:latest', 'ghcr.io/rbanka1/umf-ubuntu-22.04:latest' ] @@ -133,15 +133,6 @@ jobs: echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp - - - name: Install TBB apt package - if: matrix.install_tbb == 'OFF' - run: | - sudo apt-get install -y libtbb-dev - - - name: Install g++-7 - if: matrix.compiler.cxx == 'g++-7' - run: sudo apt-get install -y ${{matrix.compiler.cxx}} - name: Install libhwloc run: .github/scripts/install_hwloc.sh @@ -505,4 +496,4 @@ jobs: --build-type ${{env.BUILD_TYPE}} --proxy --umf-version ${{env.UMF_VERSION}} - --shared-library + --shared-library \ No newline at end of file diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index a0d4cb568..8ba51f020 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -14,33 +14,30 @@ env: jobs: ubuntu-build: name: Linux - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} container: image: ${{ matrix.image }} options: --user root --privileged volumes: - - /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework - - env: - VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" + - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: include: - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' + extra_build_options: ' -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' simple_cmake: 'OFF' + # pure C build - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_tests: 'OFF' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON' + extra_build_options: '-DUMF_BUILD_BENCHMARKS=ON' simple_cmake: 'OFF' + # simplest CMake on ubuntu-latest - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' - image: ghcr.io/rbanka1/umf-ubuntu-20.04:latest build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release' simple_cmake: 'ON' steps: @@ -54,11 +51,9 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} - -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" - -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" + -DCMAKE_BUILD_TYPE=Release -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}} -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_TESTS=${{matrix.build_tests}} -DUMF_BUILD_EXAMPLES=ON @@ -73,13 +68,12 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} - -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON ${{matrix.extra_build_options}} - name: Build - run: cmake --build ${{env.BUILD_DIR}} --config Release -j + run: cmake --build ${{env.BUILD_DIR}} --config Release -j $(nproc) - name: Run examples working-directory: ${{env.BUILD_DIR}} @@ -99,14 +93,12 @@ jobs: strategy: matrix: include: - - os: windows-latest - disjoint: 'OFF' - build_tests: 'ON' + - build_tests: 'ON' simple_cmake: 'OFF' - - os: windows-latest - disjoint: 'OFF' - build_tests: 'OFF' + - build_tests: 'OFF' simple_cmake: 'OFF' + # - build_tests: 'OFF' + # simple_cmake: 'ON' steps: - name: Checkout repository @@ -121,7 +113,7 @@ jobs: vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - - name: Install dependencies (windows-latest) + - name: Install dependencies run: vcpkg install shell: pwsh @@ -134,7 +126,6 @@ jobs: -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}} -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_TESTS=${{matrix.build_tests}} -DUMF_BUILD_EXAMPLES=ON @@ -142,7 +133,6 @@ jobs: -DUMF_BUILD_CUDA_PROVIDER=ON -DUMF_TESTS_FAIL_ON_SKIP=ON -DUMF_BUILD_SHARED_LIBRARY=ON - ${{matrix.extra_build_options}} - name: Configure CMake (simple) if: matrix.simple_cmake == 'ON' @@ -152,7 +142,6 @@ jobs: -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON - ${{matrix.extra_build_options}} - name: Build run: cmake --build ${{env.BUILD_DIR}} --config Release -j @@ -170,7 +159,6 @@ jobs: # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details - name: Print metadata of our dll's - if: matrix.os == 'windows-latest' run: | get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list From c8549420cfacf6bbb581bf66fed5706c02a025bd Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Fri, 28 Feb 2025 08:29:58 +0100 Subject: [PATCH 4/6] add GH user --- .github/workflows/build_dockers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dockers.yml b/.github/workflows/build_dockers.yml index bad1b3963..67881c34f 100644 --- a/.github/workflows/build_dockers.yml +++ b/.github/workflows/build_dockers.yml @@ -19,7 +19,7 @@ jobs: status: ${{ job.status }} env: GHCR_TOKEN: ${{ secrets.BB_GHCR_TOKEN }} # ${{ secrets.GHCR_TOKEN }} - IMG: ghcr.io/${{ github.actor }}/umf-ubuntu-${{ matrix.ubuntu-version }}:latest + IMG: ghcr.io/${{ github.bb-ur }}/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: - name: Checkout repository From c26684edfae7820c1117ee724ae125e54400c3f8 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Fri, 28 Feb 2025 16:44:36 +0100 Subject: [PATCH 5/6] corrections V3 --- .github/docker/ubuntu-20.04.Dockerfile | 5 --- .github/docker/ubuntu-22.04.Dockerfile | 5 --- .github/docker/ubuntu-24.04.Dockerfile | 7 +--- .github/workflows/pr_push.yml | 33 +++++++------------ ...dockers.yml => reusable_dockers_build.yml} | 9 +++-- .github/workflows/reusable_fast.yml | 17 +++++----- 6 files changed, 25 insertions(+), 51 deletions(-) rename .github/workflows/{build_dockers.yml => reusable_dockers_build.yml} (72%) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index f6982b7f6..738477da6 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -22,11 +22,6 @@ ARG BASE_DEPS="\ cmake \ git" -# UMF's dependencies -ARG UMF_DEPS="\ - libhwloc-dev \ - libtbb-dev" - # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index a19b8db1f..33d3fb8e6 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -22,11 +22,6 @@ ARG BASE_DEPS="\ cmake \ git" -# UMF's dependencies -ARG UMF_DEPS="\ - libhwloc-dev \ - libtbb-dev" - # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index b60d42d74..796d25f7c 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -22,11 +22,6 @@ ARG BASE_DEPS="\ cmake \ git" -# UMF's dependencies -ARG UMF_DEPS="\ - libhwloc-dev \ - libtbb-dev" - # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ @@ -75,4 +70,4 @@ RUN pip3 install --no-cache-dir --break-system-packages -r /opt/umf/requirements ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" -USER test_user +USER test_user \ No newline at end of file diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 0ecab998b..87aa6a056 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -23,34 +23,23 @@ jobs: uses: ./.github/workflows/reusable_docs_build.yml DetectChanges: runs-on: ubuntu-latest - outputs: - changed_files: ${{ steps.get-changes.outputs.all_changed_files }} + outputs: + changed_files: ${{ steps.changed-files.outputs.all_changed_files }} steps: - name: Checkout code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - fetch-depth: 2 + fetch-depth: 0 - - name: List all changed files - id: get-changes - run: | - git diff --name-only HEAD^ HEAD > all_changed_files.txt - changed_files=$(cat all_changed_files.txt | tr '\n' ' ') - echo "changed_files=$changed_files" >> $GITHUB_ENV - echo "::set-output name=all_changed_files::$changed_files" + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45.0.3 - - name: all changed files - run: | - echo "changed files: ${{ needs.DetectChanges.outputs.changed_files }}" - Check: - runs-on: ubuntu-latest - needs: [DetectChanges] - steps: - - name: check changed files + - name: List all changed files + env: + ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} run: | - echo "all_changed_files: ${{ needs.DetectChanges.outputs.changed_files }}" - echo "all_changed_files: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }}" - + echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" BuildDockers: if: ${{ contains(join(needs.DetectChanges.outputs.changed_files, ' '), '.github/docker/') }} needs: [DetectChanges] @@ -58,7 +47,7 @@ jobs: contents: read packages: write secrets: inherit - uses: ./.github/workflows/build_dockers.yml + uses: ./.github/workflows/reusable_dockers_build.yml FastBuild: if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') needs: [ CodeChecks, DocsBuild, BuildDockers] diff --git a/.github/workflows/build_dockers.yml b/.github/workflows/reusable_dockers_build.yml similarity index 72% rename from .github/workflows/build_dockers.yml rename to .github/workflows/reusable_dockers_build.yml index 67881c34f..6497cee80 100644 --- a/.github/workflows/build_dockers.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -18,8 +18,7 @@ jobs: outputs: status: ${{ job.status }} env: - GHCR_TOKEN: ${{ secrets.BB_GHCR_TOKEN }} # ${{ secrets.GHCR_TOKEN }} - IMG: ghcr.io/${{ github.bb-ur }}/umf-ubuntu-${{ matrix.ubuntu-version }}:latest + IMG: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu-version }}:latest steps: - name: Checkout repository @@ -31,10 +30,10 @@ jobs: uses: docker/login-action@30f019fb76bb54d03ec1e716054622be511a13b2 # v3.2.0 with: registry: ghcr.io - username: ${{ github.bb-ur }} # ${{ github.actor }} - password: ${{ env.BB_GHCR_TOKEN }} # ${{ env.GHCR_TOKEN }} + username: bb-ur + password: ${{ secrets.BB_GHCR_TOKEN }} - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image run: | docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ${{ env.IMG }} . - docker push ${{ env.IMG }} + docker push ${{ env.IMG }} \ No newline at end of file diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 8ba51f020..837eee522 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -16,27 +16,27 @@ jobs: name: Linux runs-on: ${{ github.repository_owner == 'oneapi-src' && 'intel-ubuntu-22.04' || 'ubuntu-22.04' }} container: - image: ${{ matrix.image }} + image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: include: - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_tests: 'ON' extra_build_options: ' -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' simple_cmake: 'OFF' # pure C build - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_tests: 'OFF' extra_build_options: '-DUMF_BUILD_BENCHMARKS=ON' simple_cmake: 'OFF' # simplest CMake on ubuntu-latest - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 24.04 build_tests: 'ON' simple_cmake: 'ON' - - image: ghcr.io/rbanka1/umf-ubuntu-20.04:latest + - ubuntu_ver: 20.04 build_tests: 'ON' simple_cmake: 'ON' @@ -97,8 +97,8 @@ jobs: simple_cmake: 'OFF' - build_tests: 'OFF' simple_cmake: 'OFF' - # - build_tests: 'OFF' - # simple_cmake: 'ON' + - build_tests: 'OFF' + simple_cmake: 'ON' steps: - name: Checkout repository @@ -140,6 +140,7 @@ jobs: cmake -B ${{env.BUILD_DIR}} -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" + -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON @@ -161,4 +162,4 @@ jobs: - name: Print metadata of our dll's run: | get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list - get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list + get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list \ No newline at end of file From eece86bcab4fda3f7555bd3e5d63f83811843661 Mon Sep 17 00:00:00 2001 From: rbanka1 Date: Mon, 3 Mar 2025 10:45:19 +0100 Subject: [PATCH 6/6] correctionsV4 --- .github/docker/ubuntu-20.04.Dockerfile | 7 +++-- .github/docker/ubuntu-22.04.Dockerfile | 18 ++++-------- .github/docker/ubuntu-24.04.Dockerfile | 20 ++++--------- .github/workflows/pr_push.yml | 2 +- .github/workflows/reusable_basic.yml | 31 +++++++++----------- .github/workflows/reusable_dockers_build.yml | 2 +- .github/workflows/reusable_fast.yml | 6 ++-- 7 files changed, 34 insertions(+), 52 deletions(-) diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 738477da6..5958824a3 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -22,10 +22,12 @@ ARG BASE_DEPS="\ cmake \ git" +# UMF's dependencies +# libhwloc-dev is required + # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libhwloc-dev \ libtbb-dev\ valgrind" @@ -54,8 +56,7 @@ RUN apt-get update \ ${TEST_DEPS} \ ${MISC_DEPS} \ ${HWLOC_DEPS} \ - && bash /opt/umf/install_hwloc.sh \ - && ldconfig \ + && /opt/umf/install_hwloc.sh \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 33d3fb8e6..8d63d2347 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -22,10 +22,13 @@ ARG BASE_DEPS="\ cmake \ git" +# UMF's dependencies +ARG UMF_DEPS="\ + libhwloc-dev" + # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libhwloc-dev \ libtbb-dev\ valgrind" @@ -33,29 +36,18 @@ ARG TEST_DEPS="\ ARG MISC_DEPS="\ automake \ clang \ - g++ \ python3-pip \ sudo \ whois \ lcov" -# Hwloc installation dependencies -ARG HWLOC_DEPS="\ - dos2unix \ - libtool" - -# Copy hwloc -COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh - # Update and install required packages RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ - ${HWLOC_DEPS} \ - && bash /opt/umf/install_hwloc.sh \ - && ldconfig \ + ${UMF_DEPS} \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile index 796d25f7c..cd45cc52c 100644 --- a/.github/docker/ubuntu-24.04.Dockerfile +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -22,10 +22,13 @@ ARG BASE_DEPS="\ cmake \ git" +# UMF's dependencies +ARG UMF_DEPS="\ + libhwloc-dev" + # Dependencies for tests (optional) ARG TEST_DEPS="\ libnuma-dev \ - libhwloc-dev \ libtbb-dev\ valgrind" @@ -33,29 +36,18 @@ ARG TEST_DEPS="\ ARG MISC_DEPS="\ automake \ clang \ - g++ \ python3-pip \ sudo \ whois \ lcov" -# Hwloc installation dependencies -ARG HWLOC_DEPS="\ - dos2unix \ - libtool" - -# Copy hwloc -COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh - # Update and install required packages RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ - ${HWLOC_DEPS} \ - && bash /opt/umf/install_hwloc.sh \ - && ldconfig \ + ${UMF_DEPS} \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all @@ -70,4 +62,4 @@ RUN pip3 install --no-cache-dir --break-system-packages -r /opt/umf/requirements ENV USER test_user ENV USERPASS pass RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" -USER test_user \ No newline at end of file +USER test_user diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 87aa6a056..9633c5402 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -50,7 +50,7 @@ jobs: uses: ./.github/workflows/reusable_dockers_build.yml FastBuild: if: always() && (needs.BuildDockers.result == 'skipped' || needs.BuildDockers.result == 'success') - needs: [ CodeChecks, DocsBuild, BuildDockers] + needs: [CodeChecks, DocsBuild, BuildDockers] uses: ./.github/workflows/reusable_fast.yml Build: name: Basic builds diff --git a/.github/workflows/reusable_basic.yml b/.github/workflows/reusable_basic.yml index ab18eb13a..9be558818 100644 --- a/.github/workflows/reusable_basic.yml +++ b/.github/workflows/reusable_basic.yml @@ -18,13 +18,13 @@ jobs: name: Ubuntu runs-on: ubuntu-latest container: - image: ${{ matrix.image }} + image: ghcr.io/bb-ur/umf-ubuntu-${{ matrix.ubuntu_ver }}:latest options: --user root --privileged volumes: - ${{ github.workspace }}:${{ github.workspace }} strategy: matrix: - image: ['ghcr.io/rbanka1/umf-ubuntu-20.04:latest', 'ghcr.io/rbanka1/umf-ubuntu-22.04:latest' ] + ubuntu_ver: ['20.04', '22.04'] build_type: [Debug, Release] compiler: [{c: gcc, cxx: g++}] shared_library: ['OFF'] @@ -34,7 +34,7 @@ jobs: disable_hwloc: ['OFF'] link_hwloc_statically: ['OFF'] include: - - image: ghcr.io/rbanka1/umf-ubuntu-20.04:latest + - ubuntu_ver: 20.04 build_type: Release compiler: {c: gcc-7, cxx: g++-7} shared_library: 'OFF' @@ -43,7 +43,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Release compiler: {c: clang, cxx: clang++} shared_library: 'OFF' @@ -52,7 +52,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -61,7 +61,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - image: ghcr.io/rbanka1/umf-ubuntu-24.04:latest + - ubuntu_ver: 24.04 build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -71,7 +71,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test level_zero_provider='OFF' and cuda_provider='OFF' - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'OFF' @@ -81,7 +81,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test icx compiler - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Release compiler: {c: icx, cxx: icpx} shared_library: 'ON' @@ -91,7 +91,7 @@ jobs: disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' # test without installing TBB - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -100,7 +100,7 @@ jobs: install_tbb: 'OFF' disable_hwloc: 'OFF' link_hwloc_statically: 'OFF' - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -109,7 +109,7 @@ jobs: install_tbb: 'ON' disable_hwloc: 'ON' link_hwloc_statically: 'OFF' - - image: ghcr.io/rbanka1/umf-ubuntu-22.04:latest + - ubuntu_ver: 22.04 build_type: Release compiler: {c: gcc, cxx: g++} shared_library: 'ON' @@ -168,14 +168,11 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh' || true }} LD_LIBRARY_PATH="${{env.BUILD_DIR}}/lib/:${LD_LIBRARY_PATH}" ctest --output-on-failure - - name: Set os name - run: echo "ARTIFACT_NAME=${{ matrix.image }}" | sed -e 's|ghcr.io/rbanka1/||' -e 's|:|-|' >> $GITHUB_ENV - - name: Check os coverage if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} working-directory: ${{env.BUILD_DIR}} run: | - export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{env.ARTIFACT_NAME}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + export COVERAGE_FILE_NAME=${{env.COVERAGE_NAME}}-${{matrix.ubuntu_ver}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} echo "COVERAGE_FILE_NAME: $COVERAGE_FILE_NAME" ../scripts/coverage/coverage_capture.sh $COVERAGE_FILE_NAME mkdir -p ${{env.COVERAGE_DIR}} @@ -184,7 +181,7 @@ jobs: - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 if: ${{ matrix.build_type == 'Debug' && matrix.compiler.c == 'gcc' }} with: - name: ${{env.COVERAGE_NAME}}-${{env.ARTIFACT_NAME}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} + name: ${{env.COVERAGE_NAME}}-${{matrix.ubuntu_ver}}-shared-${{matrix.shared_library}}-no_hwloc-${{matrix.disable_hwloc}} path: ${{env.COVERAGE_DIR}} - name: Remove the installation directory @@ -496,4 +493,4 @@ jobs: --build-type ${{env.BUILD_TYPE}} --proxy --umf-version ${{env.UMF_VERSION}} - --shared-library \ No newline at end of file + --shared-library diff --git a/.github/workflows/reusable_dockers_build.yml b/.github/workflows/reusable_dockers_build.yml index 6497cee80..fe4bb15a4 100644 --- a/.github/workflows/reusable_dockers_build.yml +++ b/.github/workflows/reusable_dockers_build.yml @@ -36,4 +36,4 @@ jobs: - name: Build and push ubuntu-${{ matrix.ubuntu-version }} Docker image run: | docker build -f .github/docker/ubuntu-${{ matrix.ubuntu-version }}.Dockerfile -t ${{ env.IMG }} . - docker push ${{ env.IMG }} \ No newline at end of file + docker push ${{ env.IMG }} diff --git a/.github/workflows/reusable_fast.yml b/.github/workflows/reusable_fast.yml index 837eee522..e186064e0 100644 --- a/.github/workflows/reusable_fast.yml +++ b/.github/workflows/reusable_fast.yml @@ -68,6 +68,7 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} + -DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON ${{matrix.extra_build_options}} @@ -145,7 +146,7 @@ jobs: -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build - run: cmake --build ${{env.BUILD_DIR}} --config Release -j + run: cmake --build ${{env.BUILD_DIR}} --config Release -j $Env:NUMBER_OF_PROCESSORS - name: Run examples working-directory: ${{env.BUILD_DIR}} @@ -158,8 +159,7 @@ jobs: # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps) # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details - - name: Print metadata of our dll's run: | get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list - get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list \ No newline at end of file + get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list