CI-linux.yml

on:
  workflow_call:
    inputs:
      os:
        required: true
        type: string
      distro:
        required: true
        type: string
      compiler-family:
        required: true
        type: string
      compiler-version:
        required: true
        type: string
      compiler-CC:
        required: true
        type: string
      compiler-CXX:
        required: true
        type: string
      compiler-GCOV:
        type: string
      compiler-CLANG_TIDY:
        type: string
      flavor:
        required: true
        type: string
      enable-sample-based-testing:
        type: boolean
        default: false
      rpuu-cache-key:
        type: string
      ECO:
        type: string
    secrets:
      SONAR_TOKEN:
      SONAR_PROJECT_KEY:
      SONAR_ORGANIZATION:

env:
  RPUU_DST: ${{ github.workspace }}/raw-camera-samples/raw.pixls.us-unique
  COVERAGE_REPORT_PREFIX: rawspeed-coverage # relative to the workspace!

jobs:
  linux:
    runs-on: ubuntu-latest
    env:
      SONAR_SCANNER_VERSION: 5.0.1.3006
    name: ${{ inputs.distro }}.${{ inputs.compiler-family }}${{ inputs.compiler-version }}.${{ inputs.flavor }}
    container:
      image: ${{ inputs.distro }}
      options: --tmpfs /tmp:exec --tmpfs /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}:exec
    steps:
      - name: Configure APT
        timeout-minutes: 1
        run: |
          set -xe
          tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io > /dev/null <<EOT
          force-unsafe-io
          EOT
          tee /etc/apt/apt.conf.d/tmpfs > /dev/null <<EOT
          Dir::Cache::Archives "/tmp/apt/archives";
          APT::ExtractTemplates::TempDir "/tmp/apt/temp";
          EOT
          mkdir -p /tmp/apt/archives
          tee /etc/apt/apt.conf.d/80retry > /dev/null <<EOT
          Acquire::Retries "10";
          EOT
          tee /etc/apt/apt.conf.d/80recommends > /dev/null <<EOT
          APT::Install-Recommends "false";
          EOT
          tee /etc/apt/apt.conf.d/80suggests > /dev/null <<EOT
          APT::Install-Suggests "false";
          EOT
          tee /etc/apt/apt.conf.d/80forceyes > /dev/null <<EOT
          APT::Get::Assume-Yes "true";
          EOT
          tee /etc/apt/apt.conf.d/80fixmissing > /dev/null <<EOT
          APT::Get::Fix-Missing "true";
          EOT
          rm -rf /etc/apt/sources.list*
          if [ "${{ inputs.distro }}" = "debian:unstable-slim" ]; then
          tee /etc/apt/sources.list > /dev/null <<EOT
          deb http://debian-archive.trafficmanager.net/debian unstable main
          EOT
          elif [ "${{ inputs.distro }}" = "debian:trixie-slim" ]; then
          tee /etc/apt/sources.list > /dev/null <<EOT
          deb http://debian-archive.trafficmanager.net/debian trixie main
          deb http://debian-archive.trafficmanager.net/debian trixie-updates main
          deb http://debian-archive.trafficmanager.net/debian-security trixie-security main
          deb http://debian-archive.trafficmanager.net/debian trixie-backports main
          EOT
          elif [ "${{ inputs.distro }}" = "debian:bookworm-slim" ]; then
          tee /etc/apt/sources.list > /dev/null <<EOT
          deb http://debian-archive.trafficmanager.net/debian bookworm main
          deb http://debian-archive.trafficmanager.net/debian bookworm-updates main
          deb http://debian-archive.trafficmanager.net/debian-security bookworm-security main
          deb http://debian-archive.trafficmanager.net/debian bookworm-backports main
          EOT
          fi
      - name: Update base packages
        timeout-minutes: 1
        env:
          DEBIAN_FRONTEND: noninteractive
          COMPILER_FAMILY: ${{ inputs.compiler-family }}
          FLAVOR: ${{ inputs.flavor }}
        run: |
          set -xe
          rm -rf /var/lib/apt/lists/*
          apt update
          apt install eatmydata
          eatmydata apt upgrade
      - name: Install necessary packages
        timeout-minutes: 1
        env:
          DEBIAN_FRONTEND: noninteractive
          COMPILER_FAMILY: ${{ inputs.compiler-family }}
          FLAVOR: ${{ inputs.flavor }}
          ENABLE_SAMPLE_BASED_TESTING: ${{ inputs.enable-sample-based-testing }}
        run: |
          set -xe
          eatmydata apt install ca-certificates \
                                    cmake \
                                    git \
                                    googletest \
                                    libjpeg-dev \
                                    libpugixml-dev \
                                    libxml2-utils \
                                    ninja-build \
                                    zlib1g-dev
          if [ "$COMPILER_FAMILY" = "GNU" ]; then
            eatmydata apt install g++-${{ inputs.compiler-version }} \
                                      gcc \
                                      gcc-${{ inputs.compiler-version }}
          fi
          if [ "$COMPILER_FAMILY" = "LLVM" ]; then
            eatmydata apt install clang-${{ inputs.compiler-version }} \
                                      libomp-${{ inputs.compiler-version }}-dev
          fi
          if [ "$ENABLE_SAMPLE_BASED_TESTING" = "true" ]; then
            eatmydata apt install zstd
          fi
          if [ "$FLAVOR" = "ClangTidy" ]; then
            eatmydata apt install clang-format-${{ inputs.compiler-version }} clang-tidy-${{ inputs.compiler-version }}

            eatmydata apt install curl gpg
            echo 'deb https://ftp.gwdg.de//pub/opensuse/repositories/graphics%3A/darktable%3A/master/Debian_Unstable/ /' | tee /etc/apt/sources.list
            curl -fsSL https://download.opensuse.org/repositories/graphics:darktable:master/Debian_Unstable/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/graphics_darktable_master.gpg > /dev/null
            apt update
            eatmydata apt install rawspeed-clang-tidy-module
          fi
          if [ "$FLAVOR" = "ClangStaticAnalysis" ] || [ "$FLAVOR" = "ClangCTUStaticAnalysis" ]; then
            eatmydata apt install clang-tools-${{ inputs.compiler-version }} \
                                      curl \
                                      libxml2-dev \
                                      libxslt1-dev \
                                      make \
                                      python3-dev \
                                      python3-lxml \
                                      python3-pip-whl \
                                      python3-portalocker \
                                      python3-psutil \
                                      python3-setuptools \
                                      python3-venv \
                                      python3-virtualenv \
                                      python3-wheel \
                                      python3-yaml \
                                      xz-utils
          fi
          eatmydata apt clean
          dpkg-divert --add --rename --divert /usr/bin/ld.original /usr/bin/ld
          if [ "$FLAVOR" = "ClangStaticAnalysis" ] || [ "$FLAVOR" = "ClangCTUStaticAnalysis" ]; then
            ln -s /usr/bin/${{ inputs.compiler-CC  }} /usr/local/bin/clang
            ln -s /usr/bin/${{ inputs.compiler-CXX }} /usr/local/bin/clang++
            ln -s /usr/bin/${{ inputs.compiler-CC  }} /usr/local/bin/gcc
            ln -s /usr/bin/${{ inputs.compiler-CXX }} /usr/local/bin/g++
            ln -s /usr/bin/${{ inputs.compiler-CC  }} /usr/local/bin/x86_64-linux-gnu-gcc
            ln -s /usr/bin/${{ inputs.compiler-CXX }} /usr/local/bin/x86_64-linux-gnu-g++
          fi
          ln -s /usr/bin/ld.gold /usr/bin/ld
      - name: Configure git
        timeout-minutes: 1
        run: |
          set -xe
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
      - name: Fetch/Checkout RawSpeed git repo
        timeout-minutes: 1
        uses: actions/checkout@v4
        with:
          path: 'rawspeed'
          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
      - name: Cache raw.pixls.us masterset
        timeout-minutes: 1
        if: inputs.enable-sample-based-testing
        uses: actions/cache/restore@v4
        with:
          path: ${{ env.RPUU_DST }}
          key: ${{ inputs.rpuu-cache-key }}
          fail-on-cache-miss: true
      - name: Fetch/Checkout CodeChecker git repo (for clang static analysis)
        timeout-minutes: 1
        if: inputs.flavor == 'ClangStaticAnalysis' || inputs.flavor == 'ClangCTUStaticAnalysis'
        uses: actions/checkout@v4
        with:
          repository: 'Ericsson/codechecker'
          path: 'codechecker'
          fetch-depth: '1'
      - name: Install CodeChecker (for clang static analysis)
        timeout-minutes: 1
        if: inputs.flavor == 'ClangStaticAnalysis' || inputs.flavor == 'ClangCTUStaticAnalysis'
        env:
          CC: ${{ inputs.compiler-CC }}
          CXX: ${{ inputs.compiler-CXX }}
        run: |
          set -xe
          cd codechecker
          make venv
          . $PWD/venv/bin/activate
          pip install setuptools
          BUILD_LOGGER_64_BIT_ONLY=YES BUILD_UI_DIST=NO make package
          export PATH="$PWD/build/CodeChecker/bin:$PATH"
      - name: Initialize CodeQL (for CodeQL static analysis)
        timeout-minutes: 1
        if: inputs.flavor == 'CodeQLAnalysis'
        uses: github/codeql-action/init@v3
        with:
          languages: cpp
      - name: Set up JDK 11 (for SonarCloud static analysis)
        timeout-minutes: 1
        if: inputs.flavor == 'SonarCloudStaticAnalysis' && github.repository == 'darktable-org/rawspeed' && github.event_name != 'pull_request' && github.ref_type == 'branch' && (github.ref_name == 'develop' || github.ref_name == 'stable')
        uses: actions/setup-java@v4
        with:
          distribution: zulu
          java-version: 21
      - name: Download and set up sonar-scanner (for SonarCloud static analysis)
        timeout-minutes: 1
        if: inputs.flavor == 'SonarCloudStaticAnalysis' && github.repository == 'darktable-org/rawspeed' && github.event_name != 'pull_request' && github.ref_type == 'branch' && (github.ref_name == 'develop' || github.ref_name == 'stable')
        env:
          SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
        run: |
          set -xe
          rm -rf /var/lib/apt/lists/*
          eatmydata apt update
          eatmydata apt install curl unzip
          eatmydata apt clean
          mkdir -p $HOME/.sonar
          curl -sSLo $HOME/.sonar/sonar-scanner.zip ${{ env.SONAR_SCANNER_DOWNLOAD_URL }}
          unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/
          echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> $GITHUB_PATH
      - name: Verify that the source code is properly formatted
        timeout-minutes: 1
        if: inputs.flavor == 'ClangTidy'
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
        run: |
          set -xe
          cd "$SRC_DIR"
          git ls-tree -r --name-only HEAD | grep -vf .clang-format-ignore | xargs clang-format-${{ inputs.compiler-version }} --verbose --dry-run -Werror
      - name: Configure
        timeout-minutes: 1
        env:
          CC: ${{ inputs.compiler-CC }}
          CXX: ${{ inputs.compiler-CXX }}
          CLANG_TIDY: ${{ inputs.compiler-CLANG_TIDY }}
          GCOV: ${{ inputs.compiler-GCOV }}
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          RPUU_DST: ${{ env.RPUU_DST }}
          ECO: ${{ inputs.ECO }} -DALLOW_DOWNLOADING_GOOGLETEST=ON -DALLOW_DOWNLOADING_GOOGLEBENCHMARK=ON -DUSE_CLANG_TIDY=${{ inputs.flavor == 'ClangTidy' }} -DRAWSPEED_ENABLE_SAMPLE_BASED_TESTING=${{ inputs.enable-sample-based-testing }}
          FLAVOR: ${{ inputs.flavor }}
          TARGET: configure
        run: |
          set -xe
          cmake -E make_directory "${BUILD_DIR}"
          cmake -E make_directory "${INSTALL_PREFIX}"
          export ECO="${ECO} -DRAWSPEED_REFERENCE_SAMPLE_ARCHIVE=${RPUU_DST}"
          "${SRC_DIR}/.ci/ci-script.sh"
      - name: Build
        timeout-minutes: ${{ inputs.flavor != 'ClangTidy' && (inputs.flavor != 'CodeQLAnalysis' && 7 || 12) || 25 }}
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: build
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
      - name: Test (unit tests)
        timeout-minutes: 1
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: test
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
      - name: Collect source coverage data (unit tests)
        timeout-minutes: 1
        if: inputs.flavor == 'Coverage'
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}/linux
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: coverage
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
          mkdir -p "${COVERAGE_REPORT_PREFIX}"
          mv "${BUILD_DIR}/gcov-reports-unittest" "${COVERAGE_REPORT_PREFIX}/unittests"
      - name: Test (benchmarks)
        timeout-minutes: 1
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: test_benchmarks
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
      - name: Collect source coverage data (benchmarks)
        timeout-minutes: 1
        if: inputs.flavor == 'Coverage'
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}/linux
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: coverage_benchmarks
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
          mkdir -p "${COVERAGE_REPORT_PREFIX}"
          mv "${BUILD_DIR}/gcov-reports-benchmarks" "${COVERAGE_REPORT_PREFIX}/benchmarks"
      - name: Test (integration)
        timeout-minutes: ${{ inputs.flavor != 'Coverage' && 2 || 31 }}
        if: inputs.enable-sample-based-testing
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: test_integration
          OMP_NUM_THREADS: 1
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
      - name: Collect source coverage data (integration)
        timeout-minutes: 1
        if: inputs.flavor == 'Coverage' && inputs.enable-sample-based-testing
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}/linux
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: coverage_integration
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
          mkdir -p "${COVERAGE_REPORT_PREFIX}"
          mv "${BUILD_DIR}/gcov-reports-rsa" "${COVERAGE_REPORT_PREFIX}/integration"
      - if: inputs.flavor == 'Coverage'
        timeout-minutes: 1
        name: Cache code coverage (linux reports)
        uses: actions/cache/save@v4
        with:
          path: ${{ env.COVERAGE_REPORT_PREFIX }}/linux # relative path!
          key: coverage-${{ github.sha }}-linux
          enableCrossOsArchive: true
      - name: Install
        timeout-minutes: 1
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          INSTALL_PREFIX: ${{ github.workspace }}/rawspeed-install
          FLAVOR: ${{ inputs.flavor }}
          TARGET: install
        run: |
          set -xe
          "${SRC_DIR}/.ci/ci-script.sh"
      - name: Perform clang static analysis (plain mode)
        timeout-minutes: 25
        if: inputs.flavor == 'ClangStaticAnalysis'
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          CC: ${{ inputs.compiler-CC }}
          CXX: ${{ inputs.compiler-CXX }}
        run: |
          set -xe
          . "$PWD/codechecker/venv/bin/activate"
          export PATH="$PWD/codechecker/build/CodeChecker/bin:$PATH"
          set +e
          CodeChecker check --logfile "${BUILD_DIR}/compile_commands.json" --analyzers clangsa --jobs "$(nproc --all)" --file "${SRC_DIR}/*" --disable deadcode.DeadStores -o "${BUILD_DIR}/codechecker_results"
          bugs=$?
          CodeChecker parse -e html "${BUILD_DIR}/codechecker_results" -o "${BUILD_DIR}/codechecker_report"
          set -xe
          tar -cvJ -f "$GITHUB_WORKSPACE/codechecker_report.tar.xz" -C "${BUILD_DIR}" "codechecker_report"
          [ $bugs -eq 0 ] || exit 1
      - name: Upload results of clang static analysis (plain mode)
        timeout-minutes: 1
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: codechecker_report.tar.xz
          path: ${{ github.workspace }}/codechecker_report.tar.xz
      - name: Perform clang static analysis (CTU mode)
        timeout-minutes: 25
        if: inputs.flavor == 'ClangCTUStaticAnalysis'
        env:
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          CC: ${{ inputs.compiler-CC }}
          CXX: ${{ inputs.compiler-CXX }}
        run: |
          set -xe
          . "$PWD/codechecker/venv/bin/activate"
          export PATH="$PWD/codechecker/build/CodeChecker/bin:$PATH"
          set +e
          CodeChecker check --ctu --ctu-reanalyze-on-failure --logfile "${BUILD_DIR}/compile_commands.json" --analyzers clangsa --jobs "$(nproc --all)" --file "${SRC_DIR}/*" --disable deadcode.DeadStores -o "${BUILD_DIR}/codechecker_ctu_results"
          bugs=$?
          CodeChecker parse -e html "${BUILD_DIR}/codechecker_ctu_results" -o "${BUILD_DIR}/codechecker_ctu_report"
          set -xe
          tar -cvJ -f "$GITHUB_WORKSPACE/codechecker_ctu_report.tar.xz" -C "${BUILD_DIR}" "codechecker_ctu_report"
          [ $bugs -eq 0 ] || exit 1
      - name: Upload results of clang static analysis (CTU mode)
        timeout-minutes: 1
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: codechecker_ctu_report.tar.xz
          path: ${{ github.workspace }}/codechecker_ctu_report.tar.xz
      - name: Perform CodeQL static analysis
        timeout-minutes: 11
        if: inputs.flavor == 'CodeQLAnalysis'
        uses: github/codeql-action/analyze@v3
      - name: Perform SonarCloud static analysis
        timeout-minutes: 6
        if: inputs.flavor == 'SonarCloudStaticAnalysis'
        env:
          SONAR_SERVER_URL: "https://sonarcloud.io"
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
          SRC_DIR: ${{ github.workspace }}/rawspeed
          BUILD_DIR: ${{ github.workspace }}/rawspeed-build
          COVERAGE_REPORT_PREFIX: ${{ github.workspace }}/${{ env.COVERAGE_REPORT_PREFIX }}
        run: |
          set -xe
          cd "${SRC_DIR}"
          sonar-scanner --define project.settings="${SRC_DIR}/.ci/sonar-project.properties" --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.projectKey="${{ secrets.SONAR_PROJECT_KEY }}" --define sonar.organization="${{ secrets.SONAR_ORGANIZATION }}" --define sonar.cfamily.compile-commands="${BUILD_DIR}/compile_commands.json" --define sonar.cfamily.analysisCache.mode=fs --define sonar.cfamily.analysisCache.path="${BUILD_DIR}/sonar-scanner-cache"