sycl-linux-run-tests.yml

name: SYCL E2E

on:
  workflow_call:
    inputs:
      name:
        type: string
        required: True

      runner:
        type: string
        required: True
      image:
        type: string
        required: True
      image_options:
        type: string
        required: True

      target_devices:
        type: string
        required: True
      extra_cmake_args:
        type: string
        required: False
      tests_selector:
        description: |
          Two possible options: "e2e" and "cts".
        type: string
        default: "e2e"

      extra_lit_opts:
        description: |
          Extra options to be added to LIT_OPTS.
        type: string
        default: ''

      ref:
        type: string
        required: True
      merge_ref:
        description: |
          Commit-ish to merge post-checkout if non-empty. Must be reachable from
          the default_branch input paramter.
        type: string
        default: 'FETCH_HEAD'
        required: False

      sycl_toolchain_artifact:
        type: string
        default: ''
        required: False
      sycl_toolchain_archive:
        type: string
        default: ''
        required: False
      sycl_toolchain_decompress_command:
        type: string
        default: ''
        required: False

      # TODO: add inputs for upload/download of e2e binaries artifact
      #e2e_binaries_artifact:
      #  type: string
      #  default: ''
      #  required: False
      #e2e_binaries_archive:
      #  type: string
      #  default: ''
      #  required: False
      #e2e_binaries_decompress_command:
      #  type: string
      #  default: ''
      #  required: False

      # For e2e binary artifact upload
      upload_artifact:
        type: string
        default: 'false'
      build_artifact_suffix:
        type: string
        default: 'default'
      artifact_archive_name:
        type: string
        default: e2e_binaries.tar.zst
      ##########################################################################

      reset_intel_gpu:
        type: string
        required: False
      install_igc_driver:
        type: string
        required: False
      install_dev_igc_driver:
        type: string
        required: False
      env:
        type: string
        default: '{}'
        required: False

      skip_run:
        type: string
        default: 'false'
        required: False

    # TODO: add outputs for upload of e2e binaries artifact
    outputs:
      run_conclusion:
        value: ${{ jobs.run.outputs.build_conclusion }}
      artifact_archive_name:
        value: ${{ jobs.run.outputs.artifact_archive_name }}
      artifact_decompress_command:
        value: ${{ jobs.run.outputs.artifact_decompress_command }}
    ##########################################################################

  workflow_dispatch:
    inputs:
      runner:
        type: choice
        options:
          - '["Linux", "gen12"]'
          - '["amdgpu"]'
          - '["Linux", "arc"]'
          - '["cts-cpu"]'
          - '["Linux", "build"]'
      image:
        description: |
          Use option ending with ":build" for AMDGPU, ":latest" for the rest.
        type: choice
        options:
          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest'
          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build'
      image_options:
        description: |
          Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest.
        type: choice
        options:
          - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN'
          - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN'
      target_devices:
        type: choice
        options:
          - 'opencl:cpu'
          - 'opencl:gpu'
          - 'opencl:fpga'
          - 'level_zero:gpu'
          - 'ext_oneapi_hip:gpu'
      tests_selector:
        type: choice
        options:
          - e2e
          - cts

      env:
        description: |
          Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT.
          LIT_OPTS won't work as we redefine it as part of this workflow.

          For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to
          build.

          Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}'
        default: '{}'

      extra_lit_opts:
        description: |
          Extra options to be added to LIT_OPTS.
        default: ''

      install_igc_driver:
        type: choice
        options:
          - false
          - true

      install_dev_igc_driver:
        type: choice
        options:
          - false
          - true
      # TODO: e2e binary upload params
      upload_artifact:
        type: choice
        options:
          - "false"
          - "true"
      #
      # FIXME: max of 10 dispatch arguments
      #build_artifact_suffix:
      #  type: choice
      #  options:
      #    - "default"
      #retention-days:
      #  type: choice
      #  options:
      #    - 3
      #################################

permissions:
  contents: read

jobs:
  run:
    if: github.event_name == 'workflow_dispatch' || inputs.skip_run == 'false'
    name: ${{ inputs.name }}
    runs-on: ${{ fromJSON(inputs.runner) }}
    container:
      image: ${{ inputs.image }}
      options: ${{ inputs.image_options }}
    # TODO: Add outputs for uploading e2e binaries
    outputs:
      run_conclusion: ${{ steps.run.conclusion }}
      artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
      artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
    ##########################################################################
    env: ${{ fromJSON(inputs.env) }}
    steps:
    # TODO: Deduce artifact archive params
    # NOTE: Only needs to run for build-only runs
    - name: Deduce artifact archive params
      if: inputs.upload_artifact == 'true'
      # To reduce number of inputs parameters that is limited for manual triggers.
      id: artifact_info
      run: |
        NAME="${{inputs.artifact_archive_name}}"
        if [ -z "$NAME" ]; then
          NAME=e2e_binaries.tar.zst
        fi
        echo ARCHIVE_NAME="$NAME" >> $GITHUB_OUTPUT
        if [ "${NAME}" != "${NAME%.tar.gz}" ]; then
          echo COMPRESS="gzip" >> $GITHUB_OUTPUT
          echo DECOMPRESS="gunzip" >> $GITHUB_OUTPUT
        elif [ "${NAME}" != "${NAME%.tar.zst}" ]; then
          echo COMPRESS="zstd -9" >> $GITHUB_OUTPUT
          echo DECOMPRESS="zstd" >> $GITHUB_OUTPUT
        else
          echo "Unsupported extension"
          exit 1
        fi
    ############################################################################
    - name: Reset Intel GPU
      if: inputs.reset_intel_gpu == 'true'
      run: |
        sudo mount -t debugfs none /sys/kernel/debug
        sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged'
    - uses: actions/checkout@v4
      with:
        ref: ${{ inputs.ref }}
        sparse-checkout: |
          devops
    - name: Register cleanup after job is finished
      uses: ./devops/actions/cleanup
    - name: Checkout E2E tests
      if: inputs.tests_selector == 'e2e'
      uses: ./devops/actions/cached_checkout
      with:
        path: llvm
        ref: ${{ inputs.ref || github.sha }}
        merge_ref: ${{ inputs.merge_ref }}
        cache_path: "/__w/repo_cache/"
    - name: Checkout SYCL CTS tests
      if: inputs.tests_selector == 'cts'
      uses: ./devops/actions/cached_checkout
      with:
        path: khronos_sycl_cts
        repository: 'KhronosGroup/SYCL-CTS'
        ref: 'main'
        default_branch: 'main'
        cache_path: "/__w/repo_cache/"
    - name: SYCL CTS GIT submodules init
      if: inputs.tests_selector == 'cts'
      run: |
        git -C khronos_sycl_cts submodule update --init
    - name: Install drivers
      if: inputs.install_igc_driver == 'true' || inputs.install_dev_igc_driver == 'true'
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        if [ "${{ inputs.install_dev_igc_driver }}" = "true" ]; then
            # If libllvm14 is already installed (dev igc docker), still return true.
            sudo apt-get install -yqq libllvm14 || true;
        fi
        sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.install_dev_igc_driver == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all
    - name: Source OneAPI TBB vars.sh
      shell: bash
      run: |
        # https://github.com/actions/runner/issues/1964 prevents us from using
        # the ENTRYPOINT in the image.
        env | sort > env_before
        if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
          source /runtimes/oneapi-tbb/env/vars.sh;
        elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
          source /opt/runtimes/oneapi-tbb/env/vars.sh;
        else
          echo "no TBB vars in /opt/runtimes or /runtimes";
        fi
        env | sort > env_after
        comm -13 env_before env_after >> $GITHUB_ENV
        rm env_before env_after
    - name: Download SYCL toolchain
      if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run'
      uses: actions/download-artifact@v4
      with:
        name: ${{ inputs.sycl_toolchain_artifact }}
    - name: Debug prints [workflow_run]
      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
      run: |
        pwd
        ls
    - name: Download SYCL toolchain [workflow_run]
      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
      uses: actions/github-script@v7
      with:
        script: |
          const name = '${{ inputs.sycl_toolchain_artifact }}'
          let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
             owner: context.repo.owner,
             repo: context.repo.repo,
             run_id: context.payload.workflow_run.id,
          });
          let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
            return artifact.name == name
          })[0];
          let download = await github.rest.actions.downloadArtifact({
             owner: context.repo.owner,
             repo: context.repo.repo,
             artifact_id: matchArtifact.id,
             archive_format: 'zip',
          });
          let fs = require('fs');
          fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data));
    - name: Unzip artifact [workflow_run]
      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
      run: |
        pwd
        ls
        unzip ${{ inputs.sycl_toolchain_artifact }}.zip
        rm ${{ inputs.sycl_toolchain_artifact }}.zip
    - name: Extract/Setup SYCL toolchain
      if: inputs.sycl_toolchain_artifact != ''
      run: |
        mkdir toolchain
        tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain
        rm -f ${{ inputs.sycl_toolchain_archive }}
        echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV
        echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
    - run: which clang++ sycl-ls
    - run: sycl-ls --verbose
    - run: SYCL_UR_TRACE=1 sycl-ls
    - run: |
          if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then
            cat /usr/local/lib/igc/IGCTAG.txt
          fi

    ## TODO: Download E2E Binaries
    ## TODO: Extract/Setup E2E Binaries
    # Only should be done for run-only runs (artifact name needs to be passed in)

    - name: Deduce E2E CMake options
      if: inputs.tests_selector == 'e2e'
      id: cmake_opts
      shell: bash
      env:
        CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }}
      run: |
        if [ -n "$CMAKE_EXTRA_ARGS" ]; then
          echo "opts=$CMAKE_EXTRA_ARGS" >> $GITHUB_OUTPUT
        fi
    - name: Configure E2E tests
      if: inputs.tests_selector == 'e2e'
      run: |
        cmake -GNinja -B./build-e2e -S./llvm/sycl/test-e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.target_devices }}" -DCMAKE_CXX_COMPILER="$(which clang++)" -DLLVM_LIT="$PWD/llvm/llvm/utils/lit/lit.py" ${{ steps.cmake_opts.outputs.opts }}
    - name: SYCL End-to-end tests
      shell: bash {0}
      if: inputs.tests_selector == 'e2e'
      env:
        LIT_OPTS: -v --no-progress-bar --show-unsupported --show-pass --show-xfail --max-time 3600 --time-tests ${{ inputs.extra_lit_opts }}
      run: |
        ninja -C build-e2e check-sycl-e2e > e2e.log 2>&1
        exit_code=$?
        cat e2e.log
        if [ $exit_code -ne 0 ]; then
          awk '/^Failed Tests|Unexpectedly Passed Tests|Unresolved tests|Testing Time/{flag=1}/FAILED: CMakeFiles/{flag=0}flag' e2e.log >> $GITHUB_STEP_SUMMARY
        fi
        exit $exit_code
    - name: Build SYCL CTS tests
      if: inputs.tests_selector == 'cts'
      env:
        CMAKE_EXTRA_ARGS: ${{ inputs.extra_cmake_args }}
      run: |
        cts_exclude_filter=""
        # If CTS_TESTS_TO_BUILD is null - use filter
        if [ -z "$CTS_TESTS_TO_BUILD" ]; then
          if [ "${{ contains(inputs.target_devices, 'opencl:cpu')  }}" = "true" ]; then
            cts_exclude_filter=$PWD/devops/cts_exclude_filter_OCL_CPU
          elif [ "${{ contains(inputs.target_devices, 'level_zero:gpu')  }}" = "true" ]; then
            cts_exclude_filter=$PWD/devops/cts_exclude_filter_L0_GPU
          fi

          # List excluded SYCL CTS categories:
          # SYCL_CTS_EXCLUDE_TEST_CATEGORIES - Optional file specifying a list
          # of test categories to be excluded from the build.
          echo "::group::Excluded test categories"
          cat $cts_exclude_filter
          echo "::endgroup::"
        fi

        cmake -GNinja -B./build-cts -S./khronos_sycl_cts -DCMAKE_CXX_COMPILER=$(which clang++) \
        -DSYCL_IMPLEMENTATION=DPCPP \
        -DSYCL_CTS_EXCLUDE_TEST_CATEGORIES="$cts_exclude_filter" \
        -DSYCL_CTS_ENABLE_OPENCL_INTEROP_TESTS=OFF \
        -DDPCPP_INSTALL_DIR="$(dirname $(which clang++))/.." \
        $CMAKE_EXTRA_ARGS
        # Ignore errors so that if one category build fails others still have a
        # chance to finish and be executed at the run stage. Note that
        # "test_conformance" target skips building "test_all" executable.
        ninja -C build-cts -k0 $( [ -n "$CTS_TESTS_TO_BUILD" ] && echo "$CTS_TESTS_TO_BUILD" || echo "test_conformance")

    - name: SYCL CTS List devices
      # Proceed with execution even if the 'build' step did not succeed.
      if: inputs.tests_selector == 'cts' && (success() || failure())
      env:
        ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
      run: |
        ./build-cts/bin/* --list-devices

    - name: Run SYCL CTS tests
      # Proceed with execution even if the previous two steps did not succeed.
      if: inputs.tests_selector == 'cts' && (success() || failure())
      env:
        ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
      # This job takes ~100min usually. But sometimes some test isn't
      # responding, so the job reaches the 360min limit. Setting a lower one.
      timeout-minutes: 150
      # By-default GitHub actions execute the "run" shell script with -e option,
      # so the execution terminates if any command returns a non-zero status.
      # Since we're using a loop to run all test-binaries separately, some test
      # may fail and terminate the execution. Setting "shell" value to override
      # the default behavior.
      # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#custom-shell
      shell: bash {0}
      run: |
        # Run each test category separately so that
        #  - crash on one would not affect others
        #  - multiple tests could be run in parallel
        mkdir logs
        find build-cts/bin/ -type f -print | \
          xargs -t -I % -P 8 sh -c 'log=logs/$(basename %).log ; echo % >$log ; date >>$log ; timeout 60m % >>$log 2>&1 ; ret=$? ; echo "exit code: $ret" >>$log ; date >>$log  ; exit $ret'
        ret=$?

        for f in logs/* ; do
          echo "::group::$f"
          cat $f
          echo "::endgroup::"
        done

        echo "::group::Fails:"
        grep 'exit code: [^0]' -r logs
        echo "::endgroup::"

        grep 'exit code: [^0]' -r logs >> $GITHUB_STEP_SUMMARY

        exit $ret
    # NOTE: Only should be ran on build-only runs
    # TODO: Pack E2E binaries
    - name: Pack toolchain
      if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
      run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C ./build-e2e .
    # TODO: Upload E2e binaries
    - name: Upload toolchain
      if: ${{ always() && !cancelled() && inputs.upload_artifact == 'true'}}
      uses: actions/upload-artifact@v4
      with:
        name: sycl_linux_${{ inputs.build_artifact_suffix }}
        path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
        retention-days: ${{ inputs.retention-days }}
    ############################################################################