From ebed2d4ac474ba871bc8f98a9a84713f025a1b2d Mon Sep 17 00:00:00 2001 From: Colin Davidson Date: Fri, 14 Feb 2025 10:50:03 +0000 Subject: [PATCH 1/3] Switch to calling an action for PR tests and populate the cache sequentially There was a race condition in updating the cache as it just ran the tests overnight in parallel. Move all the build parts of PR jobs to an action. Add a workflow pr_tests_cache.yml to seed the caches by calling the actions in sequence. The scheduled run_pr_tests overnight will no longer have the ability to seed the cache and this will only be done via pr_tests_cache.yml. This does not yet delete the caches. We can't be sure this works so this is initially commented out. --- .../do_build_pr/run_host_x86_64/action.yml | 28 +++ .../do_build_pr/run_riscv_m1/action.yml | 27 +++ .../action.yml | 42 ++++ .../action.yml | 27 +++ .../action.yml | 32 +++ .../action.yml | 33 +++ .../action.yml | 31 +++ .../action.yml | 26 ++ .../action.yml | 43 ++++ .github/actions/setup_build/action.yml | 2 +- .github/workflows/pr_tests_cache.yml | 228 ++++++++++++++++++ .github/workflows/run_pr_tests.yml | 166 ++----------- .github/workflows/run_pr_tests_caller.yml | 1 - 13 files changed, 536 insertions(+), 150 deletions(-) create mode 100644 .github/actions/do_build_pr/run_host_x86_64/action.yml create mode 100644 .github/actions/do_build_pr/run_riscv_m1/action.yml create mode 100644 .github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline/action.yml create mode 100644 .github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16/action.yml create mode 100644 .github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0/action.yml create mode 100644 .github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0/action.yml create mode 100644 .github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz/action.yml create mode 100644 .github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release/action.yml create mode 100644 .github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline/action.yml create mode 100644 .github/workflows/pr_tests_cache.yml diff --git a/.github/actions/do_build_pr/run_host_x86_64/action.yml b/.github/actions/do_build_pr/run_host_x86_64/action.yml new file mode 100644 index 000000000..d63ec975d --- /dev/null +++ b/.github/actions/do_build_pr/run_host_x86_64/action.yml @@ -0,0 +1,28 @@ +name: build_pr_host_x86_64 +description: Build pr host x86_64 + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build build_offline + - name: build host x86_64 online release + uses: ./.github/actions/do_build_ock + with: + build_type: Release + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} + - name: build host x86_64 offline release + uses: ./.github/actions/do_build_ock + with: + build_type: Release + extra_flags: -DCA_RUNTIME_COMPILER_ENABLED=OFF -DCA_EXTERNAL_CLC=${{ github.workspace }}/build/bin/clc + build_dir: build_offline + build_targets: UnitCL + assemble_spirv_ll_lit_test_offline: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_riscv_m1/action.yml b/.github/actions/do_build_pr/run_riscv_m1/action.yml new file mode 100644 index 000000000..f7995b1ca --- /dev/null +++ b/.github/actions/do_build_pr/run_riscv_m1/action.yml @@ -0,0 +1,27 @@ +name: build_pr_riscv_m1 +description: Build pr riscv_m1 + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build + + - name: build_ock + uses: ./.github/actions/do_build_ock + with: + build_type: ${{ inputs.build_type }} + mux_targets_enable: riscv + mux_compilers_enable: refsi_m1 + external_compiler_dirs: "${{ github.workspace }}/examples/refsi/refsi_m1/compiler/refsi_m1" + riscv_enabled: ON + enable_rvv_scalable_vecz_check: ON + enable_rvv_scalable_vp_vecz_check: ON + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline/action.yml b/.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline/action.yml new file mode 100644 index 000000000..9d72d019a --- /dev/null +++ b/.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline/action.yml @@ -0,0 +1,42 @@ +name: build_pr_ubuntu_clang_x86_llvm_latest_cl3_0_offline +description: Build pr ubuntu_clang_x86_llvm_latest_cl3_0_offline + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build build_offline install_offline + + - name: build ock x86 relassert + uses: ./.github/actions/do_build_ock + with: + build_32_bit: ON + extra_flags: -DCMAKE_C_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang -DCMAKE_CXX_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang++ + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL clc' || 'check-ock' }} + enable_api: "" + builtin_kernel: ON + use_linker: gold + debug_support: ON + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} + + - name: build ock x86 offline + uses: ./.github/actions/do_build_ock + with: + build_32_bit: ON + extra_flags: -DCMAKE_C_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang -DCMAKE_CXX_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang++ + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL' || 'check-ock' }} + runtime_compiler_enabled: OFF + assemble_spirv_ll_lit_test_offline: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} + external_clc: ${GITHUB_WORKSPACE}/build/bin/clc + use_linker: gold + debug_support: ON + install_dir: $GITHUB_WORKSPACE/install_offline + build_dir: $GITHUB_WORKSPACE/build_offline + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16/action.yml b/.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16/action.yml new file mode 100644 index 000000000..ab2209508 --- /dev/null +++ b/.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16/action.yml @@ -0,0 +1,27 @@ +name: build_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16 +description: Build pr ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16 + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build + - name: build ock + uses: ./.github/actions/do_build_ock + with: + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL clc' || 'check-ock-cross' }} + host_fp16: ON + use_linker: gold + debug_support: ON + builtin_kernel: ON + enable_api: "" + toolchain_file: "scripts/../platform/arm-linux/aarch64-toolchain.cmake" + extra_flags: -DCA_BUILTINS_TOOLS_DIR=${{ github.workspace }}/llvm_install_native/bin + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0/action.yml b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0/action.yml new file mode 100644 index 000000000..223729a6a --- /dev/null +++ b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0/action.yml @@ -0,0 +1,32 @@ +name: build_pr_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0 +description: Build pr ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0 + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build + - name: build ock + uses: ./.github/actions/do_build_ock + with: + build_targets: install # Build the install target so we don't miss compilation errors + mux_targets_enable: riscv + external_compiler_dirs: ${{ github.workspace }}/examples/refsi/refsi_g1_wi/compiler/refsi_g1_wi + mux_compilers_enable: refsi_g1_wi + riscv_enabled: ON + disable_unitcl_vecz_checks: ON + enable_rvv_scalable_vecz_check: ON + enable_rvv_scalable_vp_vecz_check: ON + use_linker: gold + hal_description: RV64GCV + hal_refsi_soc: G1 + hal_refsi_thread_mode: WI + debug_support: ON + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0/action.yml b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0/action.yml new file mode 100644 index 000000000..b4f30d0d6 --- /dev/null +++ b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0/action.yml @@ -0,0 +1,33 @@ +name: build_pr_ubuntu_gcc_x86_64_riscv_fp16_cl3-0 +description: Build pr ubuntu_gcc_x86_64_riscv_fp16_cl3-0 + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build + + - name: build ock + uses: ./.github/actions/do_build_ock + with: + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL clc' || 'check-ock' }} + mux_targets_enable: riscv + mux_compilers_enable: riscv + riscv_enabled: ON + disable_unitcl_vecz_checks: ON + enable_rvv_scalable_vecz_check: ON + enable_rvv_scalable_vp_vecz_check: ON + host_enable_builtins: OFF + use_linker: gold + hal_description: RV64GCV_Zfh + hal_refsi_soc: G1 + hal_refsi_thread_mode: WG + debug_support: ON + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz/action.yml b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz/action.yml new file mode 100644 index 000000000..ad7f18674 --- /dev/null +++ b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz/action.yml @@ -0,0 +1,31 @@ +name: build_pr_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz +description: Build pr ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build + + - name: build ock + uses: ./.github/actions/do_build_ock + with: + build_targets: ${{ inputs.cache_seed == 'true' && 'clc UnitCL' || 'check-ock-UnitCL-group-vecz' }} + mux_targets_enable: riscv + mux_compilers_enable: riscv + riscv_enabled: ON + enable_rvv_scalable_vecz_check: ON + enable_rvv_scalable_vp_vecz_check: ON + use_linker: gold + hal_description: RV64GCV_Zfh + hal_refsi_soc: G1 + hal_refsi_thread_mode: WG + debug_support: ON + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release/action.yml b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release/action.yml new file mode 100644 index 000000000..70edd8a3c --- /dev/null +++ b/.github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release/action.yml @@ -0,0 +1,26 @@ +name: build_pr_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release +description: Build pr ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release + +inputs: + cache_seed: + type: boolean + default: false + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build + + - name: build ock + uses: ./.github/actions/do_build_ock + with: + build_type: Release + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL clc' || 'check-ock' }} + host_image: ON + use_linker: gold + enable_api: "" + builtin_kernel: ON + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline/action.yml b/.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline/action.yml new file mode 100644 index 000000000..3766ef5f1 --- /dev/null +++ b/.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline/action.yml @@ -0,0 +1,43 @@ +name: build_pr_windows_msvc_x86_64_llvm_latest_cl3_0_offline +description: Build pr windows_msvc_x86_64_llvm_latest_cl3_0_offline + +inputs: + cache_seed: + type: boolean + default: false + pull_request: + type: boolean + default: true + +runs: + using: "composite" + steps: + - name: remove any old dirs + shell: bash + run: + rm -rf build build_offline install_offline + + - name: build ock x86_64 + uses: ./.github/actions/do_build_ock + with: + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL clc' || 'check-ock' }} + enable_api: "" + builtin_kernel: ON + shell_to_use: pwsh + gtest_launcher: "python3;-u;${{ github.workspace }}/scripts/gtest-terse-runner.py" + debug_support: ON + enable_unitcl_expensive: ${{ !inputs.is_pull_request && 'ON' || 'OFF' }} + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} + + - name: build ock x86_64 offline + uses: ./.github/actions/do_build_ock + with: + build_targets: ${{ inputs.cache_seed == 'true' && 'UnitCL' || 'check-ock' }} + runtime_compiler_enabled: OFF + external_clc: "${{ github.workspace }}/build/bin/clc.exe" + shell_to_use: pwsh + gtest_launcher: "python3;-u;${{ github.workspace }}/scripts/gtest-terse-runner.py" + debug_support: ON + install_dir: ${{ github.workspace }}/install_offline + build_dir: ${{ github.workspace }}/build_offline + offline_kernel_tests: ${{ inputs.cache_seed == 'true' && 'OFF' || 'ON' }} diff --git a/.github/actions/setup_build/action.yml b/.github/actions/setup_build/action.yml index 355c830b5..91f6bf9fa 100644 --- a/.github/actions/setup_build/action.yml +++ b/.github/actions/setup_build/action.yml @@ -95,6 +95,6 @@ runs: uses: hendrikmuhs/ccache-action@53911442209d5c18de8a31615e0923161e435875 # v1.2.16 with: max-size: 200M - key: ccache-build + key: ccache-build-${{ inputs.os }} variant: ccache save: ${{ inputs.save }} diff --git a/.github/workflows/pr_tests_cache.yml b/.github/workflows/pr_tests_cache.yml new file mode 100644 index 000000000..b87c92841 --- /dev/null +++ b/.github/workflows/pr_tests_cache.yml @@ -0,0 +1,228 @@ +name: Seed the cache for ock builds +on: + # pull_request: + # paths: + # - '.github/workflows/pr_tests_cache.yml' + push: + branch: main + paths: + - 'source/**' + - 'clik/**' + - 'modules/**' + - 'examples/**' + - 'cmake/**' + - 'hal/**' + - '.github/workflow/pr_tests_cache.yml' + - '.github/actions/do_build_ock/**' + - '.github/actions/setup_build/**' + - '.github/action/do_build_pr/**' + - 'CMakeLists.txt' + + workflow_dispatch: + +env: + llvm_previous: '18' + llvm_current: '19' + +concurrency: + group: pr-test-cache-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + actions: write +jobs: + ubuntu_22_llvm_prev_jobs: + if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' + runs-on: ubuntu-22.04 + container: + image: ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest + volumes: + - ${{github.workspace}}:${{github.workspace}} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: setup ubuntu + uses: ./.github/actions/setup_build + with: + llvm_version: ${{ env.llvm_previous }} + llvm_build_type: RelAssert + save: true + + - name: build host_x86_64 + uses: ./.github/actions/do_build_pr/run_host_x86_64 + with: + cache_seed: true + + - name: build riscv M1 + uses: ./.github/actions/do_build_pr/run_riscv_m1 + with: + cache_seed: true + + - name: build ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz + with: + cache_seed: true + + ubuntu_22_llvm_current_jobs: + if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' + needs: [ubuntu_22_llvm_prev_jobs] + runs-on: ubuntu-22.04 + container: + image: ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest + volumes: + - ${{github.workspace}}:${{github.workspace}} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: setup-ubuntu + uses: ./.github/actions/setup_build + with: + llvm_version: ${{ env.llvm_current }} + llvm_build_type: RelAssert + save: true + + - name: build ubuntu_gcc_x86_64_riscv_fp16_cl3_0 + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0 + with: + cache_seed: true + + - name: build ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release + with: + cache_seed: true + + - name: build ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0 + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0 + with: + cache_seed: true + + # 32 bit x86 + ubuntu_22_llvm_current_x86_jobs: + if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' + needs: [ubuntu_22_llvm_current_jobs] + runs-on: ubuntu-22.04 + container: + image: ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest + volumes: + - ${{github.workspace}}:${{github.workspace}} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + # installs tools, ninja, installs llvm and sets up sccache + - name: setup-ubuntu + uses: ./.github/actions/setup_build + with: + llvm_version: ${{ env.llvm_current }} + llvm_build_type: RelAssert + save: true + cross_arch: x86 + + - name: build ubuntu_clang_x86_llvm_latest_cl3_0_offline + uses: ./.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline + with: + cache_seed: true + + # aarch 64 + ubuntu_22_llvm_current_aarch64_jobs: + if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' + needs: [ubuntu_22_llvm_current_x86_jobs] + runs-on: ubuntu-22.04 + container: + image: ghcr.io/uxlfoundation/ock_ubuntu_22.04-x86-64:latest + volumes: + - ${{github.workspace}}:${{github.workspace}} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + # installs tools, ninja, installs llvm and sets up sccache + - name: setup-ubuntu + uses: ./.github/actions/setup_build + with: + llvm_version: ${{ env.llvm_current }} + llvm_build_type: RelAssert + save: true + cross_arch: aarch64 + + - name: build ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16 + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16 + with: + cache_seed: true + + windows_llvm_current_jobs: + if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' + needs: [ubuntu_22_llvm_current_aarch64_jobs] + runs-on: windows-2019 + steps: + - name: Setup Windows llvm base + uses: llvm/actions/setup-windows@main + with: + arch: amd64 + + - name: Checkout repo + uses: actions/checkout@v4 + + # installs tools, ninja, installs llvm and sets up ccache + - name: setup-windows + uses: ./.github/actions/setup_build + with: + llvm_version: ${{ env.llvm_current }} + llvm_build_type: RelAssert + save: true + os: windows + - name: build windows_msvc_x86_64_llvm_latest_cl3_0_offline + uses: ./.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline + with: + cache_seed: true + + + # The following tries to delete old caches but fails on the branch due to permissions errors + # Look to uncomment in the future. + + # clean_cache: + # if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' + # name: Cache clean + # needs: [windows_llvm_current_jobs] + # runs-on: ubuntu-latest + # permissions: + # actions: write + # steps: + # - name: Checkout repo + # uses: actions/checkout@v4 + # with: + # sparse-checkout: .github + # - name: Cache clean + # env: + # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # # Define unique cache prefixes for deletion + # CACHE_PREFIX_LIST: "ccache-ccache-build-ubuntu ccache-ccache-build-windows" + # run: | + # set -x + # echo Cache branch name is ${{ github.ref }} + # # Define args for gh cache commands + # GH_LIST_ARGS="-r ${{ github.ref }} -L 100 --order desc --sort created-at" + # echo CACHE PREFIXES FOR CLEANING ... $CACHE_PREFIX_LIST_UBUNTU $CACHE_PREFIX_LIST_WINDOWS + # # Generate current cache list for ${{ github.ref_name }}, newest first (note: 100 cache entries is gh maximum) + # echo CACHE LIST BEFORE ... + # gh cache list $GH_LIST_ARGS | tee CACHE_LIST + # echo > CCACHE_LIST + + + # # Generate corresponding list of cache keys for deletion - retain only the newest key for each prefix + # for CACHE_PREFIX in $CACHE_PREFIX_LIST + # do + # grep -E -o "^${CACHE_PREFIX}[^[:space:]]+" CACHE_LIST | sed '1d' >> CCACHE_LIST + # done + + # cat CCACHE_LIST + + # DELETE_LIST=$(cat CCACHE_LIST) + # echo Ubuntu Delete List is $DELETE_LIST, Delete Args = $GH_DELETE_ARGS_KEY + # for KEY in $DELETE_LIST ; do gh cache delete $KEY ; done + + # # Generate post-clean list + # echo CACHE LIST AFTER ... + # gh cache list $GH_LIST_ARGS diff --git a/.github/workflows/run_pr_tests.yml b/.github/workflows/run_pr_tests.yml index 70308a278..45e1ba407 100644 --- a/.github/workflows/run_pr_tests.yml +++ b/.github/workflows/run_pr_tests.yml @@ -3,10 +3,6 @@ name: Run ock tests for PR testing on: workflow_call: inputs: - update_cache: - required: false - type: boolean - default: false is_pull_request: required: false type: boolean @@ -15,10 +11,6 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: inputs: - update_cache: - required: false - type: boolean - default: false is_pull_request: required: false type: bool @@ -46,13 +38,9 @@ jobs: with: llvm_version: 18 llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} - # These need to match the configurations of build_pr_cache to use the cache effectively - - name: build host x86_64 online release - uses: ./.github/actions/do_build_ock - with: - build_type: Release + - name: build + uses: ./.github/actions/do_build_pr/run_host_x86_64 - name: run just online lit run: @@ -62,16 +50,6 @@ jobs: run: ninja -C build check-ock-UnitCL - # use the previous build for online to get clc - - name: build host x86_64 offline release - uses: ./.github/actions/do_build_ock - with: - build_type: Release - extra_flags: -DCA_RUNTIME_COMPILER_ENABLED=OFF -DCA_EXTERNAL_CLC=${{ github.workspace }}/build/bin/clc - build_dir: build_offline - build_targets: UnitCL - assemble_spirv_ll_lit_test_offline: ON - - name: run host x86_64 offline run: ninja -C build_offline check-ock-UnitCL @@ -94,10 +72,9 @@ jobs: with: llvm_version: 18 llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} - name: build riscv M1 - uses: ./.github/actions/do_build_ock/do_build_m1 + uses: ./.github/actions/do_build_pr/run_riscv_m1 - name: run riscv M1 lit run: @@ -191,35 +168,16 @@ jobs: # installs tools, ninja, installs llvm and sets up ccache - name: setup-windows - uses: ./.github/actions/setup_build + uses: ./.github/actions/setup_build with: llvm_version: 19 llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} os: windows - - name: build ock x86_64 relassert - uses: ./.github/actions/do_build_ock + - name: build and test ock + uses: ./.github/actions/do_build_pr/run_windows_msvc_x86_64_llvm_latest_cl3_0_offline with: - build_targets: check-ock - enable_api: "" - builtin_kernel: ON - shell_to_use: pwsh - gtest_launcher: "python3;-u;${{ github.workspace }}/scripts/gtest-terse-runner.py" - debug_support: ON - enable_unitcl_expensive: ${{ !inputs.is_pull_request && 'ON' || 'OFF' }} - - - name: build ock x86_64 offline - uses: ./.github/actions/do_build_ock - with: - build_targets: check-ock - runtime_compiler_enabled: OFF - external_clc: "${{ github.workspace }}/build/bin/clc.exe" - shell_to_use: pwsh - gtest_launcher: "python3;-u;${{ github.workspace }}/scripts/gtest-terse-runner.py" - debug_support: ON - install_dir: ${{ github.workspace }}/install_offline - build_dir: ${{ github.workspace }}/build_offline + is_pull_request: inputs.is_pull_request # Based on: mr-ubuntu-gcc-x86_64-riscv-fp16-cl3.0-unitcl_vecz run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz: @@ -237,22 +195,8 @@ jobs: with: llvm_version: '18' llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} - - run: echo WORKSPACE is $GITHUB_WORKSPACE && echo PWD is `pwd` && ls -al - name: build ock - uses: ./.github/actions/do_build_ock - with: - build_targets: check-ock-UnitCL-group-vecz - mux_targets_enable: riscv - mux_compilers_enable: riscv - riscv_enabled: ON - enable_rvv_scalable_vecz_check: ON - enable_rvv_scalable_vp_vecz_check: ON - use_linker: gold - hal_description: RV64GCV_Zfh - hal_refsi_soc: G1 - hal_refsi_thread_mode: WG - debug_support: ON + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0_unitcl_vecz # Based on: mr-ubuntu-clang-x86-llvm-previous-cl3-0-offline run-ubuntu-clang-x86-llvm-latest-cl3-0-offline: @@ -270,32 +214,9 @@ jobs: with: llvm_version: '19' llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} cross_arch: x86 - - run: echo WORKSPACE is $GITHUB_WORKSPACE && echo PWD is `pwd` && ls -al - - name: build ock x86 relassert - uses: ./.github/actions/do_build_ock - with: - build_32_bit: ON - extra_flags: -DCMAKE_C_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang -DCMAKE_CXX_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang++ - build_targets: check-ock - enable_api: "" - builtin_kernel: ON - use_linker: gold - debug_support: ON - - name: build ock x86 offline - uses: ./.github/actions/do_build_ock - with: - build_32_bit: ON - extra_flags: -DCMAKE_C_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang -DCMAKE_CXX_COMPILER=$GITHUB_WORKSPACE/llvm_install/bin/clang++ - build_targets: check-ock - runtime_compiler_enabled: OFF - assemble_spirv_ll_lit_test_offline: ON - external_clc: ${GITHUB_WORKSPACE}/build/bin/clc - use_linker: gold - debug_support: ON - install_dir: $GITHUB_WORKSPACE/install_offline - build_dir: $GITHUB_WORKSPACE/build_offline + - name: build and run ock + uses: ./.github/actions/do_build_pr/run_ubuntu_clang_x86_llvm_latest_cl3_0_offline # Based on: mr-ubuntu-gcc-x86_64-riscv-fp16-cl3-0 run-ubuntu-gcc-x86_64-riscv-fp16-cl3-0: @@ -313,24 +234,8 @@ jobs: with: llvm_version: '19' llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} - - run: echo WORKSPACE is $GITHUB_WORKSPACE && echo PWD is `pwd` && ls -al - - name: build ock - uses: ./.github/actions/do_build_ock - with: - build_targets: check-ock - mux_targets_enable: riscv - mux_compilers_enable: riscv - riscv_enabled: ON - disable_unitcl_vecz_checks: ON - enable_rvv_scalable_vecz_check: ON - enable_rvv_scalable_vp_vecz_check: ON - host_enable_builtins: OFF - use_linker: gold - hal_description: RV64GCV_Zfh - hal_refsi_soc: G1 - hal_refsi_thread_mode: WG - debug_support: ON + - name: build and run ock + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_riscv_fp16_cl3_0 # Based on: mr-ubuntu-gcc-x86-llvm-latest-x86_64-images-cl3-0-release run-ubuntu-gcc-x86-llvm-latest-x86_64-images-cl3-0-release: @@ -348,17 +253,8 @@ jobs: with: llvm_version: '19' llvm_build_type: Release - save: ${{ inputs.update_cache }} - - run: echo WORKSPACE is $GITHUB_WORKSPACE && echo PWD is `pwd` && ls -al - - name: build ock - uses: ./.github/actions/do_build_ock - with: - build_type: Release - build_targets: check-ock - host_image: ON - use_linker: gold - enable_api: "" - builtin_kernel: ON + - name: build and run ock + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_llvm_latest_x86_64_images_cl3_0_release # Based on: mr-ubuntu-gcc-aarch64-llvm-previous-cl3-0-fp16 run-ubuntu-gcc-aarch64-llvm-latest-cl3-0-fp16: @@ -376,20 +272,9 @@ jobs: with: llvm_version: '19' llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} cross_arch: aarch64 - - run: echo WORKSPACE is $GITHUB_WORKSPACE && echo PWD is `pwd` && ls -al - - name: build ock - uses: ./.github/actions/do_build_ock - with: - build_targets: check-ock-cross - host_fp16: ON - use_linker: gold - debug_support: ON - builtin_kernel: ON - enable_api: "" - toolchain_file: "scripts/../platform/arm-linux/aarch64-toolchain.cmake" - extra_flags: -DCA_BUILTINS_TOOLS_DIR=${{ github.workspace }}/llvm_install_native/bin + - name: build and run ock + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_aarch64_llvm_latest_cl3_0_fp16 # Based on a combination of: mr-ubuntu-gcc-x86_64-clik # and: mr-ubuntu-gcc-x86_64-clik-refsi @@ -432,24 +317,9 @@ jobs: with: llvm_version: '19' llvm_build_type: RelAssert - save: ${{ inputs.update_cache }} - - run: echo WORKSPACE is $GITHUB_WORKSPACE && echo PWD is `pwd` && ls -al - name: build ock - uses: ./.github/actions/do_build_ock - with: - build_targets: install # Build the install target so we don't miss compilation errors - mux_targets_enable: riscv - external_compiler_dirs: ${{ github.workspace }}/examples/refsi/refsi_g1_wi/compiler/refsi_g1_wi - mux_compilers_enable: refsi_g1_wi - riscv_enabled: ON - disable_unitcl_vecz_checks: ON - enable_rvv_scalable_vecz_check: ON - enable_rvv_scalable_vp_vecz_check: ON - use_linker: gold - hal_description: RV64GCV - hal_refsi_soc: G1 - hal_refsi_thread_mode: WI - debug_support: ON + uses: ./.github/actions/do_build_pr/run_ubuntu_gcc_x86_64_refsi_g1_wi_cl3_0 + # For now DO NOT include run_cities.py testing. Run commands generated by the import tool are: #- run: python3 -u scripts/storage.py pull artefact.ca-opencl-cts --verbose --clean --path CA-OpenCL-CTS Ubuntu20 x86_64 14 Release #- run: echo 'Subgroups,subgroups/test_subgroups barrier_functions_core' >> skipped.txt diff --git a/.github/workflows/run_pr_tests_caller.yml b/.github/workflows/run_pr_tests_caller.yml index 3df7768d1..bb4bd885e 100644 --- a/.github/workflows/run_pr_tests_caller.yml +++ b/.github/workflows/run_pr_tests_caller.yml @@ -28,7 +28,6 @@ jobs: if: ${{ (github.event_name == 'schedule' && github.repository == 'uxlfoundation/oneapi-construction-kit') || github.event_name == 'pull_request' }} uses: ./.github/workflows/run_pr_tests.yml with: - update_cache: ${{ github.event_name == 'schedule' }} is_pull_request: ${{ github.event_name != 'schedule' }} # additional ones here for cron and/or push to main - also can be in different file. From 2c1e1d1a4534be500835befc839070b342d4ae74 Mon Sep 17 00:00:00 2001 From: Colin Davidson Date: Wed, 19 Feb 2025 16:59:43 +0000 Subject: [PATCH 2/3] Added Xfail and Mayfail attributes to city runner Support attribute Xfail and Mayfail for city runner. These can be set as additional elements in the csv file. Xfail means we expect it to fail and will give a failing return code if it unexpectedly pass, otherwise it does not contribute to an overall fail. Mayfail means it sometimes fails and will not contribute to a failing return code. Unlike some of the other attributes, these cannot be set with additional csv files, as it was felt this leads to overcomplexity. Instead it is expected that some form of csv combination script could be used alongside this. --- .github/actions/run_opencl_cts/action.yml | 34 +++------ .github/actions/run_sycl_cts/action.yml | 41 ++++------- .github/opencl_cts/expect_fail_all.csv | 1 - .../expect_fail_host_riscv64_linux.csv | 1 - .github/opencl_cts/override_all.csv | 1 + .../override_host_riscv64_linux.csv | 1 + .github/sycl_cts/expect_fail_all.csv | 2 - .github/sycl_cts/override_all.csv | 3 + doc/scripts/city_runner.rst | 20 ++++++ scripts/testing/city_runner/profile.py | 11 ++- scripts/testing/city_runner/profiles/basic.py | 5 +- scripts/testing/city_runner/profiles/gtest.py | 4 +- .../city_runner/profiles/tensorflow.py | 5 +- scripts/testing/city_runner/runner.py | 11 ++- scripts/testing/city_runner/test_info.py | 69 ++++++++++++++++--- scripts/testing/city_runner/ui.py | 35 ++++++++-- 16 files changed, 167 insertions(+), 77 deletions(-) delete mode 100644 .github/opencl_cts/expect_fail_all.csv delete mode 100644 .github/opencl_cts/expect_fail_host_riscv64_linux.csv create mode 100644 .github/opencl_cts/override_all.csv create mode 100644 .github/opencl_cts/override_host_riscv64_linux.csv delete mode 100644 .github/sycl_cts/expect_fail_all.csv create mode 100644 .github/sycl_cts/override_all.csv diff --git a/.github/actions/run_opencl_cts/action.yml b/.github/actions/run_opencl_cts/action.yml index 396bab891..41780cfc1 100644 --- a/.github/actions/run_opencl_cts/action.yml +++ b/.github/actions/run_opencl_cts/action.yml @@ -34,11 +34,16 @@ runs: fi echo QEMU SETTING: $QEMU_SETTING set -x - echo > expect_fail.csv + # Build override file, all is done first, then the target specific. + # The last file can overwrite previous overrides. + for csv in .github/opencl_cts/override_all.csv .github/opencl_cts/override_${{ inputs.target }}.csv + do + [ -f $csv ] && cat $csv >> override.csv + done > override.csv + echo override file: + cat override.csv + # $CTS_FILTER ignores certain test, so is treated differently to temporary fails. - [ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv - [ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv - cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv # Note: use 'eval' built-in to handle quoting/escaping/splitting reqs RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \ --color=always --timeout $CTS_TIMEOUT \ @@ -49,23 +54,6 @@ runs: -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \ -e CL_PLATFORM_INDEX=0 \ -s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \ - -i disable.csv" + -i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER \ + -o override.csv" eval $RUN_CITIES - - - name: Run expected failed opencl cts - shell: bash - env: - CTS_TIMEOUT: 18:00:00 - run: | - echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER" - set -x - RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \ - --color=always --timeout $CTS_TIMEOUT \ - $QEMU_SETTING \ - -b $GITHUB_WORKSPACE/test_conformance \ - -L $GITHUB_WORKSPACE/install_icd/lib \ - -e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \ - -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \ - -e CL_PLATFORM_INDEX=0 \ - -s expect_fail.csv" - eval $RUN_CITIES || echo failed as expected diff --git a/.github/actions/run_sycl_cts/action.yml b/.github/actions/run_sycl_cts/action.yml index 780cf221b..26967740e 100644 --- a/.github/actions/run_sycl_cts/action.yml +++ b/.github/actions/run_sycl_cts/action.yml @@ -67,12 +67,18 @@ runs: export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib export ONEAPI_DEVICE_SELECTOR=opencl:0 export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv - echo > expect_fail.csv + echo > override.csv # $CTS_FILTER ignores certain test, so is treated differently to temporary fails. - [ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv - [ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv - cp expect_fail.csv disable.csv - [ -f "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv + + # Build override file, all is done first, then the target specific. The last file can overwrite prevous overrides. + for csv in .github/sycl_cts/override_all.csv ..github/sycl_cts/override_${{ inputs.target }}.csv + do + [ -f $csv ] && cat $csv + done > override.csv + + echo override file: + cat override.csv + python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \ --color=always \ --timeout $SYCL_CTS_TIMEOUT \ @@ -85,30 +91,11 @@ runs: -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \ -r SYCL-CTS/cts.xml \ -v \ - -i disable.csv || exitcode=$? + -o override.csv \ + $SYCL_CTS_FILTER + export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so $GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt # output a diff of the generated list csv.txt and cts_all.txt diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above" exit $exitcode - - - name: run sycl cts expected fails - shell: bash - env: - PREPEND_PATH: '' # TODO: have qemu as input and set up this - SYCL_CTS_TIMEOUT: '02:00:00' - run: | - echo running sycl cts - export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib - export ONEAPI_DEVICE_SELECTOR=opencl:0 - python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \ - --color=always \ - --timeout $SYCL_CTS_TIMEOUT \ - $PREPEND_PATH \ - -p sycl_cts \ - -b SYCL-CTS/bin \ - -L SYCL-CTS/lib \ - -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \ - -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \ - -r SYCL-CTS/cts.xml \ - -s expect_fail.csv || echo failed as expected diff --git a/.github/opencl_cts/expect_fail_all.csv b/.github/opencl_cts/expect_fail_all.csv deleted file mode 100644 index b2098fddd..000000000 --- a/.github/opencl_cts/expect_fail_all.csv +++ /dev/null @@ -1 +0,0 @@ -API,api/test_api diff --git a/.github/opencl_cts/expect_fail_host_riscv64_linux.csv b/.github/opencl_cts/expect_fail_host_riscv64_linux.csv deleted file mode 100644 index 7c78bc659..000000000 --- a/.github/opencl_cts/expect_fail_host_riscv64_linux.csv +++ /dev/null @@ -1 +0,0 @@ -Math,math_brute_force/test_bruteforce -w diff --git a/.github/opencl_cts/override_all.csv b/.github/opencl_cts/override_all.csv new file mode 100644 index 000000000..51566cdca --- /dev/null +++ b/.github/opencl_cts/override_all.csv @@ -0,0 +1 @@ +API,api/test_api,Xfail diff --git a/.github/opencl_cts/override_host_riscv64_linux.csv b/.github/opencl_cts/override_host_riscv64_linux.csv new file mode 100644 index 000000000..6b8c9e266 --- /dev/null +++ b/.github/opencl_cts/override_host_riscv64_linux.csv @@ -0,0 +1 @@ +Math,math_brute_force/test_bruteforce -w,Xfail diff --git a/.github/sycl_cts/expect_fail_all.csv b/.github/sycl_cts/expect_fail_all.csv deleted file mode 100644 index d5f5bd439..000000000 --- a/.github/sycl_cts/expect_fail_all.csv +++ /dev/null @@ -1,2 +0,0 @@ -SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*" -SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*" diff --git a/.github/sycl_cts/override_all.csv b/.github/sycl_cts/override_all.csv new file mode 100644 index 000000000..612f05741 --- /dev/null +++ b/.github/sycl_cts/override_all.csv @@ -0,0 +1,3 @@ +SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*",Xfail +SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*",Xfail +SYCL_CTS,test_event "event::wait does not report asynchronous errors",Mayfail diff --git a/doc/scripts/city_runner.rst b/doc/scripts/city_runner.rst index d95b5d7ec..5414dbe70 100644 --- a/doc/scripts/city_runner.rst +++ b/doc/scripts/city_runner.rst @@ -612,6 +612,16 @@ Other Options test list. Tests marked as ``Ignored`` are not run and not counted. This is only supported for the CTS and GTest profiles at the moment. +``--override-source``, ``-o`` + A csv file containing a list of tests which will override tests in the test + csv provided by `-s`. These will override if they match the first two values + of an entry in the csv file. This is useful for updating the attribute or pool + values (the 3rd and 4th optional values). For example if we have an + expected fail, we can add an `Xfail` element. + + Note this works in order so if there are more than one line in the override + matching the test csv file it will choose the last one. + CSV File Format --------------- @@ -654,6 +664,16 @@ Unimplemented associated with tests which have been left unimplemented by the CTS, or test things that aren't in CL 1.x. +Xfail + Tests marked with the ``Xfail`` attribute are run and expected to fail. Fails + are counted in the pass rate and showed up in number of expected fails. It will only + count towards a failing exit code if it unexpectedly passes. +Mayfail + Tests marked with the ``Mayfail`` attribute are run and are allowed to fail. + This should usually be reserved for those tests that intermittently fail. + Fails are counted in the pass rate and show up in may fails. It will + not count towards a failing exit code. + .. _pools: Pools diff --git a/scripts/testing/city_runner/profile.py b/scripts/testing/city_runner/profile.py index 8dbe109dc..374c81b7e 100644 --- a/scripts/testing/city_runner/profile.py +++ b/scripts/testing/city_runner/profile.py @@ -121,6 +121,11 @@ def add_options(self, parser): "--ignored-source", default="", help="File containing a list of ignored tests to skip.") + parser.add_argument( + "-o", + "--override-source", + default="", + help="File containing a list of tests which if match the first 2 columns will override, in order.") parser.add_argument( "-l", "--log-file", type=str, help="File to log test output to") parser.add_argument( @@ -225,7 +230,7 @@ def create_run(self, test, worker_state=None): """ Create a new test run from a test description. """ raise NotImplementedError() - def load_tests(self, csv_paths, disabled_path, ignored_path): + def load_tests(self, csv_paths, disabled_path, ignored_path, override_path): """ Create the list of tests to run from a CSV. """ if not csv_paths or any(not csv_path or not os.path.exists(csv_path) for csv_path in csv_paths): raise Exception("Test list file not specified or does not exist") @@ -233,8 +238,10 @@ def load_tests(self, csv_paths, disabled_path, ignored_path): raise Exception("Disabled test list file does not exist") if ignored_path and not os.path.exists(ignored_path): raise Exception("Ignored test list file does not exist") + if override_path and not os.path.exists(override_path): + raise Exception("Override test list file does not exist") tests = (TestList - .from_file(csv_paths, disabled_path, ignored_path, self.args.test_prefix) + .from_file(csv_paths, disabled_path, ignored_path, override_path, self.args.test_prefix) .filter(self.args.patterns)) return tests diff --git a/scripts/testing/city_runner/profiles/basic.py b/scripts/testing/city_runner/profiles/basic.py index 036cdad61..53efc7084 100644 --- a/scripts/testing/city_runner/profiles/basic.py +++ b/scripts/testing/city_runner/profiles/basic.py @@ -100,7 +100,7 @@ def build_environment_vars(self): return env - def load_tests(self, csv_paths, disabled_path, ignored_path): + def load_tests(self, csv_paths, disabled_path, ignored_path, override_path): """ Find the list of tests from CSV. """ if disabled_path: print("Warning: disabled list not supported for basic profile") @@ -108,6 +108,9 @@ def load_tests(self, csv_paths, disabled_path, ignored_path): if ignored_path: print("Warning: ignored list not supported for basic profile") + if override_path: + print("Warning: override list not supported for basic profile") + parsed_tests = [] # Load tests from CSV if any were provided if csv_paths: diff --git a/scripts/testing/city_runner/profiles/gtest.py b/scripts/testing/city_runner/profiles/gtest.py index 7da47b798..3eae73e59 100644 --- a/scripts/testing/city_runner/profiles/gtest.py +++ b/scripts/testing/city_runner/profiles/gtest.py @@ -195,10 +195,12 @@ def parse_gtest_csv(self, csv_file): return test_names - def load_tests(self, csv_paths, disabled_path, ignored_path): + def load_tests(self, csv_paths, disabled_path, ignored_path, override_path): """ Find the list of tests from CSV or fallback to gtest binary. """ if disabled_path: print("Warning: disabled list not supported for gtest profile") + if override_path: + print("Warning: override list not supported for gtest profile") executable = TestExecutable(self.args.binary_name, self.args.binary_name) diff --git a/scripts/testing/city_runner/profiles/tensorflow.py b/scripts/testing/city_runner/profiles/tensorflow.py index ba288200e..3d0e29375 100644 --- a/scripts/testing/city_runner/profiles/tensorflow.py +++ b/scripts/testing/city_runner/profiles/tensorflow.py @@ -118,7 +118,7 @@ def parse_options(self, argv): return args - def load_tests(self, csv_paths, disabled_path, ignored_path): + def load_tests(self, csv_paths, disabled_path, ignored_path, override_path): """ Find the list of tests from CSV or fallback to Tensorflow binary. """ @@ -128,6 +128,9 @@ def load_tests(self, csv_paths, disabled_path, ignored_path): if ignored_path: print("Warning: ignored list not supported for tensorflow profile") + if override_path: + print("Warning: override list not supported for tensorflow profile") + # Find path to Tensorflow executable Tensorflow_exe_path = os.path.abspath(self.args.binary_path) Tensorflow_dir = os.path.dirname(Tensorflow_exe_path) diff --git a/scripts/testing/city_runner/runner.py b/scripts/testing/city_runner/runner.py index 1d80ffc59..fd603802d 100644 --- a/scripts/testing/city_runner/runner.py +++ b/scripts/testing/city_runner/runner.py @@ -121,9 +121,11 @@ def execute(self): test_source = self.args.test_source # CSV disabled_source = self.args.disabled_source # Disabled CSV ignored_source = self.args.ignored_source # Ignored CSV + override_source = self.args.override_source # Override CSV self.tests = self.profile.load_tests(test_source, disabled_source, - ignored_source) + ignored_source, + override_source) if self.args.repeat > 1: self.tests *= self.args.repeat self.num_tests = len(self.tests) @@ -260,8 +262,11 @@ def process_results(self): # Return the city runner exit code. if self.aborted: return 130 - if self.results.num_fails and not self.args.relaxed: - return 1 + if not self.args.relaxed: + if self.results.num_fails > 0: + return 1 + if self.results.num_xfail_unexpectedly_passed > 0: + return 1 return 0 def process_output(self, run): diff --git a/scripts/testing/city_runner/test_info.py b/scripts/testing/city_runner/test_info.py index 42ab6c21a..b550625cd 100644 --- a/scripts/testing/city_runner/test_info.py +++ b/scripts/testing/city_runner/test_info.py @@ -103,7 +103,7 @@ def __mul__(self, x): return TestList(new_tests, self.executables) @classmethod - def from_file(cls, list_file_paths, disabled_file_path, ignored_file_path, prefix=""): + def from_file(cls, list_file_paths, disabled_file_path, ignored_file_path, override_file_path, prefix=""): """ Load a list of tests from a CTS list file. """ tests = [] disabled_tests = [] @@ -125,7 +125,15 @@ def from_file(cls, list_file_paths, disabled_file_path, ignored_file_path, prefi line for line in stripped if line and not line.startswith("#")) chunked = csv.reader(filtered) filter_tests.extend(json.dumps(chunks) for chunks in chunked) - + # override_matches + override_tests = [] + if override_file_path: + with open(override_file_path, "r") as f: + stripped = (line.strip() for line in f) + filtered = ( + line for line in stripped if line and not line.startswith("#")) + chunked = csv.reader(filtered) + override_tests.extend(chunks for chunks in chunked) for list_file_path in list_file_paths: with open(list_file_path, "r") as f: stripped = (line.strip() for line in f) @@ -138,11 +146,20 @@ def from_file(cls, list_file_paths, disabled_file_path, ignored_file_path, prefi device_filter = chunks.pop(0) if len(chunks) < 2: raise Exception("Not enough columns in the CSV file") + + # match on the first 2 + for o in override_tests: + if chunks[:2] == o[:2]: + chunks = o argv = chunks[1].strip() serialized = json.dumps(chunks) ignored = serialized in ignored_tests disabled = serialized in disabled_tests unimplemented = False + xfail = False + mayfail = False + + pool = Pool.NORMAL if len(chunks) >= 4: @@ -156,12 +173,16 @@ def from_file(cls, list_file_paths, disabled_file_path, ignored_file_path, prefi if len(chunks) >= 3: attribute = chunks[2] - if attribute == 'Ignore': + if attribute.casefold() == 'Ignore'.casefold(): ignored = True - elif attribute == 'Disabled': + elif attribute.casefold() == 'Disabled'.casefold(): disabled = True - elif attribute == 'Unimplemented': + elif attribute.casefold() == 'Unimplemented'.casefold(): unimplemented = True + elif attribute.casefold() == 'Xfail'.casefold(): + xfail = True + elif attribute.casefold() == 'Mayfail'.casefold(): + mayfail = True elif attribute: raise Exception( "Unknown attribute '%s'" % attribute) @@ -204,6 +225,8 @@ def from_file(cls, list_file_paths, disabled_file_path, ignored_file_path, prefi test.ignore = ignored test.disabled = disabled test.unimplemented = unimplemented + test.xfail = xfail + test.mayfail = mayfail # Tests with predetermined pools based on resource usage if test.match("allocations") or test.match("integer_ops"): @@ -265,6 +288,9 @@ def __init__(self, tests): self.num_tests = len(tests) self.num_passes = 0 self.num_fails = 0 + self.num_xfail_unexpectedly_passed = 0 + self.num_xfail_expected_fails = 0 + self.num_mayfail_fails = 0 self.num_skipped = 0 self.num_timeouts = 0 self.num_passes_cts = 0 @@ -290,14 +316,30 @@ def add_run(self, run): """ Add a test run to the list of results. """ self.runs[run.test.name] = run run.num = len(self.runs) - if run.status == "PASS": - self.num_passes += 1 - elif run.status == "SKIP": + + if run.status == "SKIP": self.num_skipped += 1 elif run.status == "TIMEOUT": self.num_timeouts += 1 - else: + + if run.test.xfail: + if run.status == "PASS": + self.num_xfail_unexpectedly_passed += 1 + run.status="XFAIL_UNEXPECTEDLY_PASSED" + elif run.status == "FAIL": + self.num_xfail_expected_fails += 1 + run.status="XFAIL_EXPECTEDLY_FAILED" + elif run.test.mayfail: + if run.status == "FAIL": + self.num_mayfail_fails += 1 + run.status="MAYFAIL_FAILED" + elif run.status == "PASS": + self.num_passes += 1 + elif run.status == "PASS": + self.num_passes += 1 + elif run.status == "FAIL": self.num_fails += 1 + if run.total_tests is not None: self.num_total_cts += run.total_tests if run.passed_tests is not None: @@ -317,6 +359,9 @@ def finish(self, profile): not_runs = [] self.fail_list = [] self.timeout_list = [] + self.xfail_unexpectedly_passed_list = [] + self.may_fail_failed_list = [] + for test in self.tests: try: run = self.runs[test.name] @@ -327,6 +372,10 @@ def finish(self, profile): self.fail_list.append(run) elif run.status == "TIMEOUT": self.timeout_list.append(run) + elif run.status == "XFAIL_UNEXPECTEDLY_PASSED": + self.xfail_unexpectedly_passed_list.append(run) + elif run.status == "MAYFAIL_FAILED": + self.may_fail_failed_list.append(run) for test in not_runs: run = profile.create_run(test) run.status = "FAIL" @@ -338,6 +387,8 @@ def finish(self, profile): self.add_run(run) self.fail_list.append(run) self.fail_list.sort(key=lambda r: r.test.name) + self.xfail_unexpectedly_passed_list.sort(key=lambda r: r.test.name) + self.may_fail_failed_list.sort(key=lambda r: r.test.name) def write_junit(self, out, suite_name): """ Print results to the Junit XML file for reading by Jenkins.""" diff --git a/scripts/testing/city_runner/ui.py b/scripts/testing/city_runner/ui.py index fceb97f6b..7e708fc82 100644 --- a/scripts/testing/city_runner/ui.py +++ b/scripts/testing/city_runner/ui.py @@ -99,6 +99,9 @@ def print_test_output(self, run, results): def print_results(self, results): pass_rate = self.calc_progress(results.num_passes, results.num_tests, 1) + xfail_exp_fail_rate = self.calc_progress(results.num_xfail_expected_fails, results.num_tests, 1) + xfail_unexp_pass_rate = self.calc_progress(results.num_xfail_unexpectedly_passed, results.num_tests, 1) + mayfail_fail_rate = self.calc_progress(results.num_mayfail_fails, results.num_tests, 1) fail_rate = self.calc_progress(results.num_fails, results.num_tests, 1) timeout_rate = self.calc_progress(results.num_timeouts, results.num_tests, 1) skip_rate = self.calc_progress(results.num_skipped, results.num_tests, 1) @@ -112,6 +115,18 @@ def print_results(self, results): self.out.write(" %s\n" % run.test.name) self.out.write("\n") + if results.xfail_unexpectedly_passed_list: + self.out.write(self.fmt.red("Unexpected passing XFail tests:\n")) + for run in results.xfail_unexpectedly_passed_list: + self.out.write(" %s\n" % run.test.name) + self.out.write("\n") + + if results.may_fail_failed_list: + self.out.write(self.fmt.red("May Fail failing tests:\n")) + for run in results.may_fail_failed_list: + self.out.write(" %s\n" % run.test.name) + self.out.write("\n") + if results.timeout_list: self.out.write(self.fmt.blue("Timeout tests:\n")) for run in results.timeout_list: @@ -133,17 +148,25 @@ def print_results(self, results): self.out.write(self.fmt.white("Finished in ")) self.out.write("%s\n" % duration) # Print test figures. - self.out.write(self.fmt.green("\nPassed: ")) + self.out.write(self.fmt.green("\nPassed expectedly: ")) self.out.write("%6d (%5.1f %%)\n" % (results.num_passes, pass_rate)) - self.out.write(self.fmt.red("Failed: ")) + self.out.write(self.fmt.red("Failed unexpectedly:")) self.out.write("%6d (%5.1f %%)\n" % (results.num_fails, fail_rate)) - self.out.write(self.fmt.blue("Timeouts: ")) + if results.num_xfail_unexpectedly_passed > 0: + self.out.write(self.fmt.red("Passing unexpectedly:")) + self.out.write("%5d (%5.1f %%)\n" % (results.num_xfail_unexpectedly_passed, xfail_unexp_pass_rate)) + if results.num_xfail_expected_fails > 0: + self.out.write("Failed expectedly: %5d (%5.1f %%)\n" % (results.num_xfail_expected_fails, xfail_exp_fail_rate)) + if results.num_mayfail_fails > 0: + self.out.write(self.fmt.red("Failing may fail: ")) + self.out.write("%4d (%5.1f %%)\n" % (results.num_mayfail_fails, mayfail_fail_rate)) + self.out.write(self.fmt.blue("Timeouts: ")) self.out.write("%6d (%5.1f %%)\n" % (results.num_timeouts, timeout_rate)) - self.out.write(self.fmt.yellow("Skipped: ")) + self.out.write(self.fmt.yellow("Skipped: ")) self.out.write("%6d (%5.1f %%)\n" % (results.num_skipped, skip_rate)) - self.out.write(self.fmt.white("Overall Pass: ")) + self.out.write(self.fmt.white("Overall Pass: ")) self.out.write("%6d (%5.1f %%)\n" % (results.num_passes_cts, cts_rate)) - self.out.write(self.fmt.white("Overall Fail: ")) + self.out.write(self.fmt.white("Overall Fail: ")) self.out.write("%6d (%5.1f %%)\n" % (results.num_total_cts - results.num_passes_cts, cts_fail_rate)) From 3b259da0a76973f7f976b765dca1d0b0f451ba47 Mon Sep 17 00:00:00 2001 From: Colin Davidson Date: Fri, 21 Feb 2025 16:29:27 +0000 Subject: [PATCH 3/3] TEMPORARY - TO test it --- .github/workflows/planned_testing_caller.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/planned_testing_caller.yml b/.github/workflows/planned_testing_caller.yml index 2e7903e01..47f0d69de 100644 --- a/.github/workflows/planned_testing_caller.yml +++ b/.github/workflows/planned_testing_caller.yml @@ -23,12 +23,14 @@ jobs: if: github.repository == 'uxlfoundation/oneapi-construction-kit' || github.event_name != 'schedule' uses: ./.github/workflows/planned_testing.yml with: - target_list: '[ "host_x86_64_linux", - "host_aarch64_linux", - "host_riscv64_linux", - "host_i686_linux", - "host_refsi_linux", - "host_x86_64_windows" ]' + # target_list: '[ "host_x86_64_linux", + # "host_aarch64_linux", + # "host_riscv64_linux", + # "host_i686_linux", + # "host_refsi_linux", + # "host_x86_64_windows" ]' + target_list: '[ "host_x86_64_linux", "host_riscv64_linux"]' + #', "host_aarch64_linux", "host_riscv64_linux", "host_i686_linux", "host_refsi_linux", "host_x86_64_windows" ]' ock: true test_tornado: true test_sycl_cts: true @@ -37,7 +39,7 @@ jobs: # in a reasonable time # The following can be used to download from a previous workflow run (change id) - # download_ock_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445 - # download_dpcpp_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445 - # download_sycl_cts_artefact: host_x86_64_linux=12915462445;host_aarch64_linux=12915462445 + download_ock_artefact: host_x86_64_linux=13442977877;host_aarch64_linux=13442977877 + download_dpcpp_artefact: host_x86_64_linux=13442977877;host_aarch64_linux=13442977877 + download_sycl_cts_artefact: host_x86_64_linux=13442977877;host_aarch64_linux=13442977877 pull_request: ${{ github.event_name == 'pull_request' }}