diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7b234249f90..b515dbff9f3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,188 +12,188 @@ concurrency: jobs: pr-builder: needs: - # - changed-files - # - checks + - changed-files + - checks - conda-cpp-build - # - conda-cpp-checks - # - conda-cpp-tests + - conda-cpp-checks + - conda-cpp-tests - conda-python-build - # - conda-python-cudf-tests - # - conda-python-other-tests - # - conda-java-tests - # - static-configure - # - conda-notebook-tests - # - docs-build + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests + - static-configure + - conda-notebook-tests + - docs-build - wheel-build-libcudf - wheel-build-pylibcudf - wheel-build-cudf - # - wheel-tests-cudf + - wheel-tests-cudf - wheel-build-cudf-polars - # - wheel-tests-cudf-polars + - wheel-tests-cudf-polars - wheel-build-dask-cudf - # - wheel-tests-dask-cudf + - wheel-tests-dask-cudf - devcontainer - # - unit-tests-cudf-pandas - # - pandas-tests - # - pandas-tests-diff + - unit-tests-cudf-pandas + - pandas-tests + - pandas-tests-diff secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 if: always() with: needs: ${{ toJSON(needs) }} - # changed-files: - # runs-on: ubuntu-latest - # name: "Check changed files" - # outputs: - # test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} - # test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} - # test_notebooks: ${{ steps.changed-files.outputs.notebooks_any_changed == 'true' }} - # test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} - # steps: - # - name: Get PR info - # id: get-pr-info - # uses: nv-gha-runners/get-pr-info@main - # - name: Checkout code repo - # uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # persist-credentials: false - # - name: Calculate merge base - # id: calculate-merge-base - # env: - # PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} - # BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} - # run: | - # (echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") > "$GITHUB_OUTPUT" - # - name: Get changed files - # id: changed-files - # uses: tj-actions/changed-files@v45 - # with: - # base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }} - # sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} - # files_yaml: | - # cpp: - # - '**' - # - '!CONTRIBUTING.md' - # - '!README.md' - # - '!docs/**' - # - '!img/**' - # - '!java/**' - # - '!notebooks/**' - # - '!python/**' - # java: - # - '**' - # - '!CONTRIBUTING.md' - # - '!README.md' - # - '!docs/**' - # - '!img/**' - # - '!notebooks/**' - # - '!python/**' - # notebooks: - # - '**' - # - '!CONTRIBUTING.md' - # - '!README.md' - # - '!java/**' - # python: - # - '**' - # - '!CONTRIBUTING.md' - # - '!README.md' - # - '!docs/**' - # - '!img/**' - # - '!java/**' - # - '!notebooks/**' - # checks: - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 - # with: - # enable_check_generated_files: false + changed-files: + runs-on: ubuntu-latest + name: "Check changed files" + outputs: + test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} + test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} + test_notebooks: ${{ steps.changed-files.outputs.notebooks_any_changed == 'true' }} + test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + steps: + - name: Get PR info + id: get-pr-info + uses: nv-gha-runners/get-pr-info@main + - name: Checkout code repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Calculate merge base + id: calculate-merge-base + env: + PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} + BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + run: | + (echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") > "$GITHUB_OUTPUT" + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }} + sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} + files_yaml: | + cpp: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!java/**' + - '!notebooks/**' + - '!python/**' + java: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + notebooks: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!java/**' + python: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!java/**' + - '!notebooks/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + with: + enable_check_generated_files: false conda-cpp-build: - # needs: checks + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 with: build_type: pull-request - # conda-cpp-checks: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - # with: - # build_type: pull-request - # enable_check_symbols: true - # conda-cpp-tests: - # needs: [conda-cpp-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_cpp == 'true' - # with: - # build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + with: + build_type: pull-request + enable_check_symbols: true + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_cpp == 'true' + with: + build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 with: build_type: pull-request - # conda-python-cudf-tests: - # needs: [conda-python-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # build_type: pull-request - # script: "ci/test_python_cudf.sh" - # conda-python-other-tests: - # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - # needs: [conda-python-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # build_type: pull-request - # script: "ci/test_python_other.sh" - # conda-java-tests: - # needs: [conda-cpp-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_java == 'true' - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_java.sh" - # static-configure: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # # Use the wheel container so we can skip conda solves and since our - # # primary static consumers (Spark) are not in conda anyway. - # container_image: "rapidsai/ci-wheel:latest" - # run_script: "ci/configure_cpp_static.sh" - # conda-notebook-tests: - # needs: [conda-python-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_notebooks == 'true' - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_notebooks.sh" - # docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/build_docs.sh" + conda-python-cudf-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + build_type: pull-request + script: "ci/test_python_cudf.sh" + conda-python-other-tests: + # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + build_type: pull-request + script: "ci/test_python_other.sh" + conda-java-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_java == 'true' + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_java.sh" + static-configure: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + # Use the wheel container so we can skip conda solves and since our + # primary static consumers (Spark) are not in conda anyway. + container_image: "rapidsai/ci-wheel:latest" + run_script: "ci/configure_cpp_static.sh" + conda-notebook-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_notebooks == 'true' + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-libcudf: - # needs: checks + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -202,7 +202,7 @@ jobs: build_type: pull-request script: "ci/build_wheel_libcudf.sh" wheel-build-pylibcudf: - needs: [wheel-build-libcudf] + needs: [checks, wheel-build-libcudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 with: @@ -215,14 +215,14 @@ jobs: with: build_type: pull-request script: "ci/build_wheel_cudf.sh" - # wheel-tests-cudf: - # needs: [wheel-build-cudf, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # build_type: pull-request - # script: ci/test_wheel_cudf.sh + wheel-tests-cudf: + needs: [wheel-build-cudf, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + build_type: pull-request + script: ci/test_wheel_cudf.sh wheel-build-cudf-polars: needs: wheel-build-pylibcudf secrets: inherit @@ -232,18 +232,18 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/build_wheel_cudf_polars.sh" - # wheel-tests-cudf-polars: - # needs: [wheel-build-cudf-polars, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - # build_type: pull-request - # # This always runs, but only fails if this PR touches code in - # # pylibcudf or cudf_polars - # script: "ci/test_wheel_cudf_polars.sh" + wheel-tests-cudf-polars: + needs: [wheel-build-cudf-polars, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + # This always runs, but only fails if this PR touches code in + # pylibcudf or cudf_polars + script: "ci/test_wheel_cudf_polars.sh" wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit @@ -253,16 +253,16 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/build_wheel_dask_cudf.sh" - # wheel-tests-dask-cudf: - # needs: [wheel-build-dask-cudf, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - # build_type: pull-request - # script: ci/test_wheel_dask_cudf.sh + wheel-tests-dask-cudf: + needs: [wheel-build-dask-cudf, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: ci/test_wheel_dask_cudf.sh devcontainer: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 @@ -273,34 +273,34 @@ jobs: sccache -z; build-all -DBUILD_BENCHMARKS=ON --verbose; sccache -s; - # unit-tests-cudf-pandas: - # needs: [wheel-build-cudf, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - # build_type: pull-request - # script: ci/cudf_pandas_scripts/run_tests.sh - # pandas-tests: - # # run the Pandas unit tests using PR branch - # needs: [wheel-build-cudf, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - # if: needs.changed-files.outputs.test_python == 'true' - # with: - # # This selects "ARCH=amd64 + the latest supported Python + CUDA". - # matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - # build_type: pull-request - # script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr - # # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. - # test_summary_show: "none" - # pandas-tests-diff: - # # diff the results of running the Pandas unit tests and publish a job summary - # needs: pandas-tests - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - # with: - # node_type: cpu4 - # build_type: pull-request - # run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh" + unit-tests-cudf-pandas: + needs: [wheel-build-cudf, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: ci/cudf_pandas_scripts/run_tests.sh + pandas-tests: + # run the Pandas unit tests using PR branch + needs: [wheel-build-cudf, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. + test_summary_show: "none" + pandas-tests-diff: + # diff the results of running the Pandas unit tests and publish a job summary + needs: pandas-tests + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + node_type: cpu4 + build_type: pull-request + run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh"