From cea9b2e4740dea578465ed4df9608630b9e3299e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 7 Feb 2025 12:27:08 +0100 Subject: [PATCH] GH-45159: [CI][Integration] Remove substrait consumer-testing integration job --- ci/docker/conda-python-substrait.dockerfile | 59 --------------------- ci/scripts/install_substrait_consumer.sh | 36 ------------- ci/scripts/integration_substrait.sh | 31 ----------- dev/tasks/tasks.yml | 8 --- docker-compose.yml | 28 ---------- 5 files changed, 162 deletions(-) delete mode 100644 ci/docker/conda-python-substrait.dockerfile delete mode 100755 ci/scripts/install_substrait_consumer.sh delete mode 100755 ci/scripts/integration_substrait.sh diff --git a/ci/docker/conda-python-substrait.dockerfile b/ci/docker/conda-python-substrait.dockerfile deleted file mode 100644 index 36dd64e51e7ad..0000000000000 --- a/ci/docker/conda-python-substrait.dockerfile +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -ARG repo -ARG arch -ARG python=3.9 - -FROM ${repo}:${arch}-conda-python-${python} - -COPY ci/conda_env_python.txt \ - ci/conda_env_sphinx.txt \ - /arrow/ci/ - -# Note: openjdk is pinned to 17 because the -# substrait repo currently pins to jdk 17. -# Newer jdk versions are currently failing -# due to the recent upgrade to Gradle 8 via -# install_substrait_consumer.sh. -# https://github.com/substrait-io/substrait-java/issues/274 -RUN mamba install -q -y \ - --file arrow/ci/conda_env_python.txt \ - --file arrow/ci/conda_env_sphinx.txt \ - $([ "$python" == "3.9" ] && echo "pickle5") \ - python=${python} \ - openjdk=17 \ - nomkl && \ - mamba clean --all - - -ARG substrait=latest -COPY ci/scripts/install_substrait_consumer.sh /arrow/ci/scripts/ - -RUN /arrow/ci/scripts/install_substrait_consumer.sh - -ENV ARROW_ACERO=ON \ - ARROW_COMPUTE=ON \ - ARROW_CSV=ON \ - ARROW_DATASET=ON \ - ARROW_FILESYSTEM=ON \ - ARROW_FLIGHT=OFF \ - ARROW_FLIGHT_SQL=OFF \ - ARROW_GANDIVA=OFF \ - ARROW_JSON=ON \ - ARROW_SUBSTRAIT=ON \ - ARROW_TESTING=OFF diff --git a/ci/scripts/install_substrait_consumer.sh b/ci/scripts/install_substrait_consumer.sh deleted file mode 100755 index 2e6d299f68bf2..0000000000000 --- a/ci/scripts/install_substrait_consumer.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -echo "Install Substrait Consumer Test Suite"; - -git clone https://github.com/substrait-io/consumer-testing.git -cd consumer-testing -# avoid installing pyarrow -grep -v 'pyarrow\|arrow-nightlies' requirements.txt > requirements-no-arrow.txt -pip install -r requirements-no-arrow.txt - -pip install -r requirements-build.txt -# setup substrait-java -git submodule init -git submodule update --init -./build-and-copy-isthmus-shadow-jar.sh -# install substrait_consumer library -python setup.py install diff --git a/ci/scripts/integration_substrait.sh b/ci/scripts/integration_substrait.sh deleted file mode 100755 index 152a8d9440187..0000000000000 --- a/ci/scripts/integration_substrait.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -# check that optional pyarrow modules are available -# because pytest would just skip the substrait tests -echo "Substrait Integration Tests" -echo "Validating imports" -python -c "import pyarrow.substrait" -python -c "from substrait_consumer.consumers.acero_consumer import AceroConsumer" - -echo "Executing pytest" -cd consumer-testing -pytest -r s substrait_consumer/tests/functional/extension_functions/test_boolean_functions.py --producer isthmus --consumer acero diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ff0e3892f4d20..bb3e5e1d3ee8d 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1088,14 +1088,6 @@ tasks: PYTEST_ARGS: "-m hypothesis -W error" image: conda-python-pandas - test-conda-python-3.10-substrait: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: "3.10" - image: conda-python-substrait - test-ubuntu-22.04-python-313-freethreading: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index bf8093ec60307..378b560757334 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,7 +128,6 @@ x-hierarchy: - conda-python-hdfs - conda-python-no-numpy - conda-python-spark - - conda-python-substrait - conda-verify-rc - conan - debian-cpp: @@ -1467,33 +1466,6 @@ services: /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/integration_dask.sh"] - conda-python-substrait: - # Usage: - # docker compose build conda - # docker compose build conda-cpp - # docker compose build conda-python - # docker compose build conda-python-substrait - # docker compose run --rm conda-python-substrait - image: ${REPO}:${ARCH}-conda-python-${PYTHON} - build: - context: . - dockerfile: ci/docker/conda-python-substrait.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON} - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - shm_size: *shm-size - environment: - <<: [*common, *ccache] - ARROW_SUBSTRAIT: "ON" - volumes: *conda-volumes - command: - ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/integration_substrait.sh"] - conda-python-cpython-debug: # Usage: # docker compose build conda