From 1b7404a5045ba9b534c93ae8136f47ee46a9c7a9 Mon Sep 17 00:00:00 2001 From: "William D. Irons" Date: Wed, 29 Apr 2020 15:15:38 -0500 Subject: [PATCH 1/2] Bazel creates the build artifacts without the user having write access to the file, when conda tries to modify the rpath it gets a permission denied error. To fix this problem we do a chmod -R u+w $PREFIX as the last steps of the build.sh script. To solve the issue of python_abi being added as a dependency, had to remove conda-forge from the config files. Using python from conda-forge causes the python_abi dependency to get added. Reordered the channel sources to the order we want. Using the commaon build process, removed the build_scripts in ray-feedstock. --- .travis.yml | 10 ++- .../.ci_support/linux_ppc64le_python3.6.yaml | 2 +- .../.ci_support/linux_ppc64le_python3.7.yaml | 2 +- .../ray-feedstock/ci_support/build_steps.sh | 46 -------------- .../ci_support/run_docker_build.sh | 62 ------------------- conda-recipes/ray-feedstock/recipe/build.sh | 3 + conda-recipes/ray-feedstock/recipe/meta.yaml | 2 +- 7 files changed, 13 insertions(+), 114 deletions(-) delete mode 100755 conda-recipes/ray-feedstock/ci_support/build_steps.sh delete mode 100755 conda-recipes/ray-feedstock/ci_support/run_docker_build.sh diff --git a/.travis.yml b/.travis.yml index 1731f0e..292e341 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,11 +29,15 @@ branches: matrix: include: - - env: CONFIG=linux_noarch UPLOAD_PACKAGES=True DOCKER_IMAGE=ibmcom/powerai:1.6.2-pytorch-ubuntu18.04-py36-x86_64 + - env: CONFIG=linux_ppc64le_python3.6 UPLOAD_PACKAGES=True DOCKER_IMAGE=condaforge/linux-anvil-ppc64le os: linux - arch: amd64 + arch: ppc64le + language: generic + - env: CONFIG=linux_ppc64le_python3.7 UPLOAD_PACKAGES=True DOCKER_IMAGE=condaforge/linux-anvil-ppc64le + os: linux + arch: ppc64le language: generic script: - "travis_wait 60 sleep 3600 &" - - ./conda-recipes/build_scripts/run_docker_build.sh pycox-feedstock + - ./conda-recipes/build_scripts/run_docker_build.sh ray-feedstock diff --git a/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.6.yaml b/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.6.yaml index e0d6952..0bfac30 100644 --- a/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.6.yaml +++ b/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.6.yaml @@ -3,7 +3,7 @@ c_compiler: c_compiler_version: - '8' channel_sources: -- conda-forge, defaults, powerai,https://public.dhe.ibm.com/ibmdl/export/pub/software/server/ibm-ai/conda/ +- https://public.dhe.ibm.com/ibmdl/export/pub/software/server/ibm-ai/conda/,powerai,defaults channel_targets: - powerai main docker_image: diff --git a/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.7.yaml b/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.7.yaml index ecbbaef..c6afb05 100644 --- a/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.7.yaml +++ b/conda-recipes/ray-feedstock/.ci_support/linux_ppc64le_python3.7.yaml @@ -3,7 +3,7 @@ c_compiler: c_compiler_version: - '8' channel_sources: -- conda-forge, defaults, powerai, https://public.dhe.ibm.com/ibmdl/export/pub/software/server/ibm-ai/conda/ +- https://public.dhe.ibm.com/ibmdl/export/pub/software/server/ibm-ai/conda/,powerai,defaults channel_targets: - powerai main docker_image: diff --git a/conda-recipes/ray-feedstock/ci_support/build_steps.sh b/conda-recipes/ray-feedstock/ci_support/build_steps.sh deleted file mode 100755 index 4c41dd4..0000000 --- a/conda-recipes/ray-feedstock/ci_support/build_steps.sh +++ /dev/null @@ -1,46 +0,0 @@ -# (C) Copyright IBM Corp. 2018, 2020. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/usr/bin/env bash - -set -xeuo pipefail -export PYTHONUNBUFFERED=1 -export FEEDSTOCK_ROOT=/home/conda/feedstock_root -export RECIPE_ROOT=/home/conda/recipe_root -export CI_SUPPORT=/home/conda/feedstock_root/.ci_support -export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" -export IBM_POWERAI_LICENSE_ACCEPT=yes - -cat >~/.condarc </dev/null && pwd )" -PROVIDER_DIR="$(basename $THISDIR)" - -FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) -RECIPE_ROOT=$FEEDSTOCK_ROOT/recipe - -docker info - -# In order for the conda-build process in the container to write to the mounted -# volumes, we need to run with the same id as the host machine, which is -# normally the owner of the mounted volumes, or at least has write permission -export HOST_USER_ID=$(id -u) -# Check if docker-machine is being used (normally on OSX) and get the uid from -# the VM -if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then - HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) -fi - -ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" -mkdir -p "$ARTIFACTS" -DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" -rm -f "$DONE_CANARY" - -# Enable running in interactive mode ONLY if attached to a tty, -# so "-t" is made conditional -test -t 1 && USE_TTY="-t" -DOCKER_RUN_ARGS=" -i ${USE_TTY} " - -docker run ${DOCKER_RUN_ARGS} \ - -v "${RECIPE_ROOT}":/home/conda/recipe_root:ro,z \ - -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z \ - -e CONFIG \ - -e BINSTAR_TOKEN \ - -e HOST_USER_ID \ - -e UPLOAD_PACKAGES \ - -e CI \ - -a stdin -a stdout -a stderr \ - $DOCKER_IMAGE \ - bash \ - /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh - -# double-check that the build got to the end -# see https://github.com/conda-forge/conda-smithy/pull/337 -# for a possible fix - -test -f "$DONE_CANARY" diff --git a/conda-recipes/ray-feedstock/recipe/build.sh b/conda-recipes/ray-feedstock/recipe/build.sh index 790bdcc..c66da7c 100755 --- a/conda-recipes/ray-feedstock/recipe/build.sh +++ b/conda-recipes/ray-feedstock/recipe/build.sh @@ -5,3 +5,6 @@ cd python cd ray/dashboard/client && npm ci && npm run build cd ../../.. $PYTHON setup.py install --single-version-externally-managed --record=record.txt + +# Allow Conda to modify rpath in files +chmod -R u+w $PREFIX diff --git a/conda-recipes/ray-feedstock/recipe/meta.yaml b/conda-recipes/ray-feedstock/recipe/meta.yaml index e19b05d..8174a70 100644 --- a/conda-recipes/ray-feedstock/recipe/meta.yaml +++ b/conda-recipes/ray-feedstock/recipe/meta.yaml @@ -14,7 +14,7 @@ source: - 0002-Arrow-Fixes-PPC.patch build: - number: 1 + number: 2 entry_points: - ray=ray.scripts.scripts:main - rllib=ray.rllib.scripts:cli [rllib] From 3087e718d428e71f92fbe7770256f45ae2bb8c43 Mon Sep 17 00:00:00 2001 From: "William D. Irons" Date: Thu, 30 Apr 2020 08:39:04 -0500 Subject: [PATCH 2/2] try removing travis_wait to get a build to pass --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 292e341..eea4c66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,5 +39,5 @@ matrix: language: generic script: - - "travis_wait 60 sleep 3600 &" +# - "travis_wait 60 sleep 3600 &" - ./conda-recipes/build_scripts/run_docker_build.sh ray-feedstock