Migrate uses of build_host_tools.sh and delete it. (iree-org#18129)

Tested here: https://github.com/ScottTodd/iree/actions/runs/10274600040/job/28431681280. The `build_host_tools.sh` script was only used by sample projects at `experimental/web/`, so I migrated those projects to use python packages instead of a source build and re-enabled those tests. Note that since we do not include the WebGPU compiler target in packages (since it is still under development and needs complicated dependencies), the `sample_webgpu` tests are still disabled. skip-ci: only affects nightly samples.yml workflow
nod-ai · Aug 7, 2024 · 7cf0e26 · 7cf0e26
1 parent ea8b4fb
commit 7cf0e26
Show file tree

Hide file tree

Showing 7 changed files with 91 additions and 148 deletions.
diff --git a/.github/workflows/samples.yml b/.github/workflows/samples.yml
@@ -59,32 +59,37 @@ jobs:
       - name: "Testing Samples"
         run: ./build_tools/testing/test_samples.sh
 
-  # TODO(scotttodd): Convert web test scripts to use installed compiler packages and local runtime
-  #                  builds, rather than local compiler builds. The host tools are too slow to build
-  #                  on standard sized runners (and for casual users).
-
-  # web:
-  #   runs-on: ubuntu-20.04
-  #   # TODO: remove dependence on magic matching build dir names
-  #   env:
-  #     HOST_BUILD_DIR: build-host-install
-  #   steps:
-  #     - name: "Checking out repository"
-  #       uses: actions/[email protected]
-  #       with:
-  #         submodules: true
-  #     - name: "Building host tools"
-  #       run: |
-  #         ./build_tools/github_actions/docker_run.sh  \
-  #           --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
-  #           gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
-  #           ./build_tools/cmake/build_host_tools.sh "${HOST_BUILD_DIR}"
-  #     - name: "Testing Experimental Web Samples"
-  #       env:
-  #         IREE_EMPSCRIPTEN_BUILD_DIR: build-emscripten
-  #       run: |
-  #         ./build_tools/github_actions/docker_run.sh  \
-  #           --env "IREE_HOST_BUILD_DIR=${HOST_BUILD_DIR}" \
-  #           --env "IREE_EMPSCRIPTEN_BUILD_DIR=${IREE_EMPSCRIPTEN_BUILD_DIR}" \
-  #           gcr.io/iree-oss/emscripten@sha256:2dd4c52f1bb499ab365aad0111fe5538b685d88af38636b409b0cf6a576ab214 \
-  #           ./experimental/web/build_and_test_samples.sh
+  web:
+    runs-on: ubuntu-20.04
+    container: gcr.io/iree-oss/emscripten@sha256:2dd4c52f1bb499ab365aad0111fe5538b685d88af38636b409b0cf6a576ab214
+    env:
+      VENV_DIR: ${{ github.workspace }}/.venv
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: "Check out repository"
+        uses: actions/[email protected]
+      - name: "Mark git safe.directory"
+        run: git config --global --add safe.directory '*'
+      - name: "Check out runtime submodules"
+        run: ./build_tools/scripts/git/update_runtime_submodules.sh
+      - uses: actions/[email protected]
+        with:
+          python-version: "3.11"
+      - name: "Setup Python venv"
+        run: |
+          python3 -m venv ${VENV_DIR}
+          source ${VENV_DIR}/bin/activate
+          python3 -m pip install \
+            --find-links https://iree.dev/pip-release-links.html \
+            --upgrade \
+            iree-compiler \
+            iree-runtime
+      - name: "Test experimental web samples"
+        env:
+          HOST_TOOLS_BINARY_DIR: ${{ env.VENV_DIR }}/bin
+          IREE_EMPSCRIPTEN_BUILD_DIR: build-emscripten
+        run: |
+          source ${VENV_DIR}/bin/activate
+          ./experimental/web/build_and_test_samples.sh ${HOST_TOOLS_BINARY_DIR}
diff --git a/build_tools/cmake/build_host_tools.sh b/build_tools/cmake/build_host_tools.sh
diff --git a/experimental/web/build_and_test_samples.sh b/experimental/web/build_and_test_samples.sh
@@ -7,17 +7,13 @@
 
 # Builds and tests IREE samples for CI.
 #
-# Accepts no arguments. Configuration must be done via environment variables.
 # The build directory for the emscripten build is taken from the environment
 # variable IREE_EMPSCRIPTEN_BUILD_DIR, defaulting to "build-emscripten".
 # Designed for CI, but can be run manually.
-#
-# NOTE: This is different from most of our CI build scripts because we make use
-# of build_sample.sh scripts that are designed to be runnable by humans with
-# minimal configuration.
 
 set -xeuo pipefail
 
+HOST_TOOLS_BINARY_DIR="$1"
 export IREE_EMPSCRIPTEN_BUILD_DIR="${IREE_EMPSCRIPTEN_BUILD_DIR:-build-emscripten}"
 
 # These samples require that all HAL drivers be disabled to avoid linking
@@ -27,11 +23,12 @@ export IREE_EMPSCRIPTEN_BUILD_DIR="${IREE_EMPSCRIPTEN_BUILD_DIR:-build-emscripte
 test -f "${IREE_EMPSCRIPTEN_BUILD_DIR}/CMakeCache.txt" \
   && rm "${IREE_EMPSCRIPTEN_BUILD_DIR}/CMakeCache.txt"
 
-experimental/web/sample_static/build_sample.sh
-experimental/web/sample_dynamic/build_sample.sh
+experimental/web/sample_static/build_sample.sh ${HOST_TOOLS_BINARY_DIR}
+experimental/web/sample_dynamic/build_sample.sh ${HOST_TOOLS_BINARY_DIR}
 
-# Clear the cache again before building the webgpu sample.
-test -f "${IREE_EMPSCRIPTEN_BUILD_DIR}/CMakeCache.txt" \
-  && rm "${IREE_EMPSCRIPTEN_BUILD_DIR}/CMakeCache.txt"
+# TODO(scotttodd): re-enable once release packages include webgpu-spirv compiler target
 
-experimental/web/sample_webgpu/build_sample.sh
+# # Clear the cache again before building the webgpu sample.
+# test -f "${IREE_EMPSCRIPTEN_BUILD_DIR}/CMakeCache.txt" \
+#   && rm "${IREE_EMPSCRIPTEN_BUILD_DIR}/CMakeCache.txt"
+# experimental/web/sample_webgpu/build_sample.sh ${HOST_TOOLS_BINARY_DIR}
diff --git a/experimental/web/sample_dynamic/build_sample.sh b/experimental/web/sample_dynamic/build_sample.sh
@@ -9,35 +9,29 @@
 #
 # Prerequisites:
 #   * Environment must be configured for Emscripten
-#   * Host tools must be built (default at IREE_SOURCE_DIR/build-host/install).
-#     The build_tools/cmake/build_host_tools.sh script can do this for you.
+#   * Host tools must be available at the $1 arg
 #
-# Usage:
-#   build_sample.sh (optional install path) && serve_sample.sh
+# Sample usage:
+#   python -m venv .venv
+#   source .venv/bin/activate
+#   python -m pip install iree-compiler iree-runtime
+#   build_sample.sh .venv/bin && serve_sample.sh
 #
-# The desired host install directory can be passed as the first argument.
-# Otherwise, it looks for an install directory under path set in the environment
-# variable IREE_HOST_BUILD_DIR (default build-host). The build directory for the
-# emscripten build is taken from the environment variable
-# IREE_EMPSCRIPTEN_BUILD_DIR, defaulting to "build-emscripten". Designed for
-# CI, but can be run manually. It reuses the build directory if it already
-# exists.
-#
-# NOTE: This is different from most of build scripts we use for CI because it is
-# intended to also be runnable by humans with minimal configuration.
+# The build directory for the emscripten build is taken from the environment
+# variable IREE_EMPSCRIPTEN_BUILD_DIR, defaulting to "build-emscripten".
+# Designed for CI, but can be run manually. It reuses the build directory if it
+# already exists.
 
 set -euo pipefail
 
 ROOT_DIR=$(git rev-parse --show-toplevel)
 
-HOST_BUILD_DIR="${IREE_HOST_BUILD_DIR:-${ROOT_DIR}/build-host}"
+HOST_TOOLS_BINARY_DIR="$1"
 BUILD_DIR="${IREE_EMPSCRIPTEN_BUILD_DIR:-build-emscripten}"
-INSTALL_ROOT="$(realpath ${1:-${HOST_BUILD_DIR}/install})"
 SOURCE_DIR="${ROOT_DIR}/experimental/web/sample_dynamic"
 BINARY_DIR="${BUILD_DIR}/experimental/web/sample_dynamic"
 IREE_PYTHON3_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE:-$(which python3)}"
 
-
 ###############################################################################
 # Setup and checking for dependencies                                         #
 ###############################################################################
@@ -56,11 +50,9 @@ mkdir -p ${BINARY_DIR}
 # Compile from .mlir input to portable .vmfb file using host tools            #
 ###############################################################################
 
-COMPILE_TOOL="${INSTALL_ROOT}/bin/iree-compile"
-
 compile_sample() {
   echo "  Compiling '$1' sample..."
-  "${COMPILE_TOOL}" "$2" \
+  "${HOST_TOOLS_BINARY_DIR}/iree-compile" "$2" \
     --iree-input-type=stablehlo \
     --iree-hal-target-backends=llvm-cpu \
     --iree-llvmcpu-target-triple=wasm32-unknown-emscripten \
@@ -87,7 +79,7 @@ emcmake "${CMAKE_BIN}" \
   -DPython3_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE}" \
   -DPYTHON_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE}" \
   -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-  -DIREE_HOST_BIN_DIR="${INSTALL_ROOT}/bin" \
+  -DIREE_HOST_BIN_DIR="${HOST_TOOLS_BINARY_DIR}" \
   -DIREE_BUILD_EXPERIMENTAL_WEB_SAMPLES=ON \
   -DIREE_HAL_DRIVER_DEFAULTS=OFF \
   -DIREE_HAL_DRIVER_LOCAL_SYNC=ON \

diff --git a/experimental/web/sample_static/build_sample.sh b/experimental/web/sample_static/build_sample.sh
@@ -9,35 +9,29 @@
 #
 # Prerequisites:
 #   * Environment must be configured for Emscripten
-#   * Host tools must be built (default at IREE_SOURCE_DIR/build-host/install).
-#     The build_tools/cmake/build_host_tools.sh script can do this for you.
+#   * Host tools must be available at the $1 arg
 #
-# Usage:
-#   build_sample.sh (optional install path) && serve_sample.sh
+# Sample usage:
+#   python -m venv .venv
+#   source .venv/bin/activate
+#   python -m pip install iree-compiler iree-runtime
+#   build_sample.sh .venv/bin && serve_sample.sh
 #
-# The desired host install directory can be passed as the first argument.
-# Otherwise, it looks for an install directory under path set in the environment
-# variable IREE_HOST_BUILD_DIR (default build-host). The build directory for the
-# emscripten build is taken from the environment variable
-# IREE_EMPSCRIPTEN_BUILD_DIR, defaulting to "build-emscripten". Designed for
-# CI, but can be run manually. It reuses the build directory if it already
-# exists.
-#
-# NOTE: This is different from most of build scripts we use for CI because it is
-# intended to also be runnable by humans with minimal configuration.
+# The build directory for the emscripten build is taken from the environment
+# variable IREE_EMPSCRIPTEN_BUILD_DIR, defaulting to "build-emscripten".
+# Designed for CI, but can be run manually. It reuses the build directory if it
+# already exists.
 
 set -euo pipefail
 
 ROOT_DIR=$(git rev-parse --show-toplevel)
 
-HOST_BUILD_DIR="${IREE_HOST_BUILD_DIR:-${ROOT_DIR}/build-host}"
+HOST_TOOLS_BINARY_DIR="$1"
 BUILD_DIR="${IREE_EMPSCRIPTEN_BUILD_DIR:-build-emscripten}"
-INSTALL_ROOT="$(realpath ${1:-${HOST_BUILD_DIR}/install})"
 SOURCE_DIR="${ROOT_DIR}/experimental/web/sample_static"
 BINARY_DIR="${BUILD_DIR}/experimental/web/sample_static/"
 IREE_PYTHON3_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE:-$(which python3)}"
 
-
 ###############################################################################
 # Setup and checking for dependencies                                         #
 ###############################################################################
@@ -56,13 +50,11 @@ mkdir -p "${BINARY_DIR}"
 # Compile from .mlir input to static C source files using host tools          #
 ###############################################################################
 
-COMPILE_TOOL="${INSTALL_ROOT}/bin/iree-compile"
-EMBED_DATA_TOOL="${INSTALL_ROOT}/bin/iree-c-embed-data"
 INPUT_NAME="mnist"
 INPUT_PATH="${ROOT_DIR}/samples/models/mnist.mlir"
 
 echo "=== Compiling MLIR to static library output (.vmfb, .h, .o) ==="
-"${COMPILE_TOOL}" "${INPUT_PATH}" \
+"${HOST_TOOLS_BINARY_DIR}/iree-compile" "${INPUT_PATH}" \
   --iree-input-type=stablehlo \
   --iree-hal-target-backends=llvm-cpu \
   --iree-llvmcpu-target-triple=wasm32-unknown-unknown \
@@ -72,7 +64,7 @@ echo "=== Compiling MLIR to static library output (.vmfb, .h, .o) ==="
   --o "${BINARY_DIR}/${INPUT_NAME}.vmfb"
 
 echo "=== Embedding bytecode module (.vmfb) into C source files (.h, .c) ==="
-"${EMBED_DATA_TOOL}" "${BINARY_DIR}/${INPUT_NAME}.vmfb" \
+"${HOST_TOOLS_BINARY_DIR}/iree-c-embed-data" "${BINARY_DIR}/${INPUT_NAME}.vmfb" \
   --output_header="${BINARY_DIR}/${INPUT_NAME}_bytecode.h" \
   --output_impl="${BINARY_DIR}/${INPUT_NAME}_bytecode.c" \
   --identifier="iree_static_${INPUT_NAME}" \
@@ -92,7 +84,7 @@ emcmake "${CMAKE_BIN}" \
   -DPython3_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE}" \
   -DPYTHON_EXECUTABLE="${IREE_PYTHON3_EXECUTABLE}" \
   -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-  -DIREE_HOST_BIN_DIR="${INSTALL_ROOT}/bin" \
+  -DIREE_HOST_BIN_DIR="${HOST_TOOLS_BINARY_DIR}" \
   -DIREE_BUILD_EXPERIMENTAL_WEB_SAMPLES=ON \
   -DIREE_HAL_DRIVER_DEFAULTS=OFF \
   -DIREE_HAL_DRIVER_LOCAL_SYNC=ON \

diff --git a/experimental/web/sample_webgpu/build_sample.sh b/experimental/web/sample_webgpu/build_sample.sh
@@ -9,19 +9,25 @@
 #
 # Prerequisites:
 #   * Environment must be configured for Emscripten
-#   * Host tools must be built (default at IREE_SOURCE_DIR/build-host/install).
-#     The build_tools/cmake/build_host_tools.sh script can do this for you.
+#   * Host tools must be available at the $1 arg
 #
-# Usage:
-#   build_sample.sh (optional install path) && serve_sample.sh
+# Sample usage:
+#   python -m venv .venv
+#   source .venv/bin/activate
+#   python -m pip install iree-compiler iree-runtime
+#   build_sample.sh .venv/bin && serve_sample.sh
+#
+# The build directory for the emscripten build is taken from the environment
+# variable IREE_EMPSCRIPTEN_BUILD_DIR, defaulting to "build-emscripten".
+# Designed for CI, but can be run manually. It reuses the build directory if it
+# already exists.
 
 set -euo pipefail
 
 ROOT_DIR=$(git rev-parse --show-toplevel)
 
-HOST_BUILD_DIR="${IREE_HOST_BUILD_DIR:-${ROOT_DIR}/build-host}"
+HOST_TOOLS_BINARY_DIR="$1"
 BUILD_DIR="${IREE_EMPSCRIPTEN_BUILD_DIR:-build-emscripten}"
-INSTALL_ROOT="$(realpath ${1:-${HOST_BUILD_DIR}/install})"
 SOURCE_DIR="${ROOT_DIR}/experimental/web/sample_webgpu"
 BINARY_DIR="${BUILD_DIR}/experimental/web/sample_webgpu"
 
@@ -44,12 +50,11 @@ mkdir -p ${BINARY_DIR}
 ###############################################################################
 
 echo "=== Compiling sample MLIR files to VM FlatBuffer outputs (.vmfb) ==="
-COMPILE_TOOL="${INSTALL_ROOT?}/bin/iree-compile"
 
 # TODO(#11321): Enable iree-codegen-gpu-native-math-precision by default?
 compile_sample() {
   echo "  Compiling '$1' sample for WebGPU..."
-  ${COMPILE_TOOL?} $3 \
+  "${HOST_TOOLS_BINARY_DIR}/iree-compile" $3 \
     --iree-input-type=$2 \
     --iree-hal-target-backends=webgpu-spirv \
     --iree-codegen-gpu-native-math-precision=true \
@@ -82,7 +87,7 @@ pushd ${BUILD_DIR}
 # Note: The sample creates a device directly, so no drivers are required.
 emcmake "${CMAKE_BIN?}" -G Ninja .. \
   -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-  -DIREE_HOST_BIN_DIR="${INSTALL_ROOT}/bin" \
+  -DIREE_HOST_BIN_DIR="${HOST_TOOLS_BINARY_DIR}" \
   -DIREE_BUILD_EXPERIMENTAL_WEB_SAMPLES=ON \
   -DIREE_ENABLE_THREADING=OFF \
   -DIREE_HAL_DRIVER_DEFAULTS=OFF \

diff --git a/experimental/web/testing/build_tests.sh b/experimental/web/testing/build_tests.sh
@@ -9,11 +9,13 @@
 #
 # Prerequisites:
 #   * Environment must be configured for Emscripten
-#   * Host tools must be built (default at IREE_SOURCE_DIR/build-host/install).
-#     The build_tools/cmake/build_host_tools.sh script can do this for you.
+#   * Host tools must be available at the $1 arg
 #
-# Usage:
-#   build_tests.sh (optional install path) && serve_tests.sh
+# Sample usage:
+#   python -m venv .venv
+#   source .venv/bin/activate
+#   python -m pip install iree-compiler iree-runtime
+#   build_tests.sh .venv/bin && serve_tests.sh
 
 set -e
 
@@ -37,7 +39,7 @@ mkdir -p ${BUILD_DIR}
 BINARY_DIR=${BUILD_DIR}/experimental/web/testing
 mkdir -p ${BINARY_DIR}
 
-INSTALL_ROOT="${1:-${ROOT_DIR}/build-host/install}"
+HOST_TOOLS_BINARY_DIR="$1"
 
 ###############################################################################
 # Build the web artifacts using Emscripten                                    #
@@ -50,7 +52,7 @@ pushd ${ROOT_DIR?}/build-emscripten > /dev/null
 # Configure using Emscripten's CMake wrapper, then build.
 emcmake "${CMAKE_BIN?}" -G Ninja .. \
     -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-    -DIREE_HOST_BIN_DIR="${INSTALL_ROOT}/bin" \
+    -DIREE_HOST_BIN_DIR="${HOST_TOOLS_BINARY_DIR}" \
     -DIREE_BUILD_COMPILER=OFF \
     -DIREE_HAL_DRIVER_DEFAULTS=OFF \
     -DIREE_EXTERNAL_HAL_DRIVERS=webgpu \