Add GitHub Actions config (#6112)

Summary: Per Omkar's advice, I am learning from the migration of [this](https://github.com/pytorch/audio/pull/3391/files#diff-78a8a19706dbd2a4425dd72bdab0502ed7a2cef16365ab7030a5a0588927bf47L244-L269) to [this](https://github.com/pytorch/audio/blob/main/.github/workflows/unittest-linux-cpu.yml) Pull Request resolved: #6112 Test Plan: Documentation: [Optional Fixes #issue] Please see a detailed explanation of how to fill out the fields in the relevant sections in PULL_REQUEST.md. Reviewed By: osalpekar Differential Revision: D48221045 Pulled By: openrichardfb fbshipit-source-id: 48e5e789e146dd2964701e2a2dcb8031513864f3
pytorch · Aug 10, 2023 · dcc53db · dcc53db
1 parent e208dd2
commit dcc53db
Show file tree

Hide file tree

Showing 3 changed files with 358 additions and 0 deletions.
diff --git a/.github/build.sh b/.github/build.sh
@@ -0,0 +1,176 @@
+#!/bin/bash
+
+# This script setup an working environemnt for running glow tests and gtest driver.
+# By default, we run with the enabled CPU backend and disabled OpenCL backend.
+set -ex
+
+# Add support for https apt sources.
+wget http://security.ubuntu.com/ubuntu/pool/main/a/apt/apt-transport-https_1.2.32ubuntu0.2_amd64.deb
+echo "93475e4cc5e7a86de63fea0316f3f2cd8b791cf4d6ea50a6d63f5bd8e1da5726  apt-transport-https_1.2.32ubuntu0.2_amd64.deb" | sha256sum -c
+sudo dpkg -i apt-transport-https_1.2.32ubuntu0.2_amd64.deb
+rm apt-transport-https_1.2.32ubuntu0.2_amd64.deb
+
+export MAX_JOBS=8
+if [ "${CIRCLE_JOB}" != "COVERAGE" ]; then
+    if hash sccache 2>/dev/null; then
+        SCCACHE_BIN_DIR="/tmp/sccache"
+        mkdir -p "$SCCACHE_BIN_DIR"
+        for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do
+            (
+                echo "#!/bin/sh"
+                echo "exec $(which sccache) $(which $compiler) \"\$@\""
+            ) > "$SCCACHE_BIN_DIR/$compiler"
+            chmod +x "$SCCACHE_BIN_DIR/$compiler"
+        done
+        export PATH="$SCCACHE_BIN_DIR:$PATH"
+    fi
+fi
+
+install_pocl() {
+   sudo apt-get install -y ocl-icd-opencl-dev clinfo libhwloc-dev opencl-headers
+
+   git clone https://github.com/pocl/pocl.git
+   cd pocl && git checkout 4efafa82c087b5e846a9f8083d46b3cdac2f698b && cd ../
+   mkdir build_pocl
+   cd build_pocl
+   cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/clang++-8 -DCMAKE_C_COMPILER=/usr/bin/clang-8 -DENABLE_ICD=ON ../pocl
+   make -j`nproc`
+   sudo make install
+
+   sudo mkdir -p /etc/OpenCL/vendors/
+   sudo cp /usr/local/etc/OpenCL/vendors/pocl.icd /etc/OpenCL/vendors/
+
+   clinfo
+   cd ../
+}
+
+install_fmt() {
+    git clone https://github.com/fmtlib/fmt --branch 7.1.3
+    pushd fmt
+    mkdir build
+    cd build
+    cmake ..
+    make -j`nproc`
+    sudo make install
+    popd
+}
+
+upgrade_python() {
+    echo "Removing old python...";
+    sudo apt-get remove --purge -y python3.6 python3-pip libpython3-dev
+    sudo apt-get autoremove -y
+
+    echo "Installing dependencies for new python..."
+    sudo apt-get update
+    sudo apt-get install -y libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev
+
+    echo "Installing new python..."
+    mkdir python-src
+    pushd python-src
+    wget https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tgz
+    tar xvf Python-3.9.0.tgz
+    cd Python-3.9.0
+    ./configure --enable-shared
+    sudo make altinstall
+    popd
+
+    echo "Adjusting system to recognize new python..."
+    sudo touch /etc/ld.so.conf.d/glowDevLibs.conf
+    echo "/usr/local/lib/" | sudo tee -a /etc/ld.so.conf.d/glowDevLibs.conf
+    sudo ldconfig
+    sudo rm /usr/local/bin/pip
+    sudo ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip
+
+    echo "Installing virtualenv..."
+    sudo pip3.9 install virtualenv
+}
+
+GLOW_DEPS="libpng-dev libgoogle-glog-dev libboost-all-dev libdouble-conversion-dev libgflags-dev libjemalloc-dev libpthread-stubs0-dev libevent-dev libssl-dev"
+
+if [ "${CIRCLE_JOB}" == "CHECK_CLANG_AND_PEP8_FORMAT" ]; then
+    sudo apt-get update
+    upgrade_python
+else
+    # Install Glow dependencies
+    sudo apt-get update
+
+    # Redirect clang
+    sudo ln -s /usr/bin/clang-8 /usr/bin/clang
+    sudo ln -s /usr/bin/clang++-8 /usr/bin/clang++
+    sudo ln -s /usr/bin/llvm-symbolizer-8 /usr/bin/llvm-symbolizer
+    sudo ln -s /usr/bin/llvm-config-8 /usr/bin/llvm-config-8.0
+
+    sudo apt-get install -y ${GLOW_DEPS}
+    install_fmt
+fi
+
+# Since we are using llvm-7 in these two branches, we cannot use pip install cmake
+if [ "${CIRCLE_JOB}" != "PYTORCH" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ]; then
+    sudo pip install cmake==3.17.3
+else
+    sudo apt-get install cmake
+fi
+
+# Install ninja, (newest version of) autopep8 through pip
+sudo pip install ninja
+hash cmake ninja
+
+# Build glow
+GLOW_DIR=$PWD
+cd ${GLOW_DIR}
+mkdir build && cd build
+
+CMAKE_ARGS=()
+
+CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-Werror")
+CMAKE_ARGS+=("-DGLOW_WITH_CPU=ON")
+CMAKE_ARGS+=("-DGLOW_WITH_HABANA=OFF")
+
+if [[ "${CIRCLE_JOB}" == "ASAN" ]]; then
+    CMAKE_ARGS+=("-DGLOW_USE_SANITIZER='Address;Undefined'")
+    CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=OFF")
+    CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
+elif [[ "$CIRCLE_JOB" == "COVERAGE" ]]; then
+    sudo apt-get install wget
+    sudo apt-get install -y lcov
+    sudo pip install awscli --upgrade
+    ../utils/install_protobuf.sh
+    CC=gcc-5 CXX=g++-5 cmake -G Ninja \
+          -DCMAKE_BUILD_TYPE=Debug -DGLOW_WITH_OPENCL=OFF -DGLOW_WITH_CPU=ON \
+          -DLLVM_DIR=/usr/lib/llvm-7/cmake \
+          -DGLOW_USE_COVERAGE=ON \
+          ../
+elif [[ "$CIRCLE_JOB" == "CHECK_CLANG_AND_PEP8_FORMAT" ]]; then
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+    sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main"
+    sudo apt-get update
+    sudo apt-get install -y clang-format-11
+    cd /tmp
+    python3.9 -m virtualenv venv
+    source venv/bin/activate
+    pip install black==22.3.0
+    cd ${GLOW_DIR}
+elif [[ "$CIRCLE_JOB" == "OPENCL" ]]; then
+    install_pocl
+    CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=ON")
+elif [[ "$CIRCLE_JOB" == "FEATURE_COMPILATION" ]]; then
+    CMAKE_ARGS+=("-DGLOW_USE_PNG_IF_REQUIRED=OFF")
+elif [[ "$CIRCLE_JOB" == "32B_DIM_T" ]]; then
+    install_pocl
+    CMAKE_ARGS+=("-DTENSOR_DIMS_32_BITS=ON -DGLOW_WITH_OPENCL=ON")
+else
+    CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
+    if [[ "${CIRCLE_JOB}" == "SHARED" ]]; then
+        CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=ON")
+    fi
+fi
+
+if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ] && [ "${CIRCLE_JOB}" != "PYTORCH" ]; then
+    cmake -GNinja ${CMAKE_ARGS[*]} ../
+    ninja
+fi
+
+# Report sccache hit/miss stats
+if hash sccache 2>/dev/null; then
+    sccache --show-stats
+fi
diff --git a/.github/test.sh b/.github/test.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# This script runs all tests in glow, including onnxifi gtests
+
+set -euxo pipefail
+
+export GLOW_SRC=$PWD
+export GLOW_BUILD_DIR=${GLOW_SRC}/build
+export LOADER=${GLOW_BUILD_DIR}/bin/image-classifier
+export LSAN_OPTIONS="suppressions=$GLOW_SRC/.circleci/lsan_suppressions.txt"
+export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
+export IMAGES_DIR=${GLOW_SRC}/tests/images/
+
+# Pass in which tests to run (one of {test, test_unopt}).
+run_unit_tests() {
+    CTEST_PARALLEL_LEVEL=4 GLOG_minloglevel=3 ninja "${1}" || ( cat Testing/Temporary/LastTest.log && exit 1 )
+}
+
+run_and_check_lenet_mnist_bundle() {
+    for q in "" "quantized_"
+    do
+      cd "${GLOW_BUILD_DIR}/bundles/${q}lenet_mnist/"
+      rm -f raw_results.txt
+      for f in ${IMAGES_DIR}/mnist/*
+      do
+        # Assume that there is only one file with this format (prepended with Quantized or not)
+        ./*LeNetMnistBundle ${f} | grep "Result: " >> raw_results.txt
+      done
+      diff raw_results.txt "${GLOW_SRC}/.ci/lenet_mnist_expected_output.txt"
+      cd -
+    done
+}
+
+run_and_check_resnet50_bundle() {
+    for q in "" "quantized_"
+    do
+      cd "${GLOW_BUILD_DIR}/bundles/${q}resnet50/"
+      rm -f raw_results.txt
+      for f in ${IMAGES_DIR}/imagenet/*
+      do
+        # Assume that there is only one file with this format (prepended with Quantized or not)
+        ./*ResNet50Bundle ${f} | grep "Result: " >> raw_results.txt
+      done
+      diff raw_results.txt "${GLOW_SRC}/.ci/resnet50_expected_output.txt"
+      cd -
+    done
+}
+
+run_and_check_bundle_instrument() {
+    cd "${GLOW_BUILD_DIR}/bundles/bundle_instrument/"
+    # Compare console output.
+    ./BundleInstrument ${IMAGES_DIR}/mnist/0_1009.png >> raw_results.txt
+    diff raw_results.txt "${GLOW_SRC}/.ci/bundle_instrument_expected_output.txt"
+    # Compare binary dumps between instrument-debug and instrument-ir.
+    for file in ./instrument-debug-data/*.bin
+    do
+      file_name=$(basename $file)
+      diff ./instrument-debug-data/${file_name} ./instrument-ir-data/${file_name}
+    done
+    cd -
+}
+
+run_and_check_bundle_with_multiple_entries() {
+    cd "${GLOW_BUILD_DIR}/bundles/bundle_with_multiple_entries/"
+    # Compare console output.
+    ./bundle_with_multiple_entries >> raw_results.txt
+    diff raw_results.txt "${GLOW_SRC}/.ci/bundle_with_multiple_entries_expected_output.txt"
+    cd -
+}
+
+run_and_check_bundle_with_extra_objects() {
+    cd "${GLOW_BUILD_DIR}/bundles/bundle_with_extra_objects/"
+    # Compare console output.
+    ./BundleWithExtraObjects >> raw_results.txt
+    diff raw_results.txt "${GLOW_SRC}/.ci/bundle_with_extra_objects_expected_output.txt"
+    cd -
+}
+
+run_and_check_bundle_tflite_custom() {
+    cd "${GLOW_BUILD_DIR}/bundles/bundle_tflite_custom/"
+    # Compare console output.
+    ./BundleTFLiteCustom >> raw_results.txt
+    diff raw_results.txt "${GLOW_SRC}/.ci/bundle_tflite_custom_expected_output.txt"
+    cd -
+}
+
+# Run unit tests and bundle tests.
+cd "${GLOW_BUILD_DIR}"
+case ${CIRCLE_JOB} in
+    ASAN)
+        run_unit_tests check
+        ;;
+    OPENCL)
+        run_unit_tests check
+        ;;
+    FEATURE_COMPILATION)
+        # FEATURE_COMPILATION is a compilation only CI job, thus tests
+        # are not requited.
+        ;;
+    DEBUG)
+        run_unit_tests check
+        run_unit_tests test_unopt
+        ;;
+    SHARED)
+        # No tests with shared libs; it's similar to DEBUG.
+        ;;
+    32B_DIM_T)
+        # A lot of 32b dim_t issues are not revealed at build time, thus
+        # run the unit test suite also.
+        run_unit_tests check
+        ;;
+    COVERAGE)
+        cd "${GLOW_SRC}"
+        cd build
+        ../.circleci/run_coverage.sh
+        ;;
+    CHECK_CLANG_AND_PEP8_FORMAT)
+        cd "${GLOW_SRC}"
+        sudo ln -s /usr/bin/clang-format-11 /usr/bin/clang-format
+        source /tmp/venv/bin/activate
+        ./utils/format.sh check
+        ;;
+    *)
+        echo "Error, '${CIRCLE_JOB}' not valid mode; Please, check .circleci/test.sh for list of supported tests."
+        exit 1
+        ;;
+esac
diff --git a/.github/workflows/glow-build.yml b/.github/workflows/glow-build.yml
@@ -0,0 +1,55 @@
+name: build glow
+
+on:
+  pull_request:
+  push:
+    branches:
+      - nightly
+      - main
+      - release/*
+  workflow_dispatch:
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        python_version: ["3.8"]
+      fail-fast: false
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      runner: linux.12xlarge
+      repository: pytorch/glow
+      timeout: 120
+      script: |
+        # echo '::group::Setup Environment Variables'
+        # Mark Build Directory Safe
+        # git config --global --add safe.directory /__w/audio/audio
+
+        # Set up Environment Variables
+        export PYTHON_VERSION="${{ matrix.python_version }}"
+
+        # Set CHANNEL
+        if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
+          export UPLOAD_CHANNEL=test
+        else
+          export UPLOAD_CHANNEL=nightly
+        fi
+
+        original_pwd=$(pwd)
+        sudo apt-get update;
+        mkdir ~/tempdownload;
+        cd ~/tempdownload;
+        wget https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-8.1p1.tar.gz;
+        tar zxvf openssh-8.1p1.tar.gz;
+        cd openssh-8.1p1 && ./configure && make && sudo make install;
+
+        cd $original_pwd
+        git submodule sync
+        git submodule update --recursive --init
+
+        set -e
+        # Build
+        .github/build.sh
+
+        # Test
+        .github/test.sh