From 8c1d31acf651fb7c58ce46935e72038d80a53d5d Mon Sep 17 00:00:00 2001 From: Bo Qiao Date: Sun, 8 May 2022 10:50:38 +0800 Subject: [PATCH] bisect at e583b2d4a --- .github/workflows/performance_monitoring.yml | 33 -- .../workflows/scripts/win_build_test_cpu.ps1 | 65 ++++ .github/workflows/testing.yml | 332 ++---------------- 3 files changed, 89 insertions(+), 341 deletions(-) delete mode 100644 .github/workflows/performance_monitoring.yml create mode 100644 .github/workflows/scripts/win_build_test_cpu.ps1 diff --git a/.github/workflows/performance_monitoring.yml b/.github/workflows/performance_monitoring.yml deleted file mode 100644 index e2bbded40..000000000 --- a/.github/workflows/performance_monitoring.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Performance Monitoring -on: - push: - branches: - - master - -jobs: - gpu_backends: - name: Performance monitoring (NVIDIA GPU) - timeout-minutes: 60 - # Disable this workflow on forks - if: github.repository_owner == 'taichi-dev' - runs-on: [self-hosted, x64, cuda, linux, benchmark] - steps: - - uses: actions/checkout@v2 - with: - submodules: "recursive" - - - name: Build & Install - run: | - .github/workflows/scripts/unix_build.sh - python3 -m pip install dist/*.whl - - - name: Run performance-monitoring - run: | - cd .. - rm -rf performance-monitoring - git clone git@github.com:taichi-dev/performance-monitoring.git - cd performance-monitoring - export WORKFLOW_MODE=postsubmit - ./run.sh - env: - GITHUB_CONTEXT: ${{ toJson(github) }} diff --git a/.github/workflows/scripts/win_build_test_cpu.ps1 b/.github/workflows/scripts/win_build_test_cpu.ps1 new file mode 100644 index 000000000..3a2bed81c --- /dev/null +++ b/.github/workflows/scripts/win_build_test_cpu.ps1 @@ -0,0 +1,65 @@ +# Build script for windows + +param ( + [switch]$develop = $false, + [switch]$install = $false, + [string]$libsDir = "C:\" +) + +$ErrorActionPreference = "Stop" + +function WriteInfo($text) { + Write-Host -ForegroundColor Green "[BUILD] $text" +} + +$libsDir = (Resolve-Path $libsDir).Path +if (-not (Test-Path $libsDir)) { + New-Item -ItemType Directory -Path $libsDir +} +Push-Location $libsDir + +if (-not (Test-Path "taichi_llvm")) { + WriteInfo("Download and extract LLVM") + curl.exe --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO + 7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm +} +if (-not (Test-Path "taichi_clang")) { + WriteInfo("Download and extract Clang") + curl.exe --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO + 7z x clang-10.0.0-win.zip -otaichi_clang +} + +WriteInfo("Setting the env vars") + +$env:LLVM_DIR = "C://taichi_llvm" +$env:TAICHI_CMAKE_ARGS = "-DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_BUILD_TESTS:BOOL=OFF" + +#TODO: For now we have to hard code the compiler path from build tools 2019 +$env:TAICHI_CMAKE_ARGS +=' -DCMAKE_CXX_COMPILER=C:/Program\ Files\ (x86)/Microsoft\ Visual\ Studio/2019/BuildTools/vc/Tools/Llvm/x64/bin/clang++.exe -DCMAKE_C_COMPILER=C:/Program\ Files\ (x86)/Microsoft\ Visual\ Studio/2019/BuildTools/vc/Tools/Llvm/x64/bin/clang.exe' +$env:TAICHI_CMAKE_ARGS += " -DCLANG_EXECUTABLE=C:\\taichi_clang\\bin\\clang++.exe" +$env:TAICHI_CMAKE_ARGS += " -DLLVM_AS_EXECUTABLE=C:\\taichi_llvm\\bin\\llvm-as.exe -DTI_WITH_VULKAN:BOOL=OFF" + +Pop-Location +clang --version + +WriteInfo("Enter the repository") +Set-Location .\test_actions + +WriteInfo("Setting up Python environment") +conda activate py37 + +python -m pip install numpy +python -m pip install wheel +python -m pip install -r requirements_dev.txt +python -m pip install -r requirements_test.txt +if (-not $?) { exit 1 } + +WriteInfo("Building Taichi") +python setup.py develop +if (-not $?) { exit 1 } +WriteInfo("Build finished") + + +WriteInfo("Testing Taichi") +python tests/run_tests.py -vr2 -t2 -k "not torch and not paddle" -a cpu +WriteInfo("Test finished") diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a34de5b85..56095a9e0 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -57,31 +57,6 @@ jobs: echo "::set-output name=run_job::false" fi - check_static_analyzer: - name: Check Static Analyzer - runs-on: ubuntu-latest - needs: check_files - steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - - name: clang-tidy - run: | - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - # https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio - echo $CR_PAT | docker login ghcr.io -u ${{ github.actor }} --password-stdin - docker pull ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.3.0 - docker run -id --user dev --name check_clang_tidy ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.3.0 /bin/bash - tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - check_clang_tidy:/home/dev/ - docker exec --user root check_clang_tidy apt install -y clang-tidy-10 - docker exec --user dev check_clang_tidy /home/dev/taichi/.github/workflows/scripts/check_clang_tidy.sh "$CI_SETUP_CMAKE_ARGS" - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - CI_SETUP_CMAKE_ARGS: -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=OFF - build_and_test_cpu_linux: name: Build and Test linux (CPU) needs: check_files @@ -93,10 +68,6 @@ jobs: python: py39 with_cc: ON wanted_archs: 'cpu,cc' - - os: ubuntu-latest - python: py310 - with_cc: ON - wanted_archs: 'cpu,cc' runs-on: ${{ matrix.os }} permissions: packages: read @@ -134,16 +105,16 @@ jobs: docker create --user dev --name taichi_build \ -e PY -e PROJECT_NAME -e TAICHI_CMAKE_ARGS \ ghcr.io/taichi-dev/taichidev-cpu-ubuntu18.04:v0.3.0 \ - /home/dev/taichi/.github/workflows/scripts/unix_build.sh + /home/dev/test_actions/.github/workflows/scripts/unix_build.sh # A tarball is needed because sccache needs some permissions that only the file owner has. # 1000 is the uid and gid of user "dev" in the container. # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line. tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/ docker start -a taichi_build rm -rf sccache_cache - docker cp taichi_build:/home/dev/taichi/sccache_cache sccache_cache - docker cp taichi_build:/home/dev/taichi/dist shared/dist - docker cp taichi_build:/home/dev/taichi/build shared/build + docker cp taichi_build:/home/dev/test_actions/sccache_cache sccache_cache + docker cp taichi_build:/home/dev/test_actions/dist shared/dist + docker cp taichi_build:/home/dev/test_actions/build shared/build env: PY: ${{ matrix.python }} PROJECT_NAME: taichi @@ -171,301 +142,46 @@ jobs: run: | docker rm taichi_build taichi_test -f - build_and_test_cpu_mac: - name: Build and Test macos (CPU) - needs: check_files - timeout-minutes: 60 - strategy: - matrix: - include: - - os: macos-10.15 - python: 3.7 - with_cc: OFF - with_cpp_tests: ON - wanted_archs: 'cpu' - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Get sccache cache - uses: actions/cache@v2 - with: - path: sccache_cache - key: sccache-mac-${{ github.sha }} - restore-keys: | - sccache-mac- - - - name: Download Pre-Built LLVM 10.0.0 - run: | - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - python misc/ci_download.py - env: - CI_PLATFORM: ${{ matrix.os }} - - - name: Build & Install - run: | - brew install molten-vk - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - mkdir -p sccache_cache - export PATH=`pwd`/taichi-llvm/bin/:$PATH - .github/workflows/scripts/unix_build.sh - brew uninstall molten-vk - env: - TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }} -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - CXX: clang++ - # [DEBUG] Copy this step around to enable debugging inside Github Action instances. - #- name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # limit-access-to-actor: true - - - name: Test - run: | - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - .github/workflows/scripts/unix_test.sh - env: - TI_WANTED_ARCHS: ${{ matrix.wanted_archs }} - TI_CI: 1 - - build_and_test_gpu_linux: - name: Build and Test (GPU) - needs: check_files - runs-on: [self-hosted, cuda, vulkan, cn] - timeout-minutes: 60 - steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - - name: Get sccache cache - uses: actions/cache@v2 - with: - path: sccache_cache - key: sccache-linux-gpu-${{ github.sha }} - restore-keys: | - sccache-linux-gpu- - - - name: Build & Install - run: | - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - mkdir -m777 shared - docker create --user dev --name taichi_build --gpus all -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e PY -e GPU_BUILD -e PROJECT_NAME -e TAICHI_CMAKE_ARGS -e DISPLAY \ - registry.taichigraphics.com/taichidev-ubuntu18.04:v0.3.0 \ - /home/dev/taichi/.github/workflows/scripts/unix_build.sh - # A tarball is needed because sccache needs some permissions that only the file owner has. - # 1000 is the uid and gid of user "dev" in the container. - # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line. - tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build:/home/dev/ - docker start -a taichi_build - rm -rf sccache_cache - docker cp taichi_build:/home/dev/taichi/sccache_cache sccache_cache - docker cp taichi_build:/home/dev/taichi/dist shared/dist - docker cp taichi_build:/home/dev/taichi/build shared/build - env: - PY: py38 - GPU_BUILD: ON - PROJECT_NAME: taichi - TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - DISPLAY: :1 - - - name: Test - run: | - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - docker create --user dev --name taichi_test --gpus all -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e DISPLAY -e PY -e GPU_TEST -e TI_WANTED_ARCHS \ - registry.taichigraphics.com/taichidev-ubuntu18.04:v0.3.0 \ - /home/dev/unix_test.sh - docker cp .github/workflows/scripts/unix_test.sh taichi_test:/home/dev/unix_test.sh - docker cp shared/dist/ taichi_test:/home/dev/ - docker cp shared/build/ taichi_test:/home/dev/ - docker cp pyproject.toml taichi_test:/home/dev/ - docker cp tests/ taichi_test:/home/dev/ - docker cp requirements_test.txt taichi_test:/home/dev/requirements_test.txt - docker start -a taichi_test - env: - PY: py38 - GPU_TEST: ON - DISPLAY: :1 - TI_WANTED_ARCHS: 'cpu,cuda,vulkan,opengl' - TI_DEVICE_MEMORY_GB: '0.7' - - - name: clean docker container - if: always() - run: | - docker rm taichi_build taichi_test -f - - build_and_test_windows: - name: Build and Test Windows - needs: check_files - runs-on: [self-hosted, windows, gpu] - timeout-minutes: 90 - steps: - # See also https://github.com/taichi-dev/taichi/issues/4161 - - name: Cleanup - shell: powershell - run: | - remove-item '${{ github.workspace }}\*' -recurse -force - - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Add Visual Studio Shell to ENV - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - - name: Get sccache cache - uses: actions/cache@v2 - with: - path: ccache_cache - key: ccache-win64-${{ github.sha }} - restore-keys: | - ccache-win64- - - - name: Build - shell: powershell - if: ${{ needs.check_files.outputs.run_job != 'false' }} - run: | - .\.github\workflows\scripts\win_build.ps1 -installVulkan -install -libsDir C:\ - - - name: Test - shell: powershell - if: ${{ needs.check_files.outputs.run_job != 'false' }} - run: | - .\.github\workflows\scripts\win_test.ps1 - env: - TI_WANTED_ARCHS: cpu,cuda,opengl - TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF - TI_SKIP_VERSION_CHECK: ON - TI_CI: 1 - PYTHON: '3.7' - TI_DEVICE_MEMORY_GB: '0.7' - - build_and_test_m1: - name: Build and Test (Apple M1) + build_and_test_cpu_windows: + name: Build and Test Windows (CPU) needs: check_files timeout-minutes: 60 - strategy: - matrix: - include: - - os: macos-latest - python: 3.8 - defaults: - run: - # https://github.com/actions/runner/issues/805#issuecomment-844426478 - shell: '/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}' - runs-on: [self-hosted, m1] + runs-on: windows-2019 + permissions: + packages: read + contents: read steps: - uses: actions/checkout@v2 with: submodules: 'recursive' - - name: Get sccache cache - uses: actions/cache@v2 - with: - path: sccache_cache - key: sccache-m1-${{ github.sha }} - restore-keys: | - sccache-m1- - - - name: Build + - name: Get docker images + shell: bash run: | if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then exit 0 fi - export PATH=/Users/github/miniforge3/envs/$PY/bin:$PATH - brew install molten-vk - .github/workflows/scripts/unix_build.sh + echo $CR_PAT | docker login ghcr.io -u ${{ github.actor }} --password-stdin + docker pull ghcr.io/taichi-dev/taichidev-cpu-windows:v0.0.1 env: - TAICHI_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - PY: ${{ matrix.python }} - CXX: clang++ + CR_PAT: ${{ secrets.GITHUB_TOKEN }} - - name: Test + - name: Build and Test + shell: bash run: | if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then exit 0 fi - export PATH=/Users/github/miniforge3/envs/$PY/bin:$PATH - .github/workflows/scripts/unix_test.sh - env: - TI_WANTED_ARCHS: 'metal,vulkan,cpu' - PY: ${{ matrix.python }} - PLATFORM: 'm1' - TI_CI: 1 - build_libtaichi_export: - name: Build libtaichi_export.so(GPU) - needs: check_files - runs-on: [self-hosted, cuda, vulkan, cn] - timeout-minutes: 60 - strategy: - matrix: - include: - - os: ubuntu-latest - python: py39 - with_cc: ON - permissions: - packages: read - contents: read - steps: - - uses: actions/checkout@v2 - with: - submodules: "recursive" - - - name: Get sccache cache - uses: actions/cache@v2 - with: - path: sccache_cache - key: sccache-linux-${{matrix.with_cc}}-${{ github.sha }} - restore-keys: | - sccache-linux-${{matrix.with_cc}}- - - - name: Build For Desktop - run: | - if [[ ${{needs.check_files.outputs.run_job}} == false ]]; then - exit 0 - fi - docker create --user dev --name taichi_build_desktop --gpus all -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e PY -e GPU_BUILD -e PROJECT_NAME -e TAICHI_CMAKE_ARGS -e DISPLAY -e EXPORT_CORE\ - registry.taichigraphics.com/taichidev-ubuntu18.04:v0.3.0 \ - /home/dev/taichi/.github/workflows/scripts/unix_build.sh - # A tarball is needed because sccache needs some permissions that only the file owner has. - # 1000 is the uid and gid of user "dev" in the container. - # If the uid or gid of the user inside the docker changes, please change the uid and gid in the following line. - tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx --owner 1000 --group 1000 | docker cp - taichi_build_desktop:/home/dev/ - docker start -a taichi_build_desktop - env: - PY: ${{ matrix.python }} - GPU_BUILD: ON - PROJECT_NAME: taichi - TAICHI_CMAKE_ARGS: -DTI_WITH_VULKAN:BOOL=ON -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_LLVM:BOOL=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DTI_EXPORT_CORE:BOOL=ON - EXPORT_CORE: 1 - DISPLAY: :1 + docker create --name taichi_build_test \ + ghcr.io/taichi-dev/taichidev-cpu-windows:v0.0.1 \ + C:/test_actions/.github/workflows/scripts/win_build_test_cpu.ps1 + tar -cf - ../${{ github.event.repository.name }} --mode u=+rwx,g=+rwx,o=+rwx | docker cp - taichi_build_test:C:/ + docker start -a taichi_build_test - name: clean docker container + shell: bash if: always() run: | - docker rm taichi_build_desktop -f + docker rm taichi_build_test -f