[ci] Move extract zip into ci_download.py (#3251)

* move extract zip into ci_download.py * use system unzip
taichi-dev · Oct 25, 2021 · f71246e · f71246e
1 parent fac2118
commit f71246e
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 44 deletions.
diff --git a/.github/workflows/postsubmit.yml b/.github/workflows/postsubmit.yml
@@ -18,18 +18,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
 
       - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
+        run: python misc/ci_download.py
         env:
           CI_PLATFORM: ${{ matrix.os }}
 
@@ -65,18 +61,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
 
       - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
+        run: python misc/ci_download.py
         env:
           CI_PLATFORM: ${{ matrix.os }}
 
@@ -103,7 +95,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Build
         run: |
@@ -128,14 +120,13 @@ jobs:
     runs-on: windows-latest
     timeout-minutes: 60
     steps:
-
       - name: Install 7Zip PowerShell
         shell: powershell
         run: Install-Module 7Zip4PowerShell -Force -Verbose
 
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
@@ -205,7 +196,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Build
         run: |

diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml
@@ -86,18 +86,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
 
       - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
+        run: python misc/ci_download.py
         env:
           CI_PLATFORM: ${{ matrix.os }}
 
@@ -135,18 +131,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
 
       - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
+        run: python misc/ci_download.py
         env:
           CI_PLATFORM: ${{ matrix.os }}
 
@@ -174,7 +166,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Build
         run: |
@@ -201,14 +193,13 @@ jobs:
     runs-on: windows-latest
     timeout-minutes: 90
     steps:
-
       - name: Install 7Zip PowerShell
         shell: powershell
         run: Install-Module 7Zip4PowerShell -Force -Verbose
 
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
@@ -279,7 +270,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Build
         run: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -39,7 +39,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Create Python Wheel
         run: |
@@ -114,18 +114,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
 
       - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
+        run: python misc/ci_download.py
         env:
           CI_PLATFORM: macos-latest
 
@@ -183,7 +179,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Build
         run: |
@@ -237,7 +233,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - name: Build
         run: |
@@ -292,7 +288,6 @@ jobs:
           if [ $PROJECT_NAME == "taichi-nightly" ]; then export PYPI_PWD="$NIGHT_PWD" && python3 build.py upload --skip_build --testpypi --project_name $PROJECT_NAME
           elif [ $PROJECT_NAME == "taichi" ]; then export PYPI_PWD="$PROD_PWD" && python3 build.py upload --skip_build; fi
 
-
   build_and_upload_windows:
     name: Build and Upload (Windows only)
     needs: matrix_prep
@@ -307,7 +302,7 @@ jobs:
 
       - uses: actions/checkout@v2
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
 
       - uses: actions/setup-python@v2
         with:

diff --git a/misc/ci_download.py b/misc/ci_download.py
@@ -1,4 +1,6 @@
 import os
+import sys
+import urllib.request
 
 platform = os.environ['CI_PLATFORM']
 if platform.startswith('macos'):
@@ -11,5 +13,12 @@
     raise Exception(f'Bad CI_PLATFORM={platform}')
 
 llvm_url = f'https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-{platform}.zip'
+target_dir = 'taichi-llvm'
 print(f'Downloading LLVM from {llvm_url}...')
-os.system(f'wget {llvm_url} --waitretry=3 --tries=5 -O taichi-llvm.zip')
+urllib.request.urlretrieve(llvm_url, "taichi-llvm.zip")
+print(f'Extract zip to local dir {target_dir}...')
+if not os.path.exists(target_dir):
+    os.makedirs(target_dir)
+
+retcode = os.system(f"unzip taichi-llvm.zip -d {target_dir}")
+sys.exit(retcode)