Feature/backport_ea1_ops

.github/dependabot.yml

@@ -0,0 +1,15 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gitsubmodule"
+    directory: "/"
+    allow:
+      - dependency-name: "externals/llvm-project"
+    schedule:
+      interval: "daily"
+      time: "06:00"
+      timezone: "Europe/Berlin"

.github/workflows/RollPyTorch.yml

@@ -53,19 +53,19 @@ jobs:
           sudo apt-get install unzip
 
           # Fetch the most recent nightly torchvision release
-          VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
+          VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torchvision/ --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
           echo "Found torchvision release ${VISION_RELEASE}"
 
           # Fetch the whl file associated with the nightly torchvision release
           rm -f torch*.whl
-          python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torchvision==${VISION_RELEASE}"
+          python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torchvision/ --pre "torchvision==${VISION_RELEASE}"
 
           # Downloading the torchvision WHL also downloads the PyTorch WHL file
           # Read the version from the downloaded whl file without extracting it
           PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/')
           echo "Found torch release ${PT_RELEASE}"
-          printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
-          printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt
+          printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
+          printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torchvision\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt
 
           # Read the commit hash from the downloaded whl file without extracting it
           PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | tail -1 | awk '{ print $3 }' | tr -d "'")

.github/workflows/approve_dependabot.yml

@@ -0,0 +1,28 @@
+name: Dependabot auto-approve & auto-merge
+on: pull_request
+
+permissions:
+  pull-requests: write
+  # Needed to enable auto-merge
+  contents: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: github.actor == 'dependabot[bot]'
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v2
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GH_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/buildRelease.yml

@@ -14,11 +14,16 @@ on:
 jobs:
   build_linux:
     name: Manylinux x86_64 Build
-    runs-on: a100
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
     strategy:
       matrix:
         package: [torch-mlir]
-        py_version: [cp38-cp38, cp311-cp311]
+        py_version: [cp38-cp38, cp310-cp310] # cp311-cp311
+        torch-version: [stable] # nightly
 
     steps:
       - name: Prepare workspace
@@ -38,19 +43,22 @@ jobs:
           cache-enabled: 'false'
       - name: Build Python wheels and smoke test.
         run: |
-          cd $GITHUB_WORKSPACE
-          TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }}
-          printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version
-          TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} ./build_tools/python_deploy/build_linux_packages.sh
-
+            cd $GITHUB_WORKSPACE
+            TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }}
+            printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version
+            TM_SKIP_TESTS=ON \
+            TM_PYTHON_VERSIONS=${{ matrix.py_version }} \
+            TM_PACKAGES=${{ matrix.package }} \
+            TM_TORCH_VERSION="${{ matrix.torch-version }}" \
+            ./build_tools/python_deploy/build_linux_packages.sh
       # If we were given a release_id, then upload the package we just built
       # to the github releases page.
       - name: Upload Release Assets (if requested)
         if: github.event.inputs.release_id != ''
         id: upload-release-assets
         uses: dwenegar/upload-release-assets@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
           assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl
@@ -61,7 +69,7 @@ jobs:
         id: publish_release
         uses: eregon/publish-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
       - name: Create dist directory
@@ -81,6 +89,7 @@ jobs:
           path: dist
 
   build_linux_arm64:
+    if: false
     name: Manylinux arm64 Build
     runs-on: linux-arm64
     strategy:
@@ -118,7 +127,7 @@ jobs:
         id: upload-release-assets
         uses: dwenegar/upload-release-assets@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
           assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl
@@ -129,7 +138,7 @@ jobs:
         id: publish_release
         uses: eregon/publish-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
       - name: Create dist directory
@@ -149,6 +158,7 @@ jobs:
           path: dist
 
   build_macos:
+    if: false
     name: MacOS Build
     runs-on: macos-latest
     strategy:
@@ -178,7 +188,7 @@ jobs:
         id: upload-release-assets
         uses: dwenegar/upload-release-assets@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
           assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl
@@ -189,7 +199,7 @@ jobs:
         id: publish_release
         uses: eregon/publish-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
       - name: Create dist directory
@@ -209,6 +219,7 @@ jobs:
           path: dist
 
   build_windows:
+    if: false
     name: Windows Build
     runs-on: windows-latest
     strategy:
@@ -241,7 +252,7 @@ jobs:
         id: upload-release-assets
         uses: dwenegar/upload-release-assets@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
           assets_path: ./wheelhouse/torch*.whl
@@ -252,7 +263,7 @@ jobs:
         id: publish_release
         uses: eregon/publish-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ github.event.inputs.release_id }}
       - name: Create dist directory
@@ -274,11 +285,15 @@ jobs:
 
   publish_releases:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
     needs:
       - build_linux
-      - build_linux_arm64
-      - build_macos
-      - build_windows
+      #- build_linux_arm64
+      #- build_macos
+      #- build_windows
 
     # Publish even if one of the builds failed
     if: ${{ always() }}
@@ -288,7 +303,7 @@ jobs:
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: Publish releases page
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       # Wheels must be published from a linux environment.
       #

.github/workflows/ci.yml

@@ -5,9 +5,8 @@ on:
   workflow_dispatch:
   workflow_call:
   pull_request:
-    branches: [main]
   push:
-    branches: [main]
+    branches: [main, feature/*]
 
 concurrency:
   # A PR number if a pull request and otherwise the commit hash. This cancels
@@ -19,11 +18,12 @@ concurrency:
 jobs:
   build-test-linux:
     strategy:
-      fail-fast: true
+      # AMD: Disable fail-fast to see whether failures are different between stable & nightly
+      fail-fast: false
       matrix:
         torch-version: [nightly, stable]
     name: Build and Test (Linux, torch-${{ matrix.torch-version }}, assertions)
-    runs-on: torch-mlir-cpubuilder-manylinux-x86-64
+    runs-on: ubuntu-latest
     env:
       CACHE_DIR: ${{ github.workspace }}/.container-cache
     steps:
@@ -42,6 +42,11 @@ jobs:
         with:
           submodules: true
 
+      - name: Runner setup
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ccache clang
+
       - name: Enable cache
         uses: actions/cache/restore@v3
         with:

.github/workflows/gh-pages-releases.yml

@@ -9,9 +9,11 @@ jobs:
   scrape_and_publish_releases:
     name: "Scrape and publish releases"
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
 
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    if: github.repository == 'xilinx/torch-mlir'
 
     steps:
       - name: Prepare workspace
@@ -21,10 +23,8 @@ jobs:
           sudo rm -rf $GITHUB_WORKSPACE/*
       - name: Checking out repository
         uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
       - name: Run scrape releases script
-        run: python ./build_tools/scrape_releases.py llvm torch-mlir > /tmp/index.html
+        run: python ./build_tools/scrape_releases.py xilinx torch-mlir > /tmp/index.html
         shell: bash
       - run: git fetch --all
       - run: git switch github-pages

.github/workflows/oneshotSnapshotPackage.yml

@@ -9,18 +9,19 @@ jobs:
     name: "Tag snapshot release"
     runs-on: ubuntu-latest
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    #if: github.repository == 'llvm/torch-mlir'
     steps:
       - name: Prepare workspace
         run: |
           # Clear the workspace directory so that we don't run into errors about
           # existing lock files.
           sudo rm -rf $GITHUB_WORKSPACE/*
 
-      - name: Checking out repository
+      - name: Checkout torch-mlir
         uses: actions/checkout@v3
         with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+            submodules: 'true'
+            fetch-depth: 0
 
       - name: Compute version
         run: |

.github/workflows/releaseSnapshotPackage.yml

@@ -11,7 +11,12 @@ jobs:
     name: "Tag snapshot release"
     runs-on: ubuntu-latest
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    #if: github.repository == 'llvm/torch-mlir'
+    permissions:
+      contents: write
+      actions: write
+    env:
+        BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
     steps:
 
       - name: Prepare workspace
@@ -22,8 +27,6 @@ jobs:
 
       - name: Checking out repository
         uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
 
       - name: Compute version
         run: |
@@ -40,15 +43,15 @@ jobs:
       - name: Pushing changes
         uses: ad-m/[email protected]
         with:
-          github_token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-          branch: main
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ env.BRANCH_NAME }}
           tags: true
 
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
         with:
           tag_name: ${{ env.tag_name }}
           release_name: torch-mlir snapshot ${{ env.tag_name }}
@@ -57,17 +60,30 @@ jobs:
           draft: true
           prerelease: false
 
-      - name: "Invoke workflow :: Build and Test"
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: Build and Test
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-          ref: "${{ env.tag_name }}"
+      # - name: "Invoke workflow :: Build and Test"
+      #   uses: benc-uk/workflow-dispatch@v1
+      #   with:
+      #     workflow: Build and Test
+      #     ref: "${{ env.tag_name }}"
 
       - name: "Invoke workflow :: Release Build"
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: Release Build
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
           ref: "${{ env.tag_name }}"
           inputs: '{"release_id": "${{ steps.create_release.outputs.id }}", "python_package_version": "${{ env.package_version }}"}'
+
+      - name: Download nightly pytorch and torchvision wheels
+        run: |
+          pip download -r pytorch-requirements.txt -r torchvision-requirements.txt --no-deps --dest deps --python-version 3.8
+          pip download -r pytorch-requirements.txt -r torchvision-requirements.txt --no-deps --dest deps --python-version 3.10
+          pip download -r pytorch-requirements.txt -r torchvision-requirements.txt --no-deps --dest deps --python-version 3.11
+
+      - name: Upload nightly pytorch and torchvision wheels into release
+        id: upload-release-assets-nightly
+        uses: dwenegar/upload-release-assets@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
+        with:
+          release_id: ${{ steps.create_release.outputs.id }}
+          assets_path: ./deps/*.whl