Merge branch 'develop' into bugfix/PRD-704-concurrent-runs-race-condi…

…tions
zenml-io · Nov 11, 2024 · d3b68c3 · d3b68c3
2 parents 1df5ecc + a5d4531
commit d3b68c3
Show file tree

Hide file tree

Showing 99 changed files with 1,345 additions and 1,146 deletions.
diff --git a/.github/actions/setup_environment/action.yml b/.github/actions/setup_environment/action.yml
@@ -30,7 +30,7 @@ runs:
   using: composite
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v5.0.0
+      uses: actions/setup-python@v5.3.0
       with:
         python-version: ${{ inputs.python-version }}
     - name: Delete error-causing bash

diff --git a/.github/workflows/ci-fast.yml b/.github/workflows/ci-fast.yml
@@ -16,12 +16,12 @@ concurrency:
 jobs:
   docstring-check:
     if: github.event.pull_request.draft == false
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.10'
       - name: Install darglint using uv
@@ -32,19 +32,19 @@ jobs:
       - name: Check docstrings
         run: bash scripts/docstring.sh
   sqlite-db-migration-testing-random:
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
     # if team member commented, not a draft, on a PR, using /fulltest
     if: github.event.pull_request.draft == false || github.event_name == 'workflow_dispatch'
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Test migrations across versions
@@ -54,9 +54,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Spelling checker
-        uses: crate-ci/typos@v1.17.0
+        uses: crate-ci/typos@v1.27.0
         with:
           files: .
           config: ./.typos.toml
@@ -65,9 +65,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.11'
       - name: Test API docs buildable
@@ -82,33 +82,46 @@ jobs:
       python-version: '3.9'
       os: ubuntu-latest
     secrets: inherit
-  custom-ubuntu-setup-and-unit-test:
+  linting:
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.11']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
+  ubuntu-setup-and-unit-test:
+    needs: linting
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
         # IMPORTANT: Since we are using the combination of `arc-runner-set`
         # and `3.10` in our `ci-fast` workflow, this combination has been
         # excluded from the `ci-slow` workflow. If you change the configuration
         # here, please adjust the configuration of `ci-slow` accordingly.
-        os: [arc-runner-set]
-        python-version: ['3.10']
+        os: [ubuntu-latest]
+        python-version: ['3.11']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
-  custom-arc-runner-set-integration-test:
+  ubuntu-latest-integration-test:
+    needs: [linting]
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        # IMPORTANT: Since we are using the combinations of `arc-runner-set`
-        # `3.10` and two different test environments in our `ci-fast` workflow,
-        # these combination have been excluded from the `ci-slow` workflow.
-        # If you change the configuration here, please adjust the configuration
-        # of `ci-slow` accordingly.
-        os: [arc-runner-set]
-        python-version: ['3.10']
+        # IMPORTANT: Since we are using the combination of `arc-runner-set`
+        # and `3.10` in our `ci-fast` workflow, this combination has been
+        # excluded from the `ci-slow` workflow. If you change the configuration
+        # here, please adjust the configuration of `ci-slow` accordingly.
+        os: [ubuntu-latest]
+        python-version: ['3.11']
         test_environment: [default, docker-server-docker-orchestrator-mysql]
       fail-fast: false
     uses: ./.github/workflows/integration-test-fast.yml

diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml
@@ -23,7 +23,7 @@ jobs:
       # With dynamic approach dev can set label and rerun this flow to make it running.
       - name: Get PR labels
         id: pr-labels
-        uses: actions/github-script@v5
+        uses: actions/github-script@v7.0.1
         with:
           script: |
             const prNumber = ${{ github.event.pull_request.number }};
@@ -45,13 +45,13 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -68,13 +68,13 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -87,18 +87,18 @@ jobs:
         run: bash scripts/test-migrations.sh mysql random
   sqlite-db-migration-testing-full:
     needs: run-slow-ci-label-is-set
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
     if: github.event.pull_request.draft == false
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Test migrations across versions
@@ -109,13 +109,13 @@ jobs:
     env:
       ZENML_ANALYTICS_OPT_IN: false
       ZENML_DEBUG: true
-    runs-on: arc-runner-set
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
         with:
           fetch-depth: 0
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.9'
       - name: Login to Docker Hub
@@ -131,9 +131,9 @@ jobs:
     needs: run-slow-ci-label-is-set
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.2.2
       - name: Set up Python
-        uses: actions/setup-python@v5.0.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: '3.11'
       - name: Install uv
@@ -171,28 +171,48 @@ jobs:
           bash scripts/check-alembic-branches.sh
       - name: Install latest dashboard (test gitignore)
         run: bash scripts/install-dashboard.sh
-  custom-ubuntu-unit-test:
+  ubuntu-linting:
+    needs: run-slow-ci-label-is-set
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.12']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
+  ubuntu-unit-test:
     if: github.event.pull_request.draft == false
+    needs: [run-slow-ci-label-is-set, ubuntu-linting]
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.12']
+      fail-fast: false
+    uses: ./.github/workflows/unit-test.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
+  windows-linting:
     needs: run-slow-ci-label-is-set
+    if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        os: [arc-runner-set]
+        os: [windows-latest]
         python-version: ['3.9', '3.10', '3.11', '3.12']
-        # IMPORTANT: Since we are using the following combination
-        # in our `ci-fast` workflow, this combination has been
-        # excluded from the `ci-slow` workflow.
-        exclude:
-          - os: arc-runner-set
-            python-version: '3.10'
       fail-fast: false
-    uses: ./.github/workflows/unit-test.yml
+    uses: ./.github/workflows/linting.yml
     with:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
   windows-unit-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, windows-linting]
     strategy:
       matrix:
         os: [windows-latest]
@@ -203,9 +223,22 @@ jobs:
       python-version: ${{ matrix.python-version }}
       os: ${{ matrix.os }}
     secrets: inherit
+  macos-linting:
+    needs: run-slow-ci-label-is-set
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [macos-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+      fail-fast: false
+    uses: ./.github/workflows/linting.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      os: ${{ matrix.os }}
+    secrets: inherit
   macos-unit-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, macos-linting]
     strategy:
       matrix:
         os: [macos-latest]
@@ -224,7 +257,7 @@ jobs:
     secrets: inherit
   windows-integration-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, windows-unit-test]
     strategy:
       matrix:
         os: [windows-latest]
@@ -239,7 +272,7 @@ jobs:
     secrets: inherit
   macos-integration-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, macos-unit-test]
     strategy:
       matrix:
         os: [macos-13]
@@ -258,40 +291,31 @@ jobs:
       python-version: ${{ matrix.python-version }}
       test_environment: ${{ matrix.test_environment }}
     secrets: inherit
-  custom-ubuntu-integration-test:
+  ubuntu-latest-integration-test:
     if: github.event.pull_request.draft == false
-    needs: run-slow-ci-label-is-set
+    needs: [run-slow-ci-label-is-set, ubuntu-unit-test]
     strategy:
       matrix:
-        os: [arc-runner-set]
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.10', '3.12']
         test_environment:
           - default
           - docker-server-docker-orchestrator-mysql
           - docker-server-docker-orchestrator-mariadb
         exclude:
           # docker is time-consuming to run, so we only run it on 3.9
           - test_environment: docker-server-docker-orchestrator-mysql
-            python-version: '3.10'
+            python-version: '3.9'
           - test_environment: docker-server-docker-orchestrator-mysql
-            python-version: '3.11'
+            python-version: '3.10'
           - test_environment: docker-server-docker-orchestrator-mysql
             python-version: '3.12'
           - test_environment: docker-server-docker-orchestrator-mariadb
-            python-version: '3.10'
+            python-version: '3.9'
           - test_environment: docker-server-docker-orchestrator-mariadb
-            python-version: '3.11'
+            python-version: '3.10'
           - test_environment: docker-server-docker-orchestrator-mariadb
             python-version: '3.12'
-          # IMPORTANT: Since we are using the following combinations
-          # in our `ci-fast` workflow, this combination has been
-          # excluded from the `ci-slow` workflow.
-          - os: arc-runner-set
-            test_environment: default
-            python-version: '3.10'
-          - os: arc-runner-set
-            test_environment: docker-server-docker-orchestrator-mysql
-            python-version: '3.10'
       fail-fast: false
     uses: ./.github/workflows/integration-test-slow.yml
     with:

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -23,7 +23,7 @@ jobs:
         language: [python]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4.1.1
+        uses: actions/checkout@v4.2.2
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with: