Remove breeze-python-version input and propagating it everywhere (#38227

) The `breeze-python-version` has been used in a number of places where we wanted to make sure of reproducibility of prepared artifacts (because Python 3.8 produces different tar files). However that caused some problems with caching of breeze environments - where some of them were using Python 3.8 and some were using Python 3.9. This PR fixes it by: * making Python 3.9 default for all breeze installations in CI * removing the breeze-python-version parameter and all its usages from all workflows The tests of breeze (ib basic-tests.yml) are still run with the default-python-version not with Python 3.9 (which currently is Python 3.8) - so the risk that breeze will not work on Python 3.8 is low. This will also synchronize itself in October, when Python 3.8 will reach end-of-life and we will move to Python 3.9 as the default version.
apache · Mar 17, 2024 · 8ae9331 · 8ae9331
1 parent a0dd026
commit 8ae9331
Show file tree

Hide file tree

Showing 18 changed files with 9 additions and 147 deletions.
diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml
@@ -21,7 +21,10 @@ description: 'Sets up Python and Breeze'
 inputs:
   python-version:
     description: 'Python version to use'
-    default: "3.8"
+    # Version of Python used for reproducibility of the packages built
+    # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
+    # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
+    default: "3.9"
 outputs:
   host-python-version:
     description: Python version used in host
@@ -34,7 +37,7 @@ runs:
       with:
         python-version: ${{ inputs.python-version }}
         cache: 'pip'
-        cache-dependency-path: ./dev/breeze/setup*
+        cache-dependency-path: ./dev/breeze/pyproject.toml
     - name: Cache breeze
       uses: actions/cache@v4
       with:

diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml
@@ -19,9 +19,6 @@
 name: 'Prepare breeze && current python image'
 description: 'Installs breeze and pulls current python image'
 inputs:
-  python-version:
-    description: 'Python version to use'
-    default: "3.8"
   pull-image-type:
     description: 'Which image to pull'
     default: CI
@@ -35,8 +32,6 @@ runs:
     - name: "Install Breeze"
       uses: ./.github/actions/breeze
       id: breeze
-      with:
-        python-version: ${{ inputs.python-version }}
     - name: Login to ghcr.io
       shell: bash
       run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml
@@ -57,11 +57,6 @@ on:  # yamllint disable-line rule:truthy
         description: Which version of python should be used get CI image
         required: true
         type: string
-      breeze-python-version:
-        description: >
-          Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
-        required: true
-        type: string
 jobs:
   prod-image-extra-checks-main:
     name: PROD image extra checks (main)
@@ -72,7 +67,6 @@ jobs:
       image-tag: ${{ inputs.image-tag }}
       build-provider-packages: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      breeze-python-version: ${{ inputs.breeze-python-version }}
       chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}
@@ -87,7 +81,6 @@ jobs:
       image-tag: ${{ inputs.image-tag }}
       build-provider-packages: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      breeze-python-version: ${{ inputs.breeze-python-version }}
       chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}

diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
@@ -40,11 +40,6 @@ on:  # yamllint disable-line rule:truthy
         description: Which version of python should be used get CI image
         required: true
         type: string
-      breeze-python-version:
-        description: >
-          Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
-        required: true
-        type: string
       canary-run:
         description: "Whether to run canary tests (true/false)"
         required: true
@@ -73,7 +68,7 @@ jobs:
         with:
           python-version: "${{ inputs.default-python-version }}"
           cache: 'pip'
-          cache-dependency-path: ./dev/breeze/setup*
+          cache-dependency-path: ./dev/breeze/pyproject.toml
       - run: pip install --editable ./dev/breeze/
       - run: python -m pytest -n auto --color=yes
         working-directory: ./dev/breeze/
@@ -132,8 +127,6 @@ jobs:
           path: ./airflow-client-python
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          python-version: ${{ inputs.breeze-python-version }}
       - name: "Generate client with breeze"
         run: >
           breeze release-management prepare-python-client --package-format both

diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -55,11 +55,6 @@ jobs:
     env:
       TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
     outputs:
-      # Version of Python used for reproducibility of the packages built
-      # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
-      # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
-      # TODO: remove me when we switch to Python 3.9 as minimal version
-      breeze-python-version: "3.9"
       image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
       python-versions: ${{ steps.selective-checks.outputs.python-versions }}
       upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
@@ -317,8 +312,6 @@ jobs:
         uses: ./.github/actions/cleanup-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         with:

diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml
@@ -55,11 +55,6 @@ on:  # yamllint disable-line rule:truthy
         description: "JSON-formatted array of Python versions to build images from"
         required: true
         type: string
-      breeze-python-version:
-        description: >
-          Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
-        required: true
-        type: string
 jobs:
   prepare-install-verify-provider-packages-wheel:
     timeout-minutes: 80
@@ -83,8 +78,6 @@ jobs:
       - name: >
           Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}
         uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          python-version: ${{ inputs.breeze-python-version }}
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*
       - name: "Prepare provider documentation"
@@ -151,8 +144,6 @@ jobs:
       - name: >
           Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}
         uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          python-version: ${{ inputs.breeze-python-version }}
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*
       - name: "Prepare provider packages: sdist"
@@ -223,8 +214,6 @@ jobs:
         uses: ./.github/actions/cleanup-docker
       - name: "Prepare breeze & CI image: ${{ matrix.python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
-        with:
-          python-version: ${{ inputs.breeze-python-version }}
       - name: "Cleanup dist files"
         run: rm -fv ./dist/*
       - name: "Prepare provider packages: wheel"

diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
@@ -74,11 +74,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)"
         required: true
         type: string
-      breeze-python-version:
-        description: >
-          Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
-        required: true
-        type: string
       constraints-branch:
         description: "Branch used to construct constraints URL from."
         required: true
@@ -122,8 +117,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
         if: inputs.do-build == 'true'
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          python-version: ${{ inputs.breeze-python-version }}
         if: inputs.do-build == 'true'
       - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
         shell: bash

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -67,11 +67,6 @@ jobs:
     env:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
-      # Version of Python used for reproducibility of the packages built
-      # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
-      # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
-      # TODO: remove me when we switch to Python 3.9 as minimal version
-      breeze-python-version: "3.9"
       image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
       cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
       affected-providers-list-as-string: >-
@@ -179,7 +174,6 @@ jobs:
       run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
       needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
       default-python-version: ${{needs.build-info.outputs.default-python-version}}
-      breeze-python-version: ${{needs.build-info.outputs.breeze-python-version}}
       basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
       skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}}
       canary-run: ${{needs.build-info.outputs.canary-run}}
@@ -207,7 +201,6 @@ jobs:
       branch: ${{ needs.build-info.outputs.default-branch }}
       use-uv: "true"
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.cache-directive }}
 
@@ -261,7 +254,6 @@ jobs:
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
       ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
       docker-cache: ${{ needs.build-info.outputs.cache-directive }}
@@ -324,7 +316,6 @@ jobs:
       runs-on: ${{ needs.build-info.outputs.runs-on }}
       mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
     if: needs.build-info.outputs.latest-versions-only != 'true'
@@ -363,7 +354,6 @@ jobs:
     with:
       runs-on: ${{ needs.build-info.outputs.runs-on }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }}
@@ -420,7 +410,6 @@ jobs:
       runs-on: ${{ needs.build-info.outputs.runs-on }}
       helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
     if: >
       needs.build-info.outputs.needs-helm-tests == 'true' &&
@@ -449,7 +438,6 @@ jobs:
       run-migration-tests: "true"
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
     if: needs.build-info.outputs.run-tests == 'true'
 
   tests-mysql:
@@ -474,7 +462,6 @@ jobs:
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
     if: needs.build-info.outputs.run-tests == 'true'
 
   tests-sqlite:
@@ -501,7 +488,6 @@ jobs:
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       run-migration-tests: "true"
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
     if: needs.build-info.outputs.run-tests == 'true'
 
   tests-non-db:
@@ -527,7 +513,6 @@ jobs:
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ needs.build-info.outputs.run-coverage }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
     if: needs.build-info.outputs.run-tests == 'true'
 
   tests-special:
@@ -546,7 +531,6 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
     if: needs.build-info.outputs.run-tests == 'true'
 
   tests-integration:
@@ -594,7 +578,6 @@ jobs:
       use-uv: "true"
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       docker-cache: ${{ needs.build-info.outputs.cache-directive }}
@@ -654,7 +637,6 @@ jobs:
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
       docker-cache: ${{ needs.build-info.outputs.cache-directive }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
     if: needs.build-info.outputs.prod-image-build == 'true'
@@ -673,7 +655,6 @@ jobs:
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }}
       kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       debug-resources: ${{ needs.build-info.outputs.debug-resources }}
     if: >
@@ -709,7 +690,6 @@ jobs:
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
-      breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
       in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       docker-cache: ${{ needs.build-info.outputs.cache-directive }}

diff --git a/.github/workflows/early-image-checks.yml b/.github/workflows/early-image-checks.yml
@@ -49,11 +49,6 @@ on:  # yamllint disable-line rule:truthy
         description: Which version of python should be used get CI image
         required: true
         type: string
-      breeze-python-version:
-        description: >
-          Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
-        required: true
-        type: string
       chicken-egg-providers:
         description: "List of providers that should be prepared from sources"
         required: false
@@ -101,7 +96,6 @@ jobs:
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       use-uv: "true"
-      breeze-python-version: ${{ inputs.breeze-python-version }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.canary-run == 'true' && inputs.branch == 'main'
@@ -117,6 +111,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
       PYTHON_VERSION: ${{ inputs.default-python-version }}
       IMAGE_TAG: ${{ inputs.image-tag }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     if: inputs.canary-run == 'true' && inputs.branch == 'main'
     steps:
       - name: "Cleanup repo"
@@ -130,6 +125,8 @@ jobs:
         uses: ./.github/actions/cleanup-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
+      - name: "Login to ghcr.io"
+        run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
       - name: "Check that image builds quickly"
         run: breeze shell --max-time 120
 
@@ -158,8 +155,6 @@ jobs:
         uses: ./.github/actions/cleanup-docker
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
-        with:
-          python-version: ${{ inputs.breeze-python-version }}
       - name: Login to ghcr.io
         run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
       - name: Pull CI images ${{ inputs.python-versions-list-as-string }}:${{ inputs.image-tag }}
@@ -229,7 +224,6 @@ jobs:
       branch: ${{ inputs.branch }}
       use-uv: "true"
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      breeze-python-version: ${{ inputs.breeze-python-version }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.canary-run == 'true'
diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
@@ -49,11 +49,6 @@ on:  # yamllint disable-line rule:truthy
         description: Which version of python should be used get CI image
         required: true
         type: string
-      breeze-python-version:
-        description: >
-          Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
-        required: true
-        type: string
       in-workflow-build:
         description: "Whether the build is executed as part of the workflow (true/false)"
         required: true
@@ -142,7 +137,6 @@ jobs:
       image-tag: ${{ inputs.image-tag }}
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
-      breeze-python-version: ${{ inputs.breeze-python-version }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}
     if: inputs.canary-run == 'true'