Skip to content

Commit

Permalink
Merge branch 'main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
bareketamir authored Nov 18, 2024
2 parents ada9511 + 8027055 commit 0343c27
Show file tree
Hide file tree
Showing 1,091 changed files with 21,131 additions and 18,202 deletions.
23 changes: 23 additions & 0 deletions .cherry_picker.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

team = "apache"
repo = "airflow"
"check_sha"= "a85d94e6cdcd09efe93c3acee0b4ce5c9508bc23"
fix_commit_msg = false
default_branch = "main"
require_version_in_branch_name=false
7 changes: 5 additions & 2 deletions .github/actions/checkout_target_commit/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,16 @@ runs:
rm -rfv "dev"
rm -rfv ".github/actions"
rm -rfv ".github/workflows"
rm -v ".dockerignore" || true
mv -v "target-airflow/scripts/ci" "scripts"
mv -v "target-airflow/dev" "."
mv -v "target-airflow/.github/actions" "target-airflow/.github/workflows" ".github"
mv -v "target-airflow/.dockerignore" ".dockerignore" || true
if: inputs.pull-request-target == 'true' && inputs.is-committer-build != 'true'
####################################################################################################
# AFTER IT'S SAFE. THE `dev`, `scripts/ci` AND `.github/actions` ARE NOW COMING FROM THE
# BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE SAFE TO RUN.
# AFTER IT'S SAFE. THE `dev`, `scripts/ci` AND `.github/actions` and `.dockerignore` ARE NOW COMING
# FROM THE BASE_REF - WHICH IS THE TARGET BRANCH OF THE PR. WE CAN TRUST THAT THOSE SCRIPTS ARE
# SAFE TO RUN AND CODE AVAILABLE IN THE DOCKER BUILD PHASE IS CONTROLLED BY THE `.dockerignore`.
# ALL THE REST OF THE CODE COMES FROM THE PR, AND FOR EXAMPLE THE CODE IN THE `Dockerfile.ci` CAN
# BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
# ISOLATED FROM THE RUNNER.
Expand Down
11 changes: 5 additions & 6 deletions .github/actions/install-pre-commit/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ inputs:
default: 3.9
uv-version:
description: 'uv version to use'
default: 0.4.30
default: 0.5.2
pre-commit-version:
description: 'pre-commit version to use'
default: 4.0.1
Expand All @@ -36,11 +36,10 @@ runs:
steps:
- name: Install pre-commit, uv, and pre-commit-uv
shell: bash
run: >
pip install
pre-commit==${{inputs.pre-commit-version}}
uv==${{inputs.uv-version}}
pre-commit-uv==${{inputs.pre-commit-uv-version}}
run: |
pip install uv==${{inputs.uv-version}} || true
uv tool install pre-commit==${{inputs.pre-commit-version}} --with uv==${{inputs.uv-version}} \
--with pre-commit-uv==${{inputs.pre-commit-uv-version}}
- name: Cache pre-commit envs
uses: actions/cache@v4
with:
Expand Down
77 changes: 77 additions & 0 deletions .github/workflows/automatic-backport.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Automatic Backport
on: # yamllint disable-line rule:truthy
push:
branches:
- main

jobs:
get-pr-info:
name: "Get PR information"
runs-on: ubuntu-latest
outputs:
branches: ${{ steps.pr-info.outputs.branches }}
commit-sha: ${{ github.sha }}
steps:
- name: Get commit SHA
id: get-sha
run: echo "COMMIT_SHA=${GITHUB_SHA}" >> $GITHUB_ENV

- name: Find PR information
id: pr-info
uses: actions/github-script@v7
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
script: |
const { data: pullRequest } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: process.env.GITHUB_SHA
});
if (pullRequest.length > 0) {
const pr = pullRequest[0];
const backportBranches = pr.labels
.filter(label => label.name.startsWith('backport-to-'))
.map(label => label.name.replace('backport-to-', ''));
console.log(`Commit ${process.env.GITHUB_SHA} is associated with PR ${pr.number}`);
console.log(`Backport branches: ${backportBranches}`);
core.setOutput('branches', JSON.stringify(backportBranches));
} else {
console.log('No pull request found for this commit.');
core.setOutput('branches', '[]');
}
trigger-backport:
name: "Trigger Backport"
uses: ./.github/workflows/backport-cli.yml
needs: get-pr-info
if: ${{ needs.get-pr-info.outputs.branches != '[]' }}
strategy:
matrix:
branch: ${{ fromJSON(needs.get-pr-info.outputs.branches) }}
fail-fast: false
permissions:
contents: write
pull-requests: write
with:
target-branch: ${{ matrix.branch }}
commit-sha: ${{ needs.get-pr-info.outputs.commit-sha }}
115 changes: 115 additions & 0 deletions .github/workflows/backport-cli.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Backport Commit
on: # yamllint disable-line rule:truthy
workflow_dispatch:
inputs:
commit-sha:
description: "Commit sha to backport."
required: true
type: string
target-branch:
description: "Target branch to backport."
required: true
type: string

workflow_call:
inputs:
commit-sha:
description: "Commit sha to backport."
required: true
type: string
target-branch:
description: "Target branch to backport."
required: true
type: string

permissions:
contents: write
pull-requests: write
jobs:
backport:
runs-on: ubuntu-latest

steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
id: checkout-for-backport
uses: actions/checkout@v4
with:
persist-credentials: true
fetch-depth: 0

- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
python -m pip install cherry-picker==2.4.0 requests==2.32.3
- name: Run backport script
id: execute-backport
env:
GH_AUTH: ${{ secrets.GITHUB_TOKEN }}
run: |
git config --global user.email "[email protected]"
git config --global user.name "Your Name"
set +e
{
echo 'cherry_picker_output<<EOF'
cherry_picker ${{ inputs.commit-sha }} ${{ inputs.target-branch }}
echo EOF
} >> "${GITHUB_OUTPUT}"
continue-on-error: true

- name: Parse backport output
id: parse-backport-output
run: |
set +e
echo "${{ steps.execute-backport.outputs.cherry_picker_output }}"
url=$(echo "${{ steps.execute-backport.outputs.cherry_picker_output }}" | \
grep -o 'Backport PR created at https://[^ ]*' | \
awk '{print $5}')
url=${url:-"EMPTY"}
if [ "$url" == "EMPTY" ]; then
# If the backport failed, abort the workflow
cherry_picker --abort
fi
echo "backport-url=$url" >> "${GITHUB_OUTPUT}"
continue-on-error: true

- name: Update Status
id: backport-status
env:
GH_TOKEN: ${{ github.token }}
REPOSITORY: ${{ github.repository }}
RUN_ID: ${{ github.run_id }}
run: |
COMMIT_INFO_URL="https://api.github.com/repos/${{ github.repository }}/commits/"
COMMIT_INFO_URL="${COMMIT_INFO_URL}${{ inputs.commit-sha }}/pulls"
PR_NUMBER=$(gh api \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/commits/${{ inputs.commit-sha }}/pulls \
--jq '.[0].number')
python ./dev/backport/update_backport_status.py \
${{ steps.parse-backport-output.outputs.backport-url }} \
${{ inputs.commit-sha }} ${{ inputs.target-branch }} \
"$PR_NUMBER"
17 changes: 13 additions & 4 deletions .github/workflows/basic-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -196,10 +196,19 @@ jobs:
working-directory: ./clients/python
- name: "Install source version of required packages"
run: |
breeze release-management prepare-provider-packages fab standard common.sql --package-format \
wheel --skip-tag-check --version-suffix-for-pypi dev0
pip install . dist/apache_airflow_providers_fab-*.whl \
dist/apache_airflow_providers_standard-*.whl dist/apache_airflow_providers_common_sql-*.whl
breeze release-management prepare-provider-packages \
fab \
standard \
common.sql \
sqlite \
--package-format wheel \
--skip-tag-check \
--version-suffix-for-pypi dev0
pip install . \
dist/apache_airflow_providers_fab-*.whl \
dist/apache_airflow_providers_standard-*.whl \
dist/apache_airflow_providers_common_sql-*.whl \
dist/apache_airflow_providers_sqlite-*.whl
breeze release-management prepare-task-sdk-package --package-format wheel
pip install ./dist/apache_airflow_task_sdk-*.whl
- name: "Install Python client"
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/check-providers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ on: # yamllint disable-line rule:truthy
description: "Whether to upgrade to newer dependencies"
required: true
type: string
affected-providers-list-as-string:
selected-providers-list-as-string:
description: "List of affected providers as string"
required: false
type: string
Expand All @@ -54,7 +54,7 @@ on: # yamllint disable-line rule:truthy
description: "List of parallel provider test types as string"
required: true
type: string
skip-provider-tests:
skip-providers-tests:
description: "Whether to skip provider tests (true/false)"
required: true
type: string
Expand Down Expand Up @@ -163,7 +163,7 @@ jobs:
run: >
breeze release-management prepare-provider-packages --include-not-ready-providers
--version-suffix-for-pypi dev0 --package-format sdist
${{ inputs.affected-providers-list-as-string }}
${{ inputs.selected-providers-list-as-string }}
- name: "Prepare airflow package: sdist"
run: >
breeze release-management prepare-airflow-package
Expand All @@ -187,7 +187,7 @@ jobs:
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
--run-in-parallel
if: inputs.affected-providers-list-as-string == ''
if: inputs.selected-providers-list-as-string == ''
- name: "Install affected provider packages and airflow via sdist files"
run: >
breeze release-management install-provider-packages
Expand All @@ -198,7 +198,7 @@ jobs:
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
--run-in-parallel
if: inputs.affected-providers-list-as-string != ''
if: inputs.selected-providers-list-as-string != ''

providers-compatibility-checks:
timeout-minutes: 80
Expand All @@ -218,7 +218,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev0"
VERBOSE: "true"
CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
if: inputs.skip-provider-tests != 'true'
if: inputs.skip-providers-tests != 'true'
steps:
- name: "Cleanup repo"
shell: bash
Expand Down Expand Up @@ -268,7 +268,7 @@ jobs:
Airflow ${{ matrix.airflow-version }}:Python ${{ matrix.python-version }}
if: matrix.run-tests == 'true'
run: >
breeze testing tests --run-in-parallel
breeze testing providers-tests --run-in-parallel
--parallel-test-types "${{ inputs.providers-test-types-list-as-string }}"
--use-packages-from-dist
--package-format wheel
Expand Down
Loading

0 comments on commit 0343c27

Please sign in to comment.