diff --git a/.github/workflows/azure_ci.js b/.github/workflows/azure_ci.js new file mode 100644 index 0000000000000..9844fd567d984 --- /dev/null +++ b/.github/workflows/azure_ci.js @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +async function checkAzureCiAndCreateCommitStatus({ github, context, prNumber, latestCommitHash }) { + console.log(`- Checking Azure CI status of PR: ${prNumber} ${latestCommitHash}`); + const botUsername = 'hudi-bot'; + + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + }); + + // Find the latest comment from hudi-bot containing the Azure CI report + const botComments = comments.data.filter(comment => comment.user.login === botUsername); + const lastComment = botComments.pop(); + + let status = 'pending'; + let message = 'In progress'; + let azureRunLink = ''; + + if (lastComment) { + const reportPrefix = `${latestCommitHash} Azure: ` + const successReportString = `${reportPrefix}[SUCCESS]` + const failureReportString = `${reportPrefix}[FAILURE]` + + if (lastComment.body.includes(reportPrefix)) { + if (lastComment.body.includes(successReportString)) { + message = 'Successful on the latest commit'; + status = 'success'; + } else if (lastComment.body.includes(failureReportString)) { + message = 'Failed on the latest commit'; + status = 'failure'; + } + } + + const linkRegex = /\[[a-zA-Z]+\]\((https?:\/\/[^\s]+)\)/; + const parts = lastComment.body.split(reportPrefix); + const secondPart = parts.length > 1 ? parts[1] : ''; + const match = secondPart.match(linkRegex); + + if (match) { + azureRunLink = match[1]; + } + } + + console.log(`Status: ${status}`); + console.log(`Azure Run Link: ${azureRunLink}`); + console.log(`${message}`); + + console.log(`- Create commit status of PR based on Azure CI status: ${prNumber} ${latestCommitHash}`); + // Create or update the commit status for Azure CI + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: latestCommitHash, + state: status, + target_url: azureRunLink, + description: message, + context: 'Azure CI' + }); + + return { status, message, azureRunLink }; +} + +module.exports = checkAzureCiAndCreateCommitStatus; \ No newline at end of file diff --git a/.github/workflows/azure_ci_check.yml b/.github/workflows/azure_ci_check.yml index 3c4ba58a7f399..9ad88c873b540 100644 --- a/.github/workflows/azure_ci_check.yml +++ b/.github/workflows/azure_ci_check.yml @@ -18,6 +18,8 @@ name: Azure CI on: + pull_request: + types: [ opened, edited, reopened, synchronize ] issue_comment: types: [ created, edited, deleted ] @@ -27,67 +29,54 @@ permissions: issues: read jobs: - check-azure-ci-report: - if: "!contains(github.event.pull_request.body, 'HOTFIX: SKIP AZURE CI')" + check-azure-ci-and-add-commit-status: + if: true #| + #github.event.issue.pull_request != null && + # github.event.issue_comment.user.login == 'hudi-bot' runs-on: ubuntu-latest steps: - - name: Get last commit hash - id: last_commit - uses: actions/github-script@v7 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const pr = context.payload.pull_request; - const lastCommitHash = pr.head.sha; - console.log(`Last commit hash: ${lastCommitHash}`); - // Set the output variable to be used in subsequent step - core.setOutput("last_commit_hash", lastCommitHash); + - name: Checkout repository + uses: actions/checkout@v2 - - name: Check Azure CI report in PR comment + - name: Check PR state + id: check_pr_state uses: actions/github-script@v7 with: github-token: ${{secrets.GITHUB_TOKEN}} script: | - const lastCommitHash = '${{ steps.last_commit.outputs.last_commit_hash }}' - const botUsername = 'hudi-bot'; - - const issueNumber = context.payload.pull_request.number; - const comments = await github.rest.issues.listComments({ + const issueNumber = 10740; //github.event.issue.number; + const { data: pullRequest } = await github.rest.pulls.get({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: issueNumber, + pull_number: issueNumber }); - // Find the last comment from hudi-bot containing the Azure CI report - const botComments = comments.data.filter(comment => comment.user.login === botUsername); - const lastComment = botComments.pop(); + // Only check open PRs and a PR that is not a HOTFIX + const shouldSkip = (pullRequest.body.includes('HOTFIX: SKIP AZURE CI') + || pullRequest.state != 'open'); - if (lastComment) { - const reportPrefix = '${lastCommitHash} Azure: ' - const successReportString = '${reportPrefix}[SUCCESS]' - const failureReportString = '${reportPrefix}[FAILURE]' - if (lastComment.body.includes(reportPrefix)) { - if (lastComment.body.includes(successReportString)) { - console.log(`Azure CI succeeded on the latest commit of the PR.`); - return true; - } else if (lastComment.body.includes(failureReportString)) { - console.log(`Azure CI failed on the latest commit of the PR.`); - core.setFailed("Azure CI failed on the latest commit of the PR."); - return false; - } else { - console.log(`Azure CI is in progress on the latest commit of the PR.`); - core.setFailed("Azure CI is in progress on the latest commit of the PR."); - return false; - } - } else { - console.log(`No Azure CI report on the latest commit of the PR.`); - core.setFailed("No Azure CI report on the latest commit of the PR."); - return false; - } - } else { - console.log(`Azure CI report does not seem to be ready yet.`); - core.setFailed("Azure CI report does not seem to be ready yet."); - return false; + if (!shouldSkip) { + const commitHash = pullRequest.head.sha; + console.log(`Latest commit hash: ${commitHash}`); + // Set the output variable to be used in subsequent step + core.setOutput("latest_commit_hash", commitHash); } - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + console.log(`Should skip Azure CI? ${shouldSkip}`); + return shouldSkip; + + - name: Check Azure CI report and create commit status to PR + if: steps.check_pr_state.outputs.result != 'true' + uses: actions/github-script@v7 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const latestCommitHash = '${{ steps.check_pr_state.outputs.latest_commit_hash }}' + const issueNumber = 10740;//github.event.issue.number; + const checkAzureCiAndCreateCommitStatus = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`); + + await checkAzureCiAndCreateCommitStatus({ + github, + context, + prNumber: issueNumber, + latestCommitHash: latestCommitHash + }); diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index d0b809c295870..a35174ca2525f 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,24 +1,54 @@ name: Label PR -on: [ pull_request ] +on: + pull_request: + types: [ opened, edited, reopened, synchronize ] + +permissions: + issues: write + pull-requests: write jobs: labeler: runs-on: ubuntu-latest - name: Label the PR size steps: - - uses: codelytv/pr-size-labeler@v1 + - name: Label PR size + uses: actions/github-script@v7 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - xs_label: 'size-xs' - xs_max_size: '10' - s_label: 'size-s' - s_max_size: '100' - m_label: 'size-m' - m_max_size: '500' - l_label: 'size-l' - l_max_size: '1000' - xl_label: 'size-xl' - fail_if_xl: 'false' - github_api_url: 'api.github.com' - files_to_ignore: '' \ No newline at end of file + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const prNumber = context.payload.pull_request.number; + const { data: prData } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + + const additions = prData.additions; + const deletions = prData.deletions; + const totalChanges = additions + deletions; + + let label = ""; + + if (totalChanges < 10) { + label = "size:XS"; + } else if (totalChanges < 100) { + label = "size:S"; + } else if (totalChanges < 300) { + label = "size:M"; + } else if (totalChanges < 1000) { + label = "size:L"; + } else { + label = "size:XL"; + } + + // Apply the label + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [label], + }); + + console.log(`Total lines of changes: ${totalChanges}`); + console.log(`Label: ${label}`); \ No newline at end of file diff --git a/.github/workflows/scheduled_workflow.yml b/.github/workflows/scheduled_workflow.yml new file mode 100644 index 0000000000000..092929ebee2bf --- /dev/null +++ b/.github/workflows/scheduled_workflow.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Scheduled Workflow + +on: + schedule: + # Runs every 5 minutes + - cron: '*/5 * * * *' + pull_request: + types: [ opened, edited, reopened, synchronize ] + +permissions: + statuses: write + pull-requests: read + issues: read + +jobs: + process-new-and-updated-prs: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Process new and updated PRs + # We have to run any actions that require write permissions here + # since the workflow triggered by events from a PR in a fork + # (not apache/hudi but other_owner/hudi) does not run on a + # GITHUB_TOKEN with write permissions (this is prohibited by + # Apache). + uses: actions/github-script@v7 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const since = new Date(new Date().getTime() - (60 * 60 * 1000)).toISOString(); + const query = `repo:${context.repo.owner}/${context.repo.repo} type:pr updated:>=${since}`; + const response = await github.rest.search.issuesAndPullRequests({ + q: query + }); + + // Filter for open PRs + const openPrs = response.data.items.filter(pr => pr.state === 'open'); + const checkAzureCiAndCreateCommitStatus = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`); + + for (const pr of openPrs) { + console.log(`*** Processing PR: ${pr.title}, URL: ${pr.html_url}`); + + if (!pr.body.includes('HOTFIX: SKIP AZURE CI')) { + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number + }); + const latestCommitHash = pullRequest.head.sha; + + // Create commit status based on Azure CI report to PR + await checkAzureCiAndCreateCommitStatus({ + github, + context, + prNumber: pr.number, + latestCommitHash: latestCommitHash + }); + } + }