From f69d3e61b634c83520e82271865fc62610ffdd97 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 18 Aug 2023 23:52:36 +0200 Subject: [PATCH 1/3] Add a scheduled workflow to synchronize branches from `gitster/git` The Git maintainer publishes only the integration branches to `git/git`, leaving the individual contribution branches in `gitster/git`. We do want to make sure that the latter branches are available, and synchronized, in `gitgitgadget/git`, to facilitate contributions to patch series that are in flight. So let's synchronize the branches, making sure that the integration branches (`maint`, `master`, `next`, `seen`) are not synchronized from `gitster/git`; They are synchronized via the `sync-ref` workflow already. Signed-off-by: Johannes Schindelin --- .github/workflows/sync-gitster-git.yml | 112 +++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 .github/workflows/sync-gitster-git.yml diff --git a/.github/workflows/sync-gitster-git.yml b/.github/workflows/sync-gitster-git.yml new file mode 100644 index 0000000..5f32f48 --- /dev/null +++ b/.github/workflows/sync-gitster-git.yml @@ -0,0 +1,112 @@ +name: sync-gitster-git-branches + +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + +env: + SOURCE_REPOSITORY: gitster/git + TARGET_REPOSITORY: gitgitgadget/git + +jobs: + sync-gitster-git-branches: + runs-on: ubuntu-latest + steps: + - name: check which refs need to be synchronized + uses: actions/github-script@v6 + id: check + with: + script: | + const getRefs = async (repository) => { + const [owner, repo] = repository.split('/') + const { data } = await github.rest.git.listMatchingRefs({ + owner, + repo, + // We want to match `maint-*` as well as `[a-z][a-z]/*` + // sadly, this is not possible via GitHub's REST API, + // hence we do it below via the `filter()` call. + ref: 'heads/' + }) + return data + .filter(e => e.ref.match(/^refs\/heads\/(maint-\d|[a-z][a-z]\/)/)) + .sort((a, b) => a.ref.localeCompare(b.ref)) + } + + const sourceRefs = await getRefs(process.env.SOURCE_REPOSITORY) + const targetRefs = await getRefs(process.env.TARGET_REPOSITORY) + + const refspecs = [] + const toFetch = new Set() + for (let i = 0, j = 0; i < sourceRefs.length || j < targetRefs.length; ) { + const compare = i >= sourceRefs.length + ? +1 + : j >= targetRefs.length + ? -1 + : sourceRefs[i].ref.localeCompare(targetRefs[j].ref) + if (compare > 0) { + // no source ref => delete target ref + refspecs.push(`:${targetRefs[j].ref}`) + j++ + } else if (compare < 0) { + // no corresponding target ref yet => push source ref (new) + const sha = sourceRefs[i].object.sha + toFetch.add(sha) + refspecs.push(`${sha}:${sourceRefs[i].ref}`) + i++ + } else { + // the sourceRef's name matches the targetRef's + if (sourceRefs[i].object.sha !== targetRefs[j].object.sha) { + // target ref needs updating + const sha = sourceRefs[i].object.sha + toFetch.add(sha) + refspecs.push(`+${sha}:${sourceRefs[i].ref}`) + } + i++ + j++ + } + } + + core.setOutput('refspec', refspecs.join(' ')) + targetRefs.forEach((e) => toFetch.delete(e.object.sha)) + core.setOutput('to-fetch', [...toFetch].join(' ')) + - name: obtain installation token + if: steps.check.outputs.refspec != '' + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + id: token + with: + app_id: ${{ secrets.GITGITGADGET_GITHUB_APP_ID }} + private_key: ${{ secrets.GITGITGADGET_GITHUB_APP_PRIVATE_KEY }} + repository: ${{ env.TARGET_REPOSITORY }} + - name: set authorization header + if: steps.check.outputs.refspec != '' + uses: actions/github-script@v6 + id: auth + with: + script: | + // Sadly, `git push` does not work with 'Authorization: Bearer ', therefore + // we have to use the `Basic` variant + const auth = Buffer.from('PAT:${{ steps.token.outputs.token }}').toString('base64') + core.setSecret(auth) + core.setOutput('header', `Authorization: Basic ${auth}`) + - name: sync + if: steps.check.outputs.refspec != '' + shell: bash + run: | + set -ex + git init --bare + + git remote add source "${{ github.server_url }}/$SOURCE_REPOSITORY" + # pretend to be a partial clone + git config remote.source.promisor true + git config remote.source.partialCloneFilter blob:none + + # fetch some commits + printf '%s' '${{ steps.check.outputs.to-fetch }}' | + xargs -d ' ' -r git fetch --depth 10000 source + rm -f .git/shallow + + # push the commits + printf '%s' '${{ steps.check.outputs.refspec }}' | + xargs -d ' ' -r git -c http.extraHeader='${{ steps.auth.outputs.header }}' \ + push "${{ github.server_url }}/$TARGET_REPOSITORY" From 1fabadb3ec48995a524a4076ea3b0122529dab0c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 19 Aug 2023 00:22:38 +0200 Subject: [PATCH 2/3] sync-gitster-git: handle 502 gracefully There are a lot of branches in `gitster/git`. A _lot_. So let's be nice to GitHub and back off (but do try again) if encountering a 502 (which indicates that GitHub is as overloaded as the rest of us). Signed-off-by: Johannes Schindelin --- .github/workflows/sync-gitster-git.yml | 41 ++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-gitster-git.yml b/.github/workflows/sync-gitster-git.yml index 5f32f48..65b6c21 100644 --- a/.github/workflows/sync-gitster-git.yml +++ b/.github/workflows/sync-gitster-git.yml @@ -18,19 +18,36 @@ jobs: id: check with: script: | + const sleep = async (milliseconds) => { + return new Promise(resolve => setTimeout(resolve, milliseconds)) + } + const getRefs = async (repository) => { - const [owner, repo] = repository.split('/') - const { data } = await github.rest.git.listMatchingRefs({ - owner, - repo, - // We want to match `maint-*` as well as `[a-z][a-z]/*` - // sadly, this is not possible via GitHub's REST API, - // hence we do it below via the `filter()` call. - ref: 'heads/' - }) - return data - .filter(e => e.ref.match(/^refs\/heads\/(maint-\d|[a-z][a-z]\/)/)) - .sort((a, b) => a.ref.localeCompare(b.ref)) + let attemptCounter = 1 + for (;;) { + try { + const [owner, repo] = repository.split('/') + const { data } = await github.rest.git.listMatchingRefs({ + owner, + repo, + // We want to match `maint-*` as well as `[a-z][a-z]/*` + // sadly, this is not possible via GitHub's REST API, + // hence we do it below via the `filter()` call. + ref: 'heads/' + }) + return data + .filter(e => e.ref.match(/^refs\/heads\/(maint-\d|[a-z][a-z]\/)/)) + .sort((a, b) => a.ref.localeCompare(b.ref)) + } catch (e) { + if (e?.status !== 502) throw e + } + + if (++attemptCounter > 10) throw new Error('Giving up listing refs after 10 attempts') + + const seconds = attemptCounter * attemptCounter + 15 * Math.random() + core.info(`Encountered a Server Error; retrying in ${seconds} seconds`) + await sleep(1000 * seconds) + } } const sourceRefs = await getRefs(process.env.SOURCE_REPOSITORY) From 525752bcc9da74030615cb60e7efde55285b4af4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 19 Aug 2023 03:47:26 +0200 Subject: [PATCH 3/3] sync-gitster-git-branches: limit concurrency It does not make sense to try synchronizing in parallel. I considered adding `cancel-in-progress: true`, but that strikes me as wasteful: if we're already pushing an update, canceling it would require the entire work to be done again, while chances are that any queued run will likely find that everything is already up to date and no further work needs to be done. Signed-off-by: Johannes Schindelin --- .github/workflows/sync-gitster-git.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/sync-gitster-git.yml b/.github/workflows/sync-gitster-git.yml index 65b6c21..c9850b8 100644 --- a/.github/workflows/sync-gitster-git.yml +++ b/.github/workflows/sync-gitster-git.yml @@ -9,6 +9,12 @@ env: SOURCE_REPOSITORY: gitster/git TARGET_REPOSITORY: gitgitgadget/git +# We want to limit queuing to a single workflow run i.e. if there is already +# an active workflow run and a queued one, queue another one canceling the +# already queued one. +concurrency: + group: ${{ github.workflow }} + jobs: sync-gitster-git-branches: runs-on: ubuntu-latest