From f0d1fa4e9377ef2102ce48f29d167af9fd75b327 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Wed, 3 Feb 2021 10:14:05 +0000 Subject: [PATCH] ci: add first version of copybara workflow This is a simple migration of the existing copybara setup to run on top of GitHub actions. The "github" and "github-pr" copybara commands are invoked via repository dispatch workflows, and run as the cueckoo user. The repository dispatch workflows are triggered via github.com/cue-sh/tools/cmd/cuekcoo. This tool is not, for now, added as a dependency because that would create a requirement cycle. This is not a problem in and of itself, but we choose to postpone making that decision for now (plus gorelease appears to have a bug relating to this). So for now we require: github.com/cue-sh/tools/cmd/cueckoo@v0.0.0-20210205194434-b3293e86064a Change-Id: Ibf146c900c7888a1be813020fbd897f2602a60ad Reviewed-on: https://cue-review.googlesource.com/c/cue/+/8521 Reviewed-by: Paul Jolly --- .github/workflows/mirror.yml | 29 ++ .github/workflows/repository_dispatch.yml | 58 +++- _scripts/copy.bara.sky | 74 +++++ cmd/cue/cmd/testdata/script/cmd_github.txt | 310 ++++++++++++--------- cue/testdata/eval/github.txtar | 190 +++++++++++-- internal/ci/ci_tool.cue | 3 +- internal/ci/workflows.cue | 94 ++++++- 7 files changed, 589 insertions(+), 169 deletions(-) create mode 100644 .github/workflows/mirror.yml create mode 100644 _scripts/copy.bara.sky diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml new file mode 100644 index 000000000..afb22a9fc --- /dev/null +++ b/.github/workflows/mirror.yml @@ -0,0 +1,29 @@ +# Generated by internal/ci/ci_tool.cue; do not edit + +name: Scheduled repo mirror +on: + schedule: + - cron: '*/30 * * * *' +jobs: + mirror: + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Mirror Gerrit to GitHub + run: |- + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky github; \ + " diff --git a/.github/workflows/repository_dispatch.yml b/.github/workflows/repository_dispatch.yml index 0571716da..819611560 100644 --- a/.github/workflows/repository_dispatch.yml +++ b/.github/workflows/repository_dispatch.yml @@ -4,14 +4,14 @@ name: Repository Dispatch on: - repository_dispatch jobs: - start: - if: ${{ startsWith(github.event.action, 'Build for refs/changes/') }} + runtrybot: runs-on: ubuntu-18.04 defaults: run: shell: bash + if: ${{ github.event.client_payload.type == 'runtrybot' }} steps: - - name: Checkout ref + - name: Trigger trybot run: |- mkdir tmpgit cd tmpgit @@ -19,6 +19,52 @@ jobs: git config user.name cueckoo git config user.email cueckoo@gmail.com git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD - git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }} + git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD + git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} + mirror: + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash + if: ${{ github.event.client_payload.type == 'mirror' }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Mirror Gerrit to GitHub + run: |- + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky github; \ + " + importpr: + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash + if: ${{ github.event.client_payload.type == 'importpr' }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: 'Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit' + run: |- + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky github-pr ${{ github.event.client_payload.payload.pr }}; \ + " diff --git a/_scripts/copy.bara.sky b/_scripts/copy.bara.sky new file mode 100644 index 000000000..736f5f72c --- /dev/null +++ b/_scripts/copy.bara.sky @@ -0,0 +1,74 @@ +# Copyright 2021 The CUE Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The copybara configuration for the cuelang/cue repository +# +# When used in conjunction with the cueckoo/copybara Docker image this +# configuration can be run in two modes: +# +# github +# github-pr N +# +# The first mode mirrors Gerrit to GitHub. The second mode imports PR #N from +# GitHub to Gerrit. +# +# See the CUE-defined copybara workflow defined in internal/ci for an example +# of how to pass in Gerrit and GitHub credentials when running an instance of +# the cueckoo/copybara image. + +github_url = "https://github.com/cuelang/cue.git" +gerrit_url = "https://cue-review.googlesource.com/cue" + +origin_github_pr = git.github_pr_origin( + baseline_from_branch = True, + use_merge = True, + url = github_url, + ) + +origin_gerrit = git.origin( + url = gerrit_url, + ref = "master", + ) + +destination_gerrit = git.gerrit_destination( + url = gerrit_url, + fetch = "master", + push_to_refs_for = "master%hashtag=github-pr", + ) + +# Mirror from Gerrit -> GitHub +git.mirror( + name = "github", + origin = gerrit_url, + destination = github_url, + refspecs = ["refs/*"], + prune = False, + ) + +# Import a PR from GitHub to Gerrit +core.workflow( + name = "github-pr", + origin = origin_github_pr, + destination = destination_gerrit, + authoring = authoring.pass_thru("CUE team "), + transformations = [ + metadata.replace_message( + "${GITHUB_PR_TITLE}\n\n" + + "${GITHUB_PR_BODY}\n\n" + + "Closes #${GITHUB_PR_NUMBER}\n" + + "https://github.com/cuelang/cue/pull/${GITHUB_PR_NUMBER}", + ), + ], + mode = "CHANGE_REQUEST", + ) diff --git a/cmd/cue/cmd/testdata/script/cmd_github.txt b/cmd/cue/cmd/testdata/script/cmd_github.txt index 0e81aa5fb..68811dc17 100644 --- a/cmd/cue/cmd/testdata/script/cmd_github.txt +++ b/cmd/cue/cmd/testdata/script/cmd_github.txt @@ -15,6 +15,7 @@ cmp .github/workflows/rebuild_tip_cuelang_org.yml .github/workflows/rebuild_tip_ cmp .github/workflows/release.yml .github/workflows/release.yml.golden cmp .github/workflows/repository_dispatch.yml .github/workflows/repository_dispatch.yml.golden cmp .github/workflows/test.yml .github/workflows/test.yml.golden +cmp .github/workflows/mirror.yml .github/workflows/mirror.yml.golden -- cue.mod/module -- module "cuelang.org/go" @@ -100,14 +101,14 @@ name: Repository Dispatch on: - repository_dispatch jobs: - start: - if: ${{ startsWith(github.event.action, 'Build for refs/changes/') }} + runtrybot: runs-on: ubuntu-18.04 defaults: run: shell: bash + if: ${{ github.event.client_payload.type == 'runtrybot' }} steps: - - name: Checkout ref + - name: Trigger trybot run: |- mkdir tmpgit cd tmpgit @@ -115,9 +116,55 @@ jobs: git config user.name cueckoo git config user.email cueckoo@gmail.com git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD - git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }} + git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD + git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} + mirror: + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash + if: ${{ github.event.client_payload.type == 'mirror' }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Mirror Gerrit to GitHub + run: |- + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky github; \ + " + importpr: + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash + if: ${{ github.event.client_payload.type == 'importpr' }} + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: 'Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit' + run: |- + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky github-pr ${{ github.event.client_payload.payload.pr }}; \ + " -- .github/workflows/test.yml.golden -- # Generated by internal/ci/ci_tool.cue; do not edit @@ -236,6 +283,36 @@ jobs: git config user.email cueckoo@gmail.com git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" git push https://github.com/cuelang/cue :${GITHUB_REF#refs/heads/} +-- .github/workflows/mirror.yml.golden -- +# Generated by internal/ci/ci_tool.cue; do not edit + +name: Scheduled repo mirror +on: + schedule: + - cron: '*/30 * * * *' +jobs: + mirror: + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Mirror Gerrit to GitHub + run: |- + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky github; \ + " -- cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue -- package json @@ -898,125 +975,6 @@ import "strings" #: "working-directory": string } --- ci_tool.cue -- -package ci - -import ( - "path" - - "encoding/yaml" - - "tool/exec" - "tool/file" - "tool/os" -) - -// _#modroot is a common helper to get the module root -// -// TODO: use once we have a solution to cuelang.org/issue/704. -// This will then allow us to remove the use of .. below. -_#modroot: exec.Run & { - cmd: "go list -m -f {{.Dir}}" - stdout: string -} - -// Until we have the ability to inject contextual information -// we need to pass in GOOS explicitly. Either by environment -// variable (which we get for free when this is used via go generate) -// or via a tag in the case you want to manually run the CUE -// command. -_#goos: os.Getenv & { - GOOS: *"unix" | string @tag(os) -} - -// genworkflows regenerates the GitHub workflow Yaml definitions -// -// Until we have a resolution for cuelang.org/issue/704 and -// cuelang.org/issue/708 this must be run from the internal/ci package. At -// which point we can switch to using _#modroot. -// -// This also explains why the ../../ relative path specification below appear -// wrong in the context of the containing directory internal/ci/vendor. -command: genworkflows: { - goos: _#goos - - for w in workflows { - "\(w.file)": file.Create & { - _dir: path.FromSlash("../../.github/workflows", "unix") - filename: path.Join([_dir, w.file], goos.GOOS) - contents: """ - # Generated by internal/ci/ci_tool.cue; do not edit - - \(yaml.Marshal(w.schema)) - """ - } - } -} - -// updateTxtarTests ensures certain txtar tests are updated with the -// relevant files that make up the process of generating our CI -// workflows. -// -// Until we have a resolution for cuelang.org/issue/704 and -// cuelang.org/issue/708 this must be run from the internal/ci package. At -// which point we can switch to using _#modroot. -// -// This also explains why the ../../ relative path specification below appear -// wrong in the context of the containing directory internal/ci/vendor. -command: updateTxtarTests: { - goos: _#goos - - readJSONSchema: file.Read & { - _path: path.FromSlash("../../cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue", "unix") - filename: path.Join([_path], goos.GOOS) - contents: string - } - cueDefInternalCI: exec.Run & { - cmd: "go run cuelang.org/go/cmd/cue def cuelang.org/go/internal/ci" - stdout: string - } - // updateEvalTxtarTest updates the cue/testdata/eval testscript which exercises - // the evaluation of the workflows defined in internal/ci (which by definition - // means resolving and using the vendored GitHub Workflow schema) - updateEvalTxtarTest: { - _relpath: path.FromSlash("../../cue/testdata/eval/github.txtar", "unix") - _path: path.Join([_relpath], goos.GOOS) - - githubSchema: exec.Run & { - stdin: readJSONSchema.contents - cmd: "go run cuelang.org/go/internal/ci/updatetxtar - \(_path) cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue" - } - defWorkflows: exec.Run & { - $after: githubSchema - stdin: cueDefInternalCI.stdout - cmd: "go run cuelang.org/go/internal/ci/updatetxtar - \(_path) workflows.cue" - } - } - // When we have a solution for cuelang.org/issue/709 we can make this a - // file.Glob - readToolsFile: file.Read & { - filename: "ci_tool.cue" - contents: string - } - updateCmdCueCmdTxtarTest: { - _relpath: path.FromSlash("../../cmd/cue/cmd/testdata/script/cmd_github.txt", "unix") - _path: path.Join([_relpath], goos.GOOS) - - githubSchema: exec.Run & { - stdin: readJSONSchema.contents - cmd: "go run cuelang.org/go/internal/ci/updatetxtar - \(_path) cue.mod/pkg/github.com/SchemaStore/schemastore/src/schemas/json/github-workflow.cue" - } - defWorkflows: exec.Run & { - $after: githubSchema - stdin: cueDefInternalCI.stdout - cmd: "go run cuelang.org/go/internal/ci/updatetxtar - \(_path) workflows.cue" - } - toolsFile: exec.Run & { - stdin: readToolsFile.contents - cmd: "go run cuelang.org/go/internal/ci/updatetxtar - \(_path) \(readToolsFile.filename)" - } - } -} -- internal/ci/ci_tool.cue -- // Copyright 2021 The CUE Authors // @@ -1116,7 +1074,8 @@ command: updateTxtarTests: { } } // When we have a solution for cuelang.org/issue/709 we can make this a - // file.Glob + // file.Glob. Ultimately it would be better to be able to do a cue def + // on the tool "package" readToolsFile: file.Read & { filename: "ci_tool.cue" contents: string @@ -1184,6 +1143,9 @@ workflows: [...{ }, { file: "rebuild_tip_cuelang_org.yml" schema: rebuild_tip_cuelang_org +}, { + file: "mirror.yml" + schema: mirror }] test: _#bashWorkflow & { name: "Test" @@ -1282,20 +1244,60 @@ test: _#bashWorkflow & { } } repository_dispatch: _#bashWorkflow & { + // These constants are defined by github.com/cue-sh/tools/cmd/cueckoo + _#runtrybot: "runtrybot" + _#mirror: "mirror" + _#importpr: "importpr" + _#dispatchJob: _#job & { + _#type: string + "runs-on": _#linuxMachine + if: "${{ github.event.client_payload.type == '\(_#type)' }}" + } name: "Repository Dispatch" on: ["repository_dispatch"] - jobs: start: { - if: "${{ startsWith(github.event.action, 'Build for refs/changes/') }}" + jobs: { + "\(_#runtrybot)": _#dispatchJob & { + _#type: _#runtrybot + steps: [_#step & { + name: "Trigger trybot" + run: """ + \(_#tempCueckooGitDir) + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }} + git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD + git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} + """ + }] + } + "\(_#mirror)": _#dispatchJob & { + _#type: _#mirror + steps: _#copybaraSteps & { + _ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + } + } + "\(_#importpr)": _#dispatchJob & { + _#type: _#importpr + steps: _#copybaraSteps & { + _ + _#name: "Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit" + _#cmd: "github-pr ${{ github.event.client_payload.payload.pr }}" + } + } + } +} +mirror: _#bashWorkflow & { + name: "Scheduled repo mirror" + on: schedule: [{ + cron: "*/30 * * * *" + }] + jobs: mirror: { "runs-on": _#linuxMachine - steps: [_#step & { - name: "Checkout ref" - run: """ - \(_#tempCueckooGitDir) - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD - git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} - """ - }] + steps: _#copybaraSteps & { + _ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + } } } release: _#bashWorkflow & { @@ -1451,3 +1453,37 @@ _#tempCueckooGitDir: """ git config user.email cueckoo@gmail.com git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" """ + +// The cueckoo/copybara Docker image to use +_#cueckooCopybaraImage: "cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c" + +// Define the base command for copybara +_#copybaraCmd: { + """ + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" \(_#cueckooCopybaraImage) bash -c " \\ + \tset -eu; \\ + \techo \\"${{ secrets.gerritCookie }}\\" > ~/.gitcookies; \\ + \tchmod 600 ~/.gitcookies; \\ + \tgit config --global user.name cueckoo; \\ + \tgit config --global user.email cueckoo@gmail.com; \\ + \tgit config --global http.cookiefile \\$HOME/.gitcookies; \\ + \techo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \\ + \tchmod 600 ~/.git-credentials; \\ + \tjava -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky \(_#cmd); \\ + \t" + """ + _#cmd: string +} +_#copybaraSteps: { + let cmdCmd = _#cmd + [_#checkoutCode, _#step & { + name: _#name + run: _#copybaraCmd & { + _ + _#cmd: cmdCmd + } + }] + _#name: string + _#cmd: string +} diff --git a/cue/testdata/eval/github.txtar b/cue/testdata/eval/github.txtar index 97ab7295c..4595fcaba 100644 --- a/cue/testdata/eval/github.txtar +++ b/cue/testdata/eval/github.txtar @@ -31,6 +31,9 @@ workflows: [...{ }, { file: "rebuild_tip_cuelang_org.yml" schema: rebuild_tip_cuelang_org +}, { + file: "mirror.yml" + schema: mirror }] test: _#bashWorkflow & { name: "Test" @@ -129,20 +132,60 @@ test: _#bashWorkflow & { } } repository_dispatch: _#bashWorkflow & { + // These constants are defined by github.com/cue-sh/tools/cmd/cueckoo + _#runtrybot: "runtrybot" + _#mirror: "mirror" + _#importpr: "importpr" + _#dispatchJob: _#job & { + _#type: string + "runs-on": _#linuxMachine + if: "${{ github.event.client_payload.type == '\(_#type)' }}" + } name: "Repository Dispatch" on: ["repository_dispatch"] - jobs: start: { - if: "${{ startsWith(github.event.action, 'Build for refs/changes/') }}" + jobs: { + "\(_#runtrybot)": _#dispatchJob & { + _#type: _#runtrybot + steps: [_#step & { + name: "Trigger trybot" + run: """ + \(_#tempCueckooGitDir) + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }} + git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD + git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} + """ + }] + } + "\(_#mirror)": _#dispatchJob & { + _#type: _#mirror + steps: _#copybaraSteps & { + _ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + } + } + "\(_#importpr)": _#dispatchJob & { + _#type: _#importpr + steps: _#copybaraSteps & { + _ + _#name: "Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit" + _#cmd: "github-pr ${{ github.event.client_payload.payload.pr }}" + } + } + } +} +mirror: _#bashWorkflow & { + name: "Scheduled repo mirror" + on: schedule: [{ + cron: "*/30 * * * *" + }] + jobs: mirror: { "runs-on": _#linuxMachine - steps: [_#step & { - name: "Checkout ref" - run: """ - \(_#tempCueckooGitDir) - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD - git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} - """ - }] + steps: _#copybaraSteps & { + _ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + } } } release: _#bashWorkflow & { @@ -298,6 +341,40 @@ _#tempCueckooGitDir: """ git config user.email cueckoo@gmail.com git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" """ + +// The cueckoo/copybara Docker image to use +_#cueckooCopybaraImage: "cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c" + +// Define the base command for copybara +_#copybaraCmd: { + """ + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" \(_#cueckooCopybaraImage) bash -c " \\ + \tset -eu; \\ + \techo \\"${{ secrets.gerritCookie }}\\" > ~/.gitcookies; \\ + \tchmod 600 ~/.gitcookies; \\ + \tgit config --global user.name cueckoo; \\ + \tgit config --global user.email cueckoo@gmail.com; \\ + \tgit config --global http.cookiefile \\$HOME/.gitcookies; \\ + \techo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \\ + \tchmod 600 ~/.git-credentials; \\ + \tjava -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky \(_#cmd); \\ + \t" + """ + _#cmd: string +} +_#copybaraSteps: { + let cmdCmd = _#cmd + [_#checkoutCode, _#step & { + name: _#name + run: _#copybaraCmd & { + _ + _#cmd: cmdCmd + } + }] + _#name: string + _#cmd: string +} -- cue.mod/module.cue -- module: "mod.com" @@ -2047,6 +2124,10 @@ import "strings" file: "rebuild_tip_cuelang_org.yml" schema: 〈1;rebuild_tip_cuelang_org〉 }, + { + file: "mirror.yml" + schema: 〈1;mirror〉 + }, ]) test: (〈0;_#bashWorkflow〉 & { name: "Test" @@ -2164,23 +2245,66 @@ import "strings" } }) repository_dispatch: (〈0;_#bashWorkflow〉 & { + _#runtrybot: "runtrybot" + _#mirror: "mirror" + _#importpr: "importpr" + _#dispatchJob: (〈1;_#job〉 & { + _#type: string + "runs-on": 〈2;_#linuxMachine〉 + if: "${{ github.event.client_payload.type == '\(〈0;_#type〉)' }}" + }) name: "Repository Dispatch" on: [ "repository_dispatch", ] jobs: { - start: { - if: "${{ startsWith(github.event.action, 'Build for refs/changes/') }}" - "runs-on": 〈3;_#linuxMachine〉 + "\(〈1;_#runtrybot〉)": (〈1;_#dispatchJob〉 & { + _#type: 〈2;_#runtrybot〉 steps: [ (〈3;_#step〉 & { - name: "Checkout ref" + name: "Trigger trybot" run: "\(〈4;_#tempCueckooGitDir〉) - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD - git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }}" + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }} + git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD + git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }}" }), ] + }) + "\(〈1;_#mirror〉)": (〈1;_#dispatchJob〉 & { + _#type: 〈2;_#mirror〉 + steps: (〈3;_#copybaraSteps〉 & { + _ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + }) + }) + "\(〈1;_#importpr〉)": (〈1;_#dispatchJob〉 & { + _#type: 〈2;_#importpr〉 + steps: (〈3;_#copybaraSteps〉 & { + _ + _#name: "Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit" + _#cmd: "github-pr ${{ github.event.client_payload.payload.pr }}" + }) + }) + } + }) + mirror: (〈0;_#bashWorkflow〉 & { + name: "Scheduled repo mirror" + on: { + schedule: [ + { + cron: "*/30 * * * *" + }, + ] + } + jobs: { + mirror: { + "runs-on": 〈3;_#linuxMachine〉 + steps: (〈3;_#copybaraSteps〉 & { + _ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + }) } } }) @@ -2362,4 +2486,34 @@ import "strings" }) _#branchRefPrefix: "refs/heads/" _#tempCueckooGitDir: "mkdir tmpgit\ncd tmpgit\ngit init\ngit config user.name cueckoo\ngit config user.email cueckoo@gmail.com\ngit config http.https://github.com/.extraheader \"AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)\"" + _#cueckooCopybaraImage: "cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c" + _#copybaraCmd: { + "cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" \(〈1;_#cueckooCopybaraImage〉) bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky \(〈0;_#cmd〉); \ + "" + _#cmd: string + } + _#copybaraSteps: { + [ + 〈1;_#checkoutCode〉, + (〈1;_#step〉 & { + name: 〈1;_#name〉 + run: (〈2;_#copybaraCmd〉 & { + _ + _#cmd: 〈2;let cmdCmd〉 + }) + }), + ] + _#name: string + _#cmd: string + } } diff --git a/internal/ci/ci_tool.cue b/internal/ci/ci_tool.cue index ccb1e4887..ef8b8eb28 100644 --- a/internal/ci/ci_tool.cue +++ b/internal/ci/ci_tool.cue @@ -96,7 +96,8 @@ command: updateTxtarTests: { } } // When we have a solution for cuelang.org/issue/709 we can make this a - // file.Glob + // file.Glob. Ultimately it would be better to be able to do a cue def + // on the tool "package" readToolsFile: file.Read & { filename: "ci_tool.cue" contents: string diff --git a/internal/ci/workflows.cue b/internal/ci/workflows.cue index 12664cbcc..360482134 100644 --- a/internal/ci/workflows.cue +++ b/internal/ci/workflows.cue @@ -42,6 +42,10 @@ workflows: [ file: "rebuild_tip_cuelang_org.yml" schema: rebuild_tip_cuelang_org }, + { + file: "mirror.yml" + schema: mirror + }, ] test: _#bashWorkflow & { @@ -176,25 +180,66 @@ test: _#bashWorkflow & { } repository_dispatch: _#bashWorkflow & { + // These constants are defined by github.com/cue-sh/tools/cmd/cueckoo + _#runtrybot: "runtrybot" + _#mirror: "mirror" + _#importpr: "importpr" + + _#dispatchJob: _#job & { + _#type: string + "runs-on": _#linuxMachine + if: "${{ github.event.client_payload.type == '\(_#type)' }}" + } name: "Repository Dispatch" on: ["repository_dispatch"] jobs: { - start: { - if: "${{ startsWith(github.event.action, 'Build for refs/changes/') }}" - "runs-on": _#linuxMachine + "\(_#runtrybot)": _#dispatchJob & { + _#type: _#runtrybot steps: [ _#step & { - name: "Checkout ref" + name: "Trigger trybot" run: """ \(_#tempCueckooGitDir) - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD - git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.payload.ref }} + git checkout -b ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} FETCH_HEAD + git push https://github.com/cuelang/cue ci/${{ github.event.client_payload.payload.changeID }}/${{ github.event.client_payload.payload.commit }} """ }, ] } + "\(_#mirror)": _#dispatchJob & { + _#type: _#mirror + steps: _#copybaraSteps & {_ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + } + } + "\(_#importpr)": _#dispatchJob & { + _#type: _#importpr + steps: _#copybaraSteps & {_ + _#name: "Import PR #${{ github.event.client_payload.commit }} from GitHub to Gerrit" + _#cmd: "github-pr ${{ github.event.client_payload.payload.pr }}" + } + } + } +} + +mirror: _#bashWorkflow & { + name: "Scheduled repo mirror" + on: + schedule: [{ + cron: "*/30 * * * *" // every 30 mins + }] + + jobs: { + "mirror": { + "runs-on": _#linuxMachine + steps: _#copybaraSteps & {_ + _#name: "Mirror Gerrit to GitHub" + _#cmd: "github" + } + } } } @@ -364,3 +409,38 @@ _#tempCueckooGitDir: """ git config user.email cueckoo@gmail.com git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" """ + +// The cueckoo/copybara Docker image to use +_#cueckooCopybaraImage: "cueckoo/copybara:afc4ae03eed00b0c9d7415141cd1b5dfa583da7c" + +// Define the base command for copybara +_#copybaraCmd: { + _#cmd: string + #""" + cd _scripts + docker run --rm -v $PWD/cache:/root/copybara/cache -v $PWD:/usr/src/app --entrypoint="" \#(_#cueckooCopybaraImage) bash -c " \ + set -eu; \ + echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies; \ + chmod 600 ~/.gitcookies; \ + git config --global user.name cueckoo; \ + git config --global user.email cueckoo@gmail.com; \ + git config --global http.cookiefile \$HOME/.gitcookies; \ + echo https://cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }}@github.com > ~/.git-credentials; \ + chmod 600 ~/.git-credentials; \ + java -jar /opt/copybara/copybara_deploy.jar migrate copy.bara.sky \#(_#cmd); \ + " + """# +} + +_#copybaraSteps: { + _#name: string + _#cmd: string + let cmdCmd = _#cmd + [ + _#checkoutCode, // needed for copy.bara.sky file + _#step & { + name: _#name + run: _#copybaraCmd & {_, _#cmd: cmdCmd} + }, + ] +}