diff --git a/.github/workflows/rebuild_tip_cuelang_org.yml b/.github/workflows/rebuild_tip_cuelang_org.yml index 97ce23743..3e5651b39 100644 --- a/.github/workflows/rebuild_tip_cuelang_org.yml +++ b/.github/workflows/rebuild_tip_cuelang_org.yml @@ -7,7 +7,10 @@ on: - master jobs: push: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash steps: - name: Rebuild tip.cuelang.org run: curl -f -X POST -d {} https://api.netlify.com/build_hooks/${{ secrets.CuelangOrgTipRebuildHook diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3342ba007..912e2ec4f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,10 @@ on: - v* jobs: goreleaser: - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash steps: - name: Checkout code uses: actions/checkout@v2 @@ -21,7 +24,10 @@ jobs: args: release --rm-dist docker: name: docker - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash steps: - name: Check out the repo uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fa75d658f..7ddeb84aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,12 +4,9 @@ name: Test on: push: branches: - - '*' + - '**' tags-ignore: - v* -defaults: - run: - shell: bash jobs: test: strategy: @@ -20,11 +17,16 @@ jobs: - 1.14.9 - 1.15.x os: - - ubuntu-latest - - macos-latest - - windows-latest + - ubuntu-18.04 + - macos-10.15 + - windows-2019 runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash steps: + - name: Write the gitcookies file + run: echo "${{ secrets.gerritCookie }}" > ~/.gitcookies - name: Install Go uses: actions/setup-go@v2 with: @@ -38,7 +40,7 @@ jobs: key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} restore-keys: ${{ runner.os }}-${{ matrix.go-version }}-go- - - if: matrix.go-version == '1.14.9' && matrix.os != 'windows-latest' + - if: matrix.go-version == '1.14.9' && matrix.os != 'windows-2019' name: Generate run: go generate ./... - name: Test @@ -49,10 +51,41 @@ jobs: run: go run golang.org/x/exp/cmd/gorelease - name: Check that git is clean post generate and tests run: test -z "$(git status --porcelain)" || (git status; git diff; false) - - if: github.ref == 'refs/heads/master' + - if: ${{ github.ref == 'refs/heads/master' }} name: Pull this commit through the proxy on master run: |- v=$(git rev-parse HEAD) cd $(mktemp -d) go mod init mod.com GOPROXY=https://proxy.golang.org go get -d cuelang.org/go@$v + - if: ${{ startsWith(github.ref, 'refs/heads/ci/') && failure() }} + name: Post any failures for this matrix entry + run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build + failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url + }}/actions/runs/${{ github.run_id }} for more details","labels":{"Code-Review":-1}}'' + -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$(basename + $(dirname $GITHUB_REF))/revisions/$(basename $GITHUB_REF)/review' + mark_ci_success: + runs-on: ubuntu-18.04 + if: ${{ startsWith(github.ref, 'refs/heads/ci/') }} + needs: test + defaults: + run: + shell: bash + steps: + - name: Write the gitcookies file + run: echo "${{ secrets.gerritCookie }}" > ~/.gitcookies + - name: Update Gerrit CL message with success message + run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build + succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id + }}","labels":{"Code-Review":1}}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$(basename + $(dirname $GITHUB_REF))/revisions/$(basename $GITHUB_REF)/review' + delete_build_branch: + runs-on: ubuntu-18.04 + if: ${{ startsWith(github.ref, 'refs/heads/ci/') && always() }} + needs: test + defaults: + run: + shell: bash + steps: + - run: echo git push origin :${GITHUB_REF#refs/heads/} diff --git a/.github/workflows/test_dispatch.yml b/.github/workflows/test_dispatch.yml index 0aa8f64eb..ee18a3f41 100644 --- a/.github/workflows/test_dispatch.yml +++ b/.github/workflows/test_dispatch.yml @@ -1,86 +1,29 @@ # Generated by internal/ci/ci_tool.cue; do not edit name: Test Dispatch -env: - GERRIT_COOKIE: ${{ secrets.gerritCookie }} on: - repository_dispatch -defaults: - run: - shell: bash jobs: start: - runs-on: ubuntu-latest + if: ${{ startsWith(github.event.action, 'Build for refs/changes/') }} + runs-on: ubuntu-18.04 + defaults: + run: + shell: bash steps: - name: Write the gitcookies file - run: echo "$GERRIT_COOKIE" > ~/.gitcookies + run: echo "${{ secrets.gerritCookie }}" > ~/.gitcookies - name: Update Gerrit CL message with starting message run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Started the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}"}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit }}/review' - test: - runs-on: ${{ matrix.os }} - steps: - - name: Write the gitcookies file - run: echo "$GERRIT_COOKIE" > ~/.gitcookies - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Checkout code - uses: actions/checkout@v2 - name: Checkout ref run: |- + git config --global user.name cueckoo + git config --global user.email cueckoo@gmail.com + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout FETCH_HEAD - - name: Cache Go modules - uses: actions/cache@v1 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') - }} - restore-keys: ${{ runner.os }}-${{ matrix.go-version }}-go- - - if: matrix.go-version == '1.14.9' && matrix.os != 'windows-latest' - name: Generate - run: go generate ./... - - name: Test - run: go test ./... - - name: Test with -race - run: go test -race ./... - - name: gorelease check - run: go run golang.org/x/exp/cmd/gorelease - - name: Check that git is clean post generate and tests - run: test -z "$(git status --porcelain)" || (git status; git diff; false) - - if: ${{ failure() }} - name: Post any failures for this matrix entry - run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build - failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url - }}/actions/runs/${{ github.run_id }} for more details","labels":{"Code-Review":-1}}'' - -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ github.event.client_payload.changeID - }}/revisions/${{ github.event.client_payload.commit }}/review' - needs: start - strategy: - fail-fast: false - matrix: - go-version: - - 1.13.x - - 1.14.9 - - 1.15.x - os: - - ubuntu-latest - - macos-latest - - windows-latest - end: - runs-on: ubuntu-latest - steps: - - name: Write the gitcookies file - run: echo "$GERRIT_COOKIE" > ~/.gitcookies - - name: Update Gerrit CL message with success message - run: 'curl -f -s -H "Content-Type: application/json" --request POST --data ''{"message":"Build - succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id - }}","labels":{"Code-Review":1}}'' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ - github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit - }}/review' - needs: test + git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD + git push origin ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} diff --git a/internal/ci/workflows.cue b/internal/ci/workflows.cue index 268e2ae88..3eeb62cc6 100644 --- a/internal/ci/workflows.cue +++ b/internal/ci/workflows.cue @@ -27,130 +27,103 @@ workflows: [ }, ] -// TODO: drop when cuelang.org/issue/390 is fixed. -// Declare definitions for sub-schemas -#job: ((json.#Workflow & {}).jobs & {x: _}).x -#step: ((#job & {steps: _}).steps & [_])[0] - -// We need at least go1.14 for code generation -#codeGenGo: "1.14.9" - -#testStrategy: { - "fail-fast": false - matrix: { - // Use a stable version of 1.14.x for go generate - "go-version": ["1.13.x", #codeGenGo, "1.15.x"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - } -} - -#installGo: #step & { - name: "Install Go" - uses: "actions/setup-go@v2" - with: "go-version": "${{ matrix.go-version }}" -} - -#checkoutCode: #step & { - name: "Checkout code" - uses: "actions/checkout@v2" -} - -#cacheGoModules: #step & { - name: "Cache Go modules" - uses: "actions/cache@v1" - with: { - path: "~/go/pkg/mod" - key: "${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}" - "restore-keys": """ - ${{ runner.os }}-${{ matrix.go-version }}-go- - """ - } - -} - -#goGenerate: #step & { - name: "Generate" - run: "go generate ./..." - // The Go version corresponds to the precise version specified in - // the matrix. Skip windows for now until we work out why re-gen is flaky - if: "matrix.go-version == '\(#codeGenGo)' && matrix.os != 'windows-latest'" -} - -#goTest: #step & { - name: "Test" - run: "go test ./..." -} - -#goTestRace: #step & { - name: "Test with -race" - run: "go test -race ./..." -} - -#goReleaseCheck: #step & { - name: "gorelease check" - run: "go run golang.org/x/exp/cmd/gorelease" -} - -#checkGitClean: #step & { - name: "Check that git is clean post generate and tests" - run: "test -z \"$(git status --porcelain)\" || (git status; git diff; false)" -} - -#pullThroughProxy: #step & { - name: "Pull this commit through the proxy on master" - run: """ - v=$(git rev-parse HEAD) - cd $(mktemp -d) - go mod init mod.com - GOPROXY=https://proxy.golang.org go get -d cuelang.org/go@$v - """ - if: "github.ref == 'refs/heads/master'" -} - -test: { - json.#Workflow +test: _#bashWorkflow & { name: "Test" on: { push: { - branches: ["*"] + branches: ["**"] // any branch (including '/' namespaced branches) "tags-ignore": ["v*"] } } - defaults: run: shell: "bash" - jobs: test: { - strategy: #testStrategy - "runs-on": "${{ matrix.os }}" - steps: [ - #installGo, - #checkoutCode, - #cacheGoModules, - #goGenerate, - #goTest, - #goTestRace, - #goReleaseCheck, - #checkGitClean, - #pullThroughProxy, - ] + + jobs: { + test: { + strategy: _#testStrategy + "runs-on": "${{ matrix.os }}" + steps: [ + _#writeCookiesFile, + _#installGo, + _#checkoutCode, + _#cacheGoModules, + _#goGenerate, + _#goTest, + _#goTestRace, + _#goReleaseCheck, + _#checkGitClean, + _#pullThroughProxy, + _#failCLBuild, + ] + } + mark_ci_success: { + "runs-on": _#linuxMachine + if: "${{ \(_#isCLCITestBranch) }}" + needs: "test" + steps: [ + _#writeCookiesFile, + _#passCLBuild, + ] + } + delete_build_branch: { + "runs-on": _#linuxMachine + if: "${{ \(_#isCLCITestBranch) && always() }}" + needs: "test" + steps: [ + _#step & { + run: "echo git push origin :${GITHUB_REF#\(_#branchRefPrefix)}" + }, + ] + } } -} -test_dispatch: { - json.#Workflow + // _#isCLCITestBranch is an expression that evaluates to true + // if the job is running as a result of a CL triggered CI build + _#isCLCITestBranch: "startsWith(github.ref, '\(_#branchRefPrefix)ci/')" + + // _#isMaster is an expression that evaluates to true if the + // job is running as a result of a master commit push + _#isMaster: "github.ref == '\(_#branchRefPrefix)master'" - #checkoutRef: #step & { - name: "Checkout ref" + _#pullThroughProxy: _#step & { + name: "Pull this commit through the proxy on master" run: """ - git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} - git checkout FETCH_HEAD + v=$(git rev-parse HEAD) + cd $(mktemp -d) + go mod init mod.com + GOPROXY=https://proxy.golang.org go get -d cuelang.org/go@$v """ + if: "${{ \(_#isMaster) }}" } - #writeCookiesFile: #step & { - name: "Write the gitcookies file" - run: "echo \"$GERRIT_COOKIE\" > ~/.gitcookies" + + _#failCLBuild: _#step & { + if: "${{ \(_#isCLCITestBranch) && failure() }}" + name: "Post any failures for this matrix entry" + run: (_#gerrit._#setCodeReview & { + #args: { + message: "Build failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} for more details" + labels: { + "Code-Review": -1 + } + } + }).res + } + + _#passCLBuild: _#step & { + name: "Update Gerrit CL message with success message" + run: (_#gerrit._#setCodeReview & { + #args: { + message: "Build succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" + labels: { + "Code-Review": 1 + } + } + }).res } - #gerrit: { - #setCodeReview: { + + _#gerrit: { + // _#setCodeReview assumes that it is invoked from a job where + // _#isCLCITestBranch is true + _#setCodeReview: { #args: { message: string labels?: { @@ -158,83 +131,65 @@ test_dispatch: { } } res: #""" - curl -f -s -H "Content-Type: application/json" --request POST --data '\#(encjson.Marshal(#args))' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit }}/review + curl -f -s -H "Content-Type: application/json" --request POST --data '\#(encjson.Marshal(#args))' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/$(basename $(dirname $GITHUB_REF))/revisions/$(basename $GITHUB_REF)/review """# } } +} + +test_dispatch: _#bashWorkflow & { name: "Test Dispatch" - env: GERRIT_COOKIE: "${{ secrets.gerritCookie }}" on: ["repository_dispatch"] - defaults: run: shell: "bash" jobs: { start: { - "runs-on": "ubuntu-latest" + if: "${{ startsWith(github.event.action, 'Build for refs/changes/') }}" + "runs-on": _#linuxMachine steps: [ - #writeCookiesFile, - #step & { + _#writeCookiesFile, + _#step & { name: "Update Gerrit CL message with starting message" - run: (#gerrit.#setCodeReview & { + run: (_#gerrit._#setCodeReview & { #args: message: "Started the build... see progress at ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" }).res }, - ] - } - test: { - "runs-on": "${{ matrix.os }}" - steps: [ - #writeCookiesFile, - #installGo, - #checkoutCode, - #checkoutRef, - #cacheGoModules, - #goGenerate, - #goTest, - #goTestRace, - #goReleaseCheck, - #checkGitClean, - #step & { - if: "${{ failure() }}" - name: "Post any failures for this matrix entry" - run: (#gerrit.#setCodeReview & { - #args: { - message: "Build failed for ${{ runner.os }}-${{ matrix.go-version }}; see ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }} for more details" - labels: { - "Code-Review": -1 - } - } - }).res + _#step & { + name: "Checkout ref" + run: """ + git config --global user.name cueckoo + git config --global user.email cueckoo@gmail.com + git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n cueckoo:${{ secrets.CUECKOO_GITHUB_PAT }} | base64)" + git fetch https://cue-review.googlesource.com/cue ${{ github.event.client_payload.ref }} + git checkout -b ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} FETCH_HEAD + git push origin ci/${{ github.event.client_payload.changeID }}/${{ github.event.client_payload.commit }} + """ }, ] - needs: "start" - strategy: #testStrategy } - end: { - "runs-on": "ubuntu-latest" - steps: [ - #writeCookiesFile, - #step & { - name: "Update Gerrit CL message with success message" - run: (#gerrit.#setCodeReview & { - #args: { - message: "Build succeeded for ${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" - labels: { - "Code-Review": 1 - } - } - }).res - }, - ] - needs: "test" + } + + _#gerrit: { + _#setCodeReview: { + #args: { + message: string + labels?: { + "Code-Review": int + } + } + res: #""" + curl -f -s -H "Content-Type: application/json" --request POST --data '\#(encjson.Marshal(#args))' -b ~/.gitcookies https://cue-review.googlesource.com/a/changes/${{ github.event.client_payload.changeID }}/revisions/${{ github.event.client_payload.commit }}/review + """# } } } -release: { + +release: _#bashWorkflow & { + name: "Release" on: push: tags: ["v*"] jobs: { goreleaser: { - "runs-on": "ubuntu-latest" + "runs-on": _#linuxMachine steps: [{ name: "Checkout code" uses: "actions/checkout@v2" @@ -250,7 +205,7 @@ release: { } docker: { name: "docker" - "runs-on": "ubuntu-latest" + "runs-on": _#linuxMachine steps: [{ name: "Check out the repo" uses: "actions/checkout@v2" @@ -286,16 +241,98 @@ release: { } } -rebuild_tip_cuelang_org: { - json.#Workflow +rebuild_tip_cuelang_org: _#bashWorkflow & { name: "Push to tip" on: push: branches: ["master"] jobs: push: { - "runs-on": "ubuntu-latest" + "runs-on": _#linuxMachine steps: [{ name: "Rebuild tip.cuelang.org" run: "curl -f -X POST -d {} https://api.netlify.com/build_hooks/${{ secrets.CuelangOrgTipRebuildHook }}" }] } } + +_#bashWorkflow: json.#Workflow & { + jobs: [string]: defaults: run: shell: "bash" +} + +// TODO: drop when cuelang.org/issue/390 is fixed. +// Declare definitions for sub-schemas +_#job: ((json.#Workflow & {}).jobs & {x: _}).x +_#step: ((_#job & {steps: _}).steps & [_])[0] + +// We need at least go1.14 for code generation +_#codeGenGo: "1.14.9" + +_#linuxMachine: "ubuntu-18.04" +_#macosMachine: "macos-10.15" +_#windowsMachine: "windows-2019" + +_#testStrategy: { + "fail-fast": false + matrix: { + // Use a stable version of 1.14.x for go generate + "go-version": ["1.13.x", _#codeGenGo, "1.15.x"] + os: [_#linuxMachine, _#macosMachine, _#windowsMachine] + } +} + +_#installGo: _#step & { + name: "Install Go" + uses: "actions/setup-go@v2" + with: "go-version": "${{ matrix.go-version }}" +} + +_#checkoutCode: _#step & { + name: "Checkout code" + uses: "actions/checkout@v2" +} + +_#cacheGoModules: _#step & { + name: "Cache Go modules" + uses: "actions/cache@v1" + with: { + path: "~/go/pkg/mod" + key: "${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}" + "restore-keys": """ + ${{ runner.os }}-${{ matrix.go-version }}-go- + """ + } +} + +_#goGenerate: _#step & { + name: "Generate" + run: "go generate ./..." + // The Go version corresponds to the precise version specified in + // the matrix. Skip windows for now until we work out why re-gen is flaky + if: "matrix.go-version == '\(_#codeGenGo)' && matrix.os != '\(_#windowsMachine)'" +} + +_#goTest: _#step & { + name: "Test" + run: "go test ./..." +} + +_#goTestRace: _#step & { + name: "Test with -race" + run: "go test -race ./..." +} + +_#goReleaseCheck: _#step & { + name: "gorelease check" + run: "go run golang.org/x/exp/cmd/gorelease" +} + +_#checkGitClean: _#step & { + name: "Check that git is clean post generate and tests" + run: "test -z \"$(git status --porcelain)\" || (git status; git diff; false)" +} + +_#writeCookiesFile: _#step & { + name: "Write the gitcookies file" + run: "echo \"${{ secrets.gerritCookie }}\" > ~/.gitcookies" +} + +_#branchRefPrefix: "refs/heads/"