feat(apps/prod/tekton/configs/triggers): support build with pr (#1128)

feature: support build with pull request for TiBuild dev-builds, also passing the different registry to avoid mixed in the product repositories. Signed-off-by: wuhuizuo <[email protected]> --------- Signed-off-by: wuhuizuo <[email protected]>
PingCAP-QE · Apr 30, 2024 · 37b3bff · 37b3bff
1 parent 3ab3f8b
commit 37b3bff
Show file tree

Hide file tree

Showing 46 changed files with 289 additions and 1,074 deletions.
diff --git a/apps/prod/tekton/configs/pipelines/pingcap-build-package-darwin.yaml b/apps/prod/tekton/configs/pipelines/pingcap-build-package-darwin.yaml
@@ -9,6 +9,9 @@ spec:
     - name: git-ref
       type: string
       default: master
+    - name: git-refspec
+      default: +refs/heads/*:refs/heads/*
+      type: string
     - name: git-revision
       default: master
       type: string
@@ -31,6 +34,9 @@ spec:
       default: release
     - name: push
       default: "true"
+    - name: registry
+      description: the base OCI registry server for store artifacts, it can be set with prefix repo path.
+      default: "hub.pingcap.net"
   results:
     - name: pushed-binaries
       description: pushed binaries.
@@ -58,7 +64,7 @@ spec:
         - name: depth
           value: "0"
         - name: refspec
-          value: +refs/heads/*:refs/heads/*
+          value: $(params.git-refspec)
         - name: subdirectory
           value: $(params.component)
       workspaces:
@@ -147,6 +153,8 @@ spec:
           value: build
         - name: push
           value: "$(params.push)"
+        - name: registry
+          value: "$(params.registry)"
         - name: mac-builder-resource
           value: "$(tasks.acquire-mac-machine.results.leased-resource)"
       workspaces:

diff --git a/apps/prod/tekton/configs/pipelines/pingcap-build-package.yaml b/apps/prod/tekton/configs/pipelines/pingcap-build-package.yaml
@@ -9,6 +9,9 @@ spec:
     - name: git-ref
       type: string
       default: master
+    - name: git-refspec
+      default: +refs/heads/*:refs/heads/*
+      type: string
     - name: git-revision
       default: master
       type: string
@@ -33,6 +36,9 @@ spec:
       default: release
     - name: push
       default: "true"
+    - name: registry
+      description: the base OCI registry server for store artifacts, it can be set with prefix repo path.
+      default: "hub.pingcap.net"
   results:
     - name: pushed-binaries
       description: pushed binaries.
@@ -64,7 +70,7 @@ spec:
         - name: depth
           value: "0"
         - name: refspec
-          value: +refs/heads/*:refs/heads/*
+          value: $(params.git-refspec)
         - name: subdirectory
           value: $(params.component)
       workspaces:
@@ -117,7 +123,6 @@ spec:
         - checkout-ext
       taskRef:
         name: pingcap-build-binaries
-      # TODO: currently matrix feature is not support in v0.32.x, we need upgrade the K8S cluster and then upgrade Tekton.
       params:
         - name: os
           value: "$(params.os)"
@@ -139,6 +144,8 @@ spec:
           value: build
         - name: push
           value: "$(params.push)"
+        - name: registry
+          value: "$(params.registry)"
       workspaces:
         - name: source
           workspace: source
@@ -175,6 +182,8 @@ spec:
           value: build
         - name: build
           value: "false"
+        - name: registry
+          value: "$(params.registry)"
       workspaces:
         - name: source
           workspace: source

diff --git a/apps/prod/tekton/configs/tasks/pingcap-build-binaries-darwin.yaml b/apps/prod/tekton/configs/tasks/pingcap-build-binaries-darwin.yaml
@@ -44,6 +44,8 @@ spec:
     - name: push
       type: string
       default: 'false'
+    - name: registry
+      default: hub.pingcap.net
     - name: mac-builder-resource
       type: string
   steps:
@@ -68,7 +70,8 @@ spec:
           $git_ref \
           "$git_sha" \
           /workspace/artifacts/packages/packages.yaml.tmpl \
-          "$out_script"
+          "$out_script" \
+          $(params.registry)
 
         if [ -f "$out_script" ]; then
           cat "$out_script"
@@ -136,7 +139,7 @@ spec:
         - name: REMOTE_BUILDER_INFO_DIR
           value: /workspace/remote-builder
       workingDir: $(workspaces.source.path)/$(params.component)
-      # TODO: get the user and host by better way. 
+      # TODO: get the user and host by better way.
       script: |
         #!/usr/bin/env bash
         set -exo pipefail
@@ -146,7 +149,7 @@ spec:
           echo "No build script, skip build."
           exit 0
         fi
-        
+
         ##### Build mac targets in tekton task (k8s pod) ####
         # 1. Set ssh credentials
         if [ "${WORKSPACE_SSH_DIRECTORY_BOUND}" != "true" ] ; then
@@ -167,7 +170,7 @@ spec:
         # 2.1 create a randon workspace dir in the remote host:
         remote_workspace_dir="${workspace_base_dir}/$(context.taskRun.name)"
         ssh ${username}@${host} "mkdir -p '$remote_workspace_dir'"
-        
+
         # 2.2 copy the build shell to the remote host by ssh.
         scp "$script" /workspace/remote.env ${username}@${host}:"$remote_workspace_dir/"
 

diff --git a/apps/prod/tekton/configs/tasks/pingcap-build-binaries.yaml b/apps/prod/tekton/configs/tasks/pingcap-build-binaries.yaml
@@ -47,6 +47,8 @@ spec:
     - name: push
       type: string
       default: 'false'
+    - name: registry
+      default: hub.pingcap.net
     - name: mac-builder-resource
       type: string
       default: ''
@@ -72,7 +74,8 @@ spec:
           $git_ref \
           "$git_sha" \
           /workspace/artifacts/packages/packages.yaml.tmpl \
-          "$out_script"
+          "$out_script" \
+          $(params.registry)
 
         if [ -f "$out_script" ]; then
           cat "$out_script"

diff --git a/apps/prod/tekton/configs/tasks/pingcap-build-images.yaml b/apps/prod/tekton/configs/tasks/pingcap-build-images.yaml
@@ -40,6 +40,8 @@ spec:
     - name: build
       default: 'false'
       description: Does it need to build the binaries before packing the images?
+    - name: registry
+      default: hub.pingcap.net
   steps:
     - name: generate
       image: ghcr.io/pingcap-qe/cd/utils/release:v20231216-37-g8e0ca7e
@@ -62,7 +64,8 @@ spec:
           $git_ref \
           "$git_sha" \
           /workspace/artifacts/packages/packages.yaml.tmpl \
-          "$out_script" || true
+          "$out_script" \
+          $(params.registry) || true
 
         if [ -f "$out_script" ]; then
           cat "$out_script"

diff --git a/apps/prod/tekton/configs/tasks/pingcap-compose-offline-pkgs.yaml b/apps/prod/tekton/configs/tasks/pingcap-compose-offline-pkgs.yaml
@@ -29,13 +29,11 @@ spec:
     - name: push
       type: string
       default: 'false'
-    - name: tiup-tool-version
-      default: "v1.14.1"
     - name: tiup-mirror
-      description: Product is http://tiup.pingcap.net:8987, Staging is http://tiup.pingcap.net:8988, 
+      description: Product is http://tiup.pingcap.net:8987, Staging is http://tiup.pingcap.net:8988,
       default: http://tiup.pingcap.net:8988
   stepTemplate:
-    image: ghcr.io/pingcap-qe/cd/utils/release:v20231216-37-g8e0ca7e
+    image: ghcr.io/pingcap-qe/cd/utils/release:v20240325-14-gdd4a4cf
     env:
       - name: OUT_SCRIPT
         value: /workspace/compose-offline-packages-artifacts.sh
@@ -55,7 +53,8 @@ spec:
           $(params.edition) \
           /workspace/artifacts/packages/offline-packages.yaml.tmpl \
           "$OUT_SCRIPT" \
-          $(params.tiup-mirror)
+          $(params.tiup-mirror) \
+          $(params.registry)
 
         if [ -f "$OUT_SCRIPT" ]; then
           cat "$OUT_SCRIPT"
@@ -72,19 +71,10 @@ spec:
         #! /usr/bin/env bash
         set -exo pipefail
 
-        # install tiup tool.
-        TIUP_HOME=~/.tiup
-        bin_dir=$TIUP_HOME/bin
-        mkdir -vp "$bin_dir"
-        ARCH=$([ "$(arch)" = "x86_64" ] && echo amd64 || echo arm64)
-        download_url=https://tiup-mirrors.pingcap.com/tiup-$(params.tiup-tool-version)-linux-${ARCH}.tar.gz
-        wget -q -O - "$download_url" | tar -zxvf - -C "$bin_dir"
-        chmod 755 "$bin_dir/tiup"
-        export PATH=$bin_dir:$PATH
-        which tiup
+        mkdir -vp ~/.tiup/{keys,bin}
 
         # set mirror.
-        tiup mirror set $(params.tiup-mirror)
+        tiup -v && tiup mirror set $(params.tiup-mirror)
 
         if $(params.push); then
           "$OUT_SCRIPT" -c -r -w build -p -o $(results.pushed.path)

diff --git a/apps/prod/tekton/configs/tasks/publish-tiup-from-oci-artifact.yaml b/apps/prod/tekton/configs/tasks/publish-tiup-from-oci-artifact.yaml
@@ -16,8 +16,6 @@ spec:
     - name: nightly
       description: publish as nightly version.
       default: "false"
-    - name: tiup-tool-version
-      default: "v1.14.1"
     - name: tiup-mirror
       description: Staging is http://tiup.pingcap.net:8988, product is http://tiup.pingcap.net:8987.
       default: http://tiup.pingcap.net:8988
@@ -64,7 +62,7 @@ spec:
         cp $(workspaces.tiup-keys.path)/private.json $TIUP_HOME/keys/
 
         # set mirror.
-        tiup mirror set $(params.tiup-mirror)
+        tiup -v && tiup mirror set $(params.tiup-mirror)
 
         # publish the tiup packages
         chmod +x publish.sh && ./publish.sh

diff --git a/apps/prod/tekton/configs/triggers/bindings/github-branch-push.yaml b/apps/prod/tekton/configs/triggers/bindings/github-branch-push.yaml
@@ -6,6 +6,8 @@ spec:
   params:
     - name: git-url
       value: $(body.repository.clone_url)
+    - name: git-refspec
+      value: "+$(body.ref):$(body.ref)"
     - name: git-revision
       value: $(body.after)
     - name: git-ref

diff --git a/apps/prod/tekton/configs/triggers/bindings/github-pr.yaml b/apps/prod/tekton/configs/triggers/bindings/github-pr.yaml
@@ -5,26 +5,32 @@ metadata:
 spec:
   params:
     - name: git-url
-      value: $(body.pull_request.head.repo.clone_url)
+      value: $(body.pull_request.base.repo.clone_url)
+    - name: git-refspec
+      value: "refs/pull/$(body.pull_request.number)/head"
     - name: git-revision
       value: $(body.pull_request.head.sha)
+    - name: git-ref
+      value: $(body.pull_request.base.ref)
     - name: pr-number
       value: $(body.pull_request.number)
-    - name: pr-title
-      value: $(body.pull_request.title)
-    - name: pr-body
-      value: $(body.pull_request.body)
-    - name: pr-user
-      value: $(body.pull_request.user.login)
-    - name: pr-head-owner
-      value: $(body.pull_request.head.user.login)
-    - name: pr-head-repo
-      value: $(body.pull_request.head.repo.name)
     - name: pr-head-ref
       value: $(body.pull_request.head.ref)
-    - name: pr-base-owner
-      value: $(body.pull_request.base.user.login)
-    - name: pr-base-repo
-      value: $(body.pull_request.base.repo.name)
     - name: pr-base-ref
       value: $(body.pull_request.base.ref)
+    # - name: pr-title
+    #   value: $(body.pull_request.title)
+    # - name: pr-body
+    #   value: $(body.pull_request.body)
+    # - name: pr-user
+    #   value: $(body.pull_request.user.login)
+    # - name: pr-head-owner
+    #   value: $(body.pull_request.head.user.login)
+    # - name: pr-head-repo
+    #   value: $(body.pull_request.head.repo.name)
+    # - name: pr-base-owner
+    #   value: $(body.pull_request.base.user.login)
+    # - name: pr-base-repo
+    #   value: $(body.pull_request.base.repo.name)
+    # - name: pr-base-sha
+    #   value: $(body.pull_request.base.sha)
diff --git a/apps/prod/tekton/configs/triggers/event-listeners/event-listener-tibuild.yaml b/apps/prod/tekton/configs/triggers/event-listeners/event-listener-tibuild.yaml
@@ -6,30 +6,42 @@ metadata:
 spec:
   serviceAccountName: tekton-trigger-eventlistener
   triggerGroups:
-    - name: tibuild-fake-github-branch-push
+    - name: tibuild-fake-github-pr
       interceptors:
         - name: "filter on eventType"
           ref:
             name: cel
           params:
             - name: filter
               value: >-
-                header.canonical('ce-type') in ['net.pingcap.tibuild.devbuild.push', 'net.pingcap.tibuild.hotfix.push']
-        # "refs/heads/main" => "main"
-        - name: shortten the git REF
+                header.canonical('ce-type') in ['net.pingcap.tibuild.devbuild.pull_request', 'net.pingcap.tibuild.hotfix.pull_request']
+        - name: "validate PR state and add field"
           ref:
             name: cel
           params:
-            - name: overlays
-              value:
-                - key: short-ref
-                  expression: body.ref.replace('refs/heads/', '')
-        - name: extract cloud event context.
+            - name: filter
+              value: body.action in ['opened', 'synchronize', 'reopened']
+      triggerSelector:
+        namespaceSelector:
+          matchNames:
+            - ee-cd
+        labelSelector:
+          matchLabels:
+            type: fake-github-pr
+    - name: tibuild-fake-github-branch-push
+      interceptors:
+        - name: filter on event type and extract cloud event context
           ref:
             name: cel
           params:
+            - name: filter
+              value: >-
+                header.canonical('ce-type') in ['net.pingcap.tibuild.devbuild.push', 'net.pingcap.tibuild.hotfix.push']
             - name: overlays
               value:
+                - key: short-ref
+                  # "refs/heads/main" => "main"
+                  expression: body.ref.replace('refs/heads/', '')
                 - key: ce-context
                   expression: |
                     {
@@ -49,7 +61,7 @@ spec:
             type: fake-github-branch-push
     - name: tibuild-fake-github-tag-create
       interceptors:
-        - name: "filter on eventType and refs"
+        - name: filter on eventType and refs and extract cloud event context
           ref:
             name: cel
           params:
@@ -58,6 +70,18 @@ spec:
                 header.canonical('ce-type') in ['net.pingcap.tibuild.devbuild.create', 'net.pingcap.tibuild.hotfix.create']
                 &&
                 body.ref_type == 'tag'
+            - name: overlays
+              value:
+                - key: ce-context
+                  expression: |
+                    {
+                      "type": header.canonical('ce-type'),
+                      "source": header.canonical('ce-source'),
+                      "subject": header.canonical('ce-subject'),
+                      "user": header.canonical('ce-user'),
+                    }.marshalJSON()
+                - key: ce-context-user
+                  expression: header.canonical('ce-user')
       triggerSelector:
         namespaceSelector:
           matchNames: