feat(dockerfiles/cd/builders/tikv): support custom devtoolset version (…

…#323) Signed-off-by: wuhuizuo <[email protected]> --------- Signed-off-by: wuhuizuo <[email protected]>
PingCAP-QE · Jun 7, 2024 · 9d496cf · 9d496cf
1 parent dcc3884
commit 9d496cf
Show file tree

Hide file tree

Showing 4 changed files with 221 additions and 39 deletions.
diff --git a/.github/workflows/pull-cd-builder-images.yaml b/.github/workflows/pull-cd-builder-images.yaml
@@ -13,7 +13,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  skaffold:
+  skaffold-go:
     name: build images with skaffold
     runs-on: ubuntu-latest
 
@@ -23,26 +23,16 @@ jobs:
 
     strategy:
       matrix:
-        module: [builder-go, builder-others, builder-others-fips, builder-others-slow]
-        go-profile: [go-1.21, go-1.20, go-1.19]
+        module: [builder-go]
+        go-profile: [go-1.22, go-1.21, go-1.20, go-1.19, go-1.18]
         builder-profile: [local-docker]
         platform: [linux/amd64, linux/arm64]
-        exclude:
-          - module: builder-others
-            go-profile: go-1.20
-          - module: builder-others
-            go-profile: go-1.19
-          - module: builder-others-fips
-            go-profile: go-1.20
-          - module: builder-others-fips
-            go-profile: go-1.19
-          - module: builder-others-slow
-            go-profile: go-1.20
-          - module: builder-others-slow
-            go-profile: go-1.19
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+          fetch-tags: 'true'
 
       # https://github.com/docker/setup-qemu-action
       - name: Set up QEMU
@@ -84,3 +74,83 @@ jobs:
             --platform ${{ matrix.platform }} \
             --profile ${{ matrix.go-profile }},${{ matrix.builder-profile }} \
             --push=false
+
+  skaffold-nogo:
+    name: build images with skaffold
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      matrix:
+        module: [builder-others, builder-tikv, builder-tikv-fips, builder-others-slow]
+        platform: [linux/amd64, linux/arm64]
+        builder-profile: [local-docker]
+        include:
+          - module: builder-tikv
+            builder-profile: devtoolset8,local-docker
+            platform: linux/amd64
+          - module: builder-tikv
+            builder-profile: devtoolset9,local-docker
+            platform: linux/amd64
+          - module: builder-tikv
+            builder-profile: devtoolset11,local-docker
+            platform: linux/amd64
+          - module: builder-tikv
+            builder-profile: devtoolset8,local-docker
+            platform: linux/arm64
+          - module: builder-tikv
+            builder-profile: devtoolset9,local-docker
+            platform: linux/arm64
+          - module: builder-tikv
+            builder-profile: devtoolset11,local-docker
+            platform: linux/arm64
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+          fetch-tags: 'true'
+
+      # https://github.com/docker/setup-qemu-action
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
+
+      - name: Cache layers
+        uses: actions/cache@v4
+        with:
+          path: "~/.skaffold/cache"
+          key: skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}-${{ github.sha }}
+          restore-keys: |
+            skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}-
+            skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}/
+
+      - name: Build images
+        working-directory: dockerfiles/cd/builders
+        run: |
+          skaffold build \
+            --build-concurrency 1 \
+            --cache-artifacts \
+            --default-repo ghcr.io/pingcap-qe/cd/builders \
+            --module ${{ matrix.module }} \
+            --platform ${{ matrix.platform }} \
+            --profile ${{ matrix.builder-profile }} \
+            --push=false
diff --git a/.github/workflows/release-cd-builder-images.yaml b/.github/workflows/release-cd-builder-images.yaml
@@ -10,7 +10,7 @@ on:
       - "dockerfiles/cd/builders/skaffold.yaml"
 
 jobs:
-  skaffold:
+  skaffold-go:
     name: publish images with skaffold
     runs-on: ubuntu-latest
 
@@ -20,22 +20,9 @@ jobs:
 
     strategy:
       matrix:
-        module: [builder-go, builder-others, builder-others-fips, builder-others-slow]
-        go-profile: [go-1.21, go-1.20, go-1.19]
+        module: [builder-go]
+        go-profile: [go-1.22, go-1.21, go-1.20, go-1.19, go-1.18]
         builder-profile: [local-docker]
-        exclude:
-          - module: builder-others
-            go-profile: go-1.20
-          - module: builder-others
-            go-profile: go-1.19
-          - module: builder-others-fips
-            go-profile: go-1.20
-          - module: builder-others-fips
-            go-profile: go-1.19
-          - module: builder-others-slow
-            go-profile: go-1.20
-          - module: builder-others-slow
-            go-profile: go-1.19
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -81,3 +68,65 @@ jobs:
             --default-repo ghcr.io/pingcap-qe/cd/builders \
             --module ${{ matrix.module }} \
             --profile ${{ matrix.go-profile }},${{ matrix.builder-profile }}
+  skaffold-nogo:
+    name: publish images with skaffold
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        module: [builder-others, builder-tikv, builder-tikv-fips, builder-others-slow]
+        builder-profile: [local-docker]
+        include:
+          - module: builder-tikv
+            builder-profile: devtoolset8,local-docker
+          - module: builder-tikv
+            builder-profile: devtoolset9,local-docker            
+          # - module: builder-tikv
+          #   builder-profile: devtoolset11,local-docker
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+          fetch-tags: 'true'
+
+      # https://github.com/docker/setup-qemu-action
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
+
+      - name: Cache layers
+        uses: actions/cache@v4
+        with:
+          path: "~/.skaffold/cache"
+          key: skaffold/cd/${{ matrix.module }}-${{ github.sha }}
+          restore-keys: |
+            skaffold/cd/${{ matrix.module }}-
+
+      - name: Publish images
+        working-directory: dockerfiles/cd/builders
+        run: |
+          skaffold build \
+            --build-concurrency 1 \
+            --cache-artifacts \
+            --default-repo ghcr.io/pingcap-qe/cd/builders \
+            --module ${{ matrix.module }} \
+            --profile ${{ matrix.builder-profile }}
diff --git a/dockerfiles/cd/builders/skaffold.yaml b/dockerfiles/cd/builders/skaffold.yaml
@@ -157,7 +157,7 @@ profiles:
 apiVersion: skaffold/v4beta6
 kind: Config
 metadata:
-  name: builder-others
+  name: builder-tikv
 build:
   artifacts:
     - image: tikv
@@ -166,6 +166,67 @@ build:
         dockerfile: tikv/Dockerfile
         cache: {}
         target: builder
+  tagPolicy:
+    customTemplate:
+      template: "{{ .SHA }}"
+      components:
+        - name: SHA
+          gitCommit:
+            variant: Tags
+  cluster:
+    concurrency: 0
+    randomDockerConfigSecret: false
+    randomPullSecret: false
+    dockerConfig:
+      secretName: hub-pingcap-net-ee
+    resources:
+      limits:
+        cpu: "8"
+        memory: 32Gi
+profiles:
+  - name: local-docker
+    build:
+      local:
+        useDockerCLI: true
+        useBuildkit: true
+        concurrency: 0
+        tryImportMissing: true
+    patches:
+      - { op: move, from: /build/artifacts/0/kaniko, path: /build/artifacts/0/docker }
+  - name: devtoolset8
+    patches:
+      - op: replace
+        path: /build/tagPolicy/customTemplate/template
+        value: "{{ .SHA }}-devtoolset8"
+      - op: add
+        path: /build/artifacts/0/kaniko/buildArgs
+        value: 
+          DEVTOOLSET_VER: 8
+  - name: devtoolset9
+    patches:
+      - op: replace
+        path: /build/tagPolicy/customTemplate/template
+        value: "{{ .SHA }}-devtoolset9"
+      - op: add
+        path: /build/artifacts/0/kaniko/buildArgs
+        value: 
+          DEVTOOLSET_VER: 9
+  - name: devtoolset11
+    patches:
+      - op: replace
+        path: /build/tagPolicy/customTemplate/template
+        value: "{{ .SHA }}-devtoolset11"
+      - op: add
+        path: /build/artifacts/0/kaniko/buildArgs
+        value: 
+          DEVTOOLSET_VER: 11
+---
+apiVersion: skaffold/v4beta6
+kind: Config
+metadata:
+  name: builder-others
+build:
+  artifacts:
     - image: tidb-dashboard
       platforms: [linux/amd64, linux/arm64]
       kaniko:
@@ -180,8 +241,8 @@ build:
       secretName: hub-pingcap-net-ee
     resources:
       limits:
-        cpu: "8"
-        memory: 32Gi
+        cpu: "4"
+        memory: 8Gi
 profiles:
   - name: local-docker
     build:
@@ -192,7 +253,6 @@ profiles:
         tryImportMissing: true
     patches:
       - { op: move, from: /build/artifacts/0/kaniko, path: /build/artifacts/0/docker }
-      - { op: move, from: /build/artifacts/1/kaniko, path: /build/artifacts/1/docker }
 ---
 apiVersion: skaffold/v4beta6
 kind: Config
@@ -230,7 +290,7 @@ profiles:
 apiVersion: skaffold/v4beta6
 kind: Config
 metadata:
-  name: builder-others-fips
+  name: builder-tikv-fips
 build:
   artifacts:
     - image: tikv

diff --git a/dockerfiles/cd/builders/tikv/Dockerfile b/dockerfiles/cd/builders/tikv/Dockerfile
@@ -8,12 +8,14 @@
 ########### stage: Builder
 FROM centos:7.9.2009 as builder
 # install packages.
+ARG DEVTOOLSET_VER=8
 RUN yum update --nogpgcheck -y && \
     yum install --nogpgcheck -y epel-release centos-release-scl deltarpm && \
     yum update --nogpgcheck -y && \
-    yum install -y git devtoolset-8 perl cmake3 && \
+    yum install -y git devtoolset-${DEVTOOLSET_VER} perl cmake3 && \
     yum clean all && \
     ln -s /usr/bin/cmake3 /usr/bin/cmake
+ENV DEVTOOLSET_VER ${DEVTOOLSET_VER}
 
 # install protoc.
 # renovate: datasource=github-release depName=protocolbuffers/protobuf
@@ -24,12 +26,13 @@ RUN FILE=$([ "$(arch)" = "aarch64" ] && echo "protoc-${PROTOBUF_VER#?}-linux-aar
 # install rust toolchain
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none
 ENV PATH /root/.cargo/bin/:$PATH
+LABEL devtoolset-ver="${DEVTOOLSET_VER}"
 
 ########### stage: Buiding
 FROM builder as building
 COPY . /tikv
 RUN --mount=type=cache,target=/tikv/target \
-    source /opt/rh/devtoolset-8/enable && \
+    source /opt/rh/devtoolset-${DEVTOOLSET_VER}/enable && \
     ROCKSDB_SYS_STATIC=1 \
     make dist_release -C /tikv
 RUN /tikv/bin/tikv-server --version