fix(dockerfiles/cd/builders/tiflash): fix dockerfile to avoid cache c…

…onflict with docker buildx (#398) Signed-off-by: wuhuizuo <[email protected]> --------- Signed-off-by: wuhuizuo <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
PingCAP-QE · Sep 5, 2024 · 022d4cc · 022d4cc
1 parent 2ed3ea4
commit 022d4cc
Show file tree

Hide file tree

Showing 34 changed files with 98 additions and 88 deletions.
diff --git a/.github/scripts/ci.sh b/.github/scripts/ci.sh
@@ -14,7 +14,7 @@ function test_get_builder() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo "$cm $os $ac $version:"
-                    $script $cm $os $ac $version $profile
+                    $script "$cm" "$os" "$ac" "$version" $profile
                 done
             done
         done
@@ -26,7 +26,7 @@ function test_get_builder() {
         for os in $operating_systems; do
             for ac in $architectures; do
                 echo "$cm $os $ac $version:"
-                $script $cm $os $ac $version $profile
+                $script "$cm" "$os" "$ac" "$version" $profile
             done
         done
     done
@@ -38,7 +38,7 @@ function test_get_builder() {
     for ac in $architectures; do
         for version in v1.5.0 v1.6.0; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
     # tiflow-operator
@@ -47,7 +47,7 @@ function test_get_builder() {
     for ac in $architectures; do
         for version in v6.4.0-20221102-1667359250 v20221018; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
 
@@ -57,7 +57,7 @@ function test_get_builder() {
     for ac in $architectures; do
         for version in v0.5.0 v0.6.0; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
 
@@ -67,7 +67,7 @@ function test_get_builder() {
     for ac in $architectures; do
         for version in v0.1.2 v0.1.3; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile
+            $script "$cm" "$os" "$ac" "$version" $profile
         done
     done
 }
@@ -85,7 +85,7 @@ function test_gen_package_artifacts_script() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo "$cm $os $ac $version:"
-                    $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+                    $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
                     shellcheck -S error packages/scripts/build-package-artifacts.sh
                 done
             done
@@ -98,7 +98,7 @@ function test_gen_package_artifacts_script() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo "$cm $os $ac $version:"
-                    $script $cm $os $ac $version enterprise branch-xxx 123456789abcdef
+                    $script "$cm" "$os" "$ac" "$version" enterprise branch-xxx 123456789abcdef
                     shellcheck -S error packages/scripts/build-package-artifacts.sh
                 done
             done
@@ -111,7 +111,7 @@ function test_gen_package_artifacts_script() {
             for os in $operating_systems; do
                 for ac in $architectures; do
                     echo "$cm $os $ac $version:"
-                    $script $cm $os $ac $version failpoint branch-xxx 123456789abcdef
+                    $script "$cm" "$os" "$ac" "$version" failpoint branch-xxx 123456789abcdef
                     shellcheck -S error packages/scripts/build-package-artifacts.sh
                 done
             done
@@ -125,7 +125,7 @@ function test_gen_package_artifacts_script() {
     for ac in $architectures; do
         for version in v1.5.0 v1.6.0; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -136,7 +136,7 @@ function test_gen_package_artifacts_script() {
     for ac in $architectures; do
         for version in v6.4.0-20221102-1667359250 v20221018; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -147,7 +147,7 @@ function test_gen_package_artifacts_script() {
     for ac in $architectures; do
         for version in v0.5.0 v0.6.0; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -158,7 +158,7 @@ function test_gen_package_artifacts_script() {
     for ac in $architectures; do
         for version in v0.1.2 v0.1.3; do
             echo "$cm $os $ac $version:"
-            $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" "$os" "$ac" "$version" $profile branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-artifacts.sh
         done
     done
@@ -175,7 +175,7 @@ function test_gen_package_images_script() {
         for cm in $components; do
             for ac in $architectures; do
                 echo "$cm $os $ac $version:"
-                $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+                $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
                 shellcheck -S error packages/scripts/build-package-images.sh
             done
         done
@@ -186,7 +186,7 @@ function test_gen_package_images_script() {
     for version in $versions; do
         for ac in $architectures; do
             echo "$cm $os $ac $version:"
-            $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -198,7 +198,7 @@ function test_gen_package_images_script() {
     for ac in $architectures; do
         for version in v1.6.0 v1.5.0; do
             echo "$cm $os $ac $version:"
-            $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -208,7 +208,7 @@ function test_gen_package_images_script() {
     for ac in $architectures; do
         for version in v6.4.0-20221102-1667359250 v20221018; do
             echo "$cm $os $ac $version:"
-            $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -219,7 +219,7 @@ function test_gen_package_images_script() {
     for version in v0.5.0 v0.6.0; do
         for ac in $architectures; do
             echo "$cm $os $ac $version:"
-            $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -230,7 +230,7 @@ function test_gen_package_images_script() {
     for ac in $architectures; do
         for version in v0.1.2 v0.1.3; do
             echo "$cm $os $ac $version:"
-            $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+            $script "$cm" linux "$ac" "$version" "$profile" branch-xxx 123456789abcdef
             shellcheck -S error packages/scripts/build-package-images.sh
         done
     done
@@ -248,7 +248,7 @@ function test_gen_offline_package_artifacts_script() {
             for ac in $architectures; do
                 for edition in $editions; do
                     echo "$os $ac $version $edition:"
-                    $script $os $ac $version $edition
+                    $script "$os" "$ac" "$version" "$edition"
                     shellcheck -S error packages/scripts/compose-offline-packages-artifacts.sh
                 done
             done

diff --git a/.github/workflows/pull-prod-runtime-images.yaml b/.github/workflows/pull-prod-runtime-images.yaml
@@ -58,7 +58,7 @@ jobs:
         run: |
           curl -L https://github.com/estesp/manifest-tool/releases/download/v2.1.7/binaries-manifest-tool-2.1.7.tar.gz | tar -zxvf - manifest-tool-linux-amd64
           sudo install manifest-tool-linux-amd64 /usr/local/bin/manifest-tool
-      
+
       - name: Cache layers
         uses: actions/cache@v4
         with:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,16 @@
+ci:
+  autofix_commit_msg: |
+    fix: auto fixes from pre-commit.com hooks
+
+    for more information, see https://pre-commit.ci
+  autoupdate_commit_msg: "chore: pre-commit autoupdate"
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.16.1
+    hooks:
+      - id: gitleaks
diff --git a/dockerfiles-multi-stages/tiflash/Dockerfile b/dockerfiles-multi-stages/tiflash/Dockerfile
@@ -41,4 +41,4 @@ COPY --from=busybox:1.36.1 /bin/busybox /bin/busybox
 ENV LD_LIBRARY_PATH /tiflash
 COPY --from=builder /tiflash/output/tiflash /tiflash
 
-ENTRYPOINT ["/tiflash/tiflash", "server"]
+ENTRYPOINT ["/tiflash/tiflash", "server"]
diff --git a/dockerfiles/bases/build-old.sh b/dockerfiles/bases/build-old.sh
@@ -8,25 +8,25 @@ build() {
     case "${PLATFORMS}" in
     linux/amd64)
         echo "building for linux/amd64 platform..."
-        docker build --tag=$IMAGE --platform=$PLATFORMS --target amd64 . -f "$dockerfile"
+        docker build --tag="$IMAGE" --platform="$PLATFORMS" --target amd64 . -f "$dockerfile"
         if $PUSH_IMAGE; then
-            docker push $IMAGE
+            docker push "$IMAGE"
         fi
         ;;
     linux/arm64)
         echo "building for linux/arm64 platform..."
-        docker build --tag=$IMAGE --platform=$PLATFORMS --target arm64 . -f "$dockerfile"
+        docker build --tag="$IMAGE" --platform="$PLATFORMS" --target arm64 . -f "$dockerfile"
         if $PUSH_IMAGE; then
-            docker push $IMAGE
+            docker push "$IMAGE"
         fi
         ;;
     linux/arm64,linux/amd64 | linux/amd64,linux/arm64)
         echo "building for linux/arm64 and linux/amd64 platforms..."
-        docker build --tag=${IMAGE}_linux_amd64 --platform=linux/amd64 --target amd64 . -f "$dockerfile"
-        docker build --tag=${IMAGE}_linux_arm64 --platform=linux/arm64 --target arm64 . -f "$dockerfile"
+        docker build --tag="${IMAGE}_linux_amd64" --platform=linux/amd64 --target amd64 . -f "$dockerfile"
+        docker build --tag="${IMAGE}_linux_arm64" --platform=linux/arm64 --target arm64 . -f "$dockerfile"
         if $PUSH_IMAGE; then
-            docker push ${IMAGE}_linux_amd64
-            docker push ${IMAGE}_linux_arm64
+            docker push "${IMAGE}_linux_amd64"
+            docker push "${IMAGE}_linux_arm64"
 
             # compose manifest for multi-arch image.
             pushed_repo="${IMAGE%:*}"
@@ -35,14 +35,14 @@ build() {
             yq -i ".tags = [\"$tag\"]" manifest.yaml
 
             # linux/amd64
-            manifest-tool inspect --raw ${IMAGE}_linux_amd64 >manifest_linux_amd64.json
+            manifest-tool inspect --raw "${IMAGE}_linux_amd64" >manifest_linux_amd64.json
             yq -i '.manifests += [{}]' manifest.yaml
             digest=$(jq -r '.digest' manifest_linux_amd64.json)
             yq -i ".manifests[-1].image = \"${pushed_repo}@${digest}\"" manifest.yaml
             yq -i '.manifests[-1].platform.os = "linux"' manifest.yaml
             yq -i '.manifests[-1].platform.architecture = "amd64"' manifest.yaml
             # linux/arm64
-            manifest-tool inspect --raw ${IMAGE}_linux_arm64 >manifest_linux_arm64.json
+            manifest-tool inspect --raw "${IMAGE}_linux_arm64" >manifest_linux_arm64.json
             yq -i '.manifests += [{}]' manifest.yaml
             digest=$(jq -r '.digest' manifest_linux_arm64.json)
             yq -i ".manifests[-1].image = \"${pushed_repo}@${digest}\"" manifest.yaml

diff --git a/dockerfiles/bases/tikv-base/release-6.5.Dockerfile b/dockerfiles/bases/tikv-base/release-6.5.Dockerfile
@@ -13,4 +13,3 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \
 # set timezone
 ENV TZ=/etc/localtime
 ENV TZDIR=/usr/share/zoneinfo
-
diff --git a/dockerfiles/cd/builders/ng-monitoring/Dockerfile b/dockerfiles/cd/builders/ng-monitoring/Dockerfile
@@ -2,15 +2,15 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/ng-monitoring.git ng-monitoring 
+#   - git clone https://github.com/pingcap/ng-monitoring.git ng-monitoring
 #   - rm ng-monitoring/.dockerignore # make step depended on git metadata.
 #   - docker build -t ng-monitoring -f Dockerfile ./ng-monitoring
 
 ########### stage: builder
 FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for NG-Monitoring"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 # install packages.
 RUN --mount=type=cache,target=/var/cache/dnf \

diff --git a/dockerfiles/cd/builders/pd/Dockerfile b/dockerfiles/cd/builders/pd/Dockerfile
@@ -9,7 +9,7 @@
 FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for PD"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 # install packages.
 RUN --mount=type=cache,target=/var/cache/dnf \

diff --git a/dockerfiles/cd/builders/tidb-dashboard/Dockerfile b/dockerfiles/cd/builders/tidb-dashboard/Dockerfile
@@ -2,15 +2,15 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/tidb-dashbord.git tidb-dashbord 
+#   - git clone https://github.com/pingcap/tidb-dashbord.git tidb-dashbord
 #   - rm tidb-dashbord/.dockerignore # make step depended on git metadata.
 #   - docker build -t tidb-dashbord -f Dockerfile ./tidb-dashbord
 
 ########### stage: builder
 FROM quay.io/rockylinux/rockylinux:8.10.20240528-ubi as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for TiDB Dashboard"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 # install packages.
 RUN --mount=type=cache,target=/var/cache/dnf \

diff --git a/dockerfiles/cd/builders/tidb-operator/Dockerfile b/dockerfiles/cd/builders/tidb-operator/Dockerfile
@@ -2,7 +2,7 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/tidb-operator.git tidb 
+#   - git clone https://github.com/pingcap/tidb-operator.git tidb
 #   - rm tidb-operator/.dockerignore # make step depended on git metadata.
 #   - docker build -t tidb-operator -f Dockerfile ./tidb-operator
 
@@ -11,7 +11,7 @@ ARG GOLANG_VERSION=1.21.13
 FROM golang:${GOLANG_VERSION} as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for TiDB Operator"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 LABEL go-version="${GOLANG_VERSION}"
 
 # install packages.

diff --git a/dockerfiles/cd/builders/tidb/Dockerfile b/dockerfiles/cd/builders/tidb/Dockerfile
@@ -2,15 +2,15 @@
 #   - docker >= v20.10
 #
 # build steps:
-#   - git clone https://github.com/pingcap/tidb.git tidb 
+#   - git clone https://github.com/pingcap/tidb.git tidb
 #   - rm tidb/.dockerignore # make step depended on git metadata.
 #   - docker build -t tidb -f Dockerfile ./tidb
 
 ########### stage: builder
 FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for TiDB"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 # install packages.
 RUN --mount=type=cache,target=/var/cache/dnf \

diff --git a/dockerfiles/cd/builders/tiflash/Dockerfile b/dockerfiles/cd/builders/tiflash/Dockerfile
@@ -24,7 +24,7 @@ RUN zip_url=$([ "$(arch)" = "x86_64" ] && \
 FROM quay.io/rockylinux/rockylinux:8.10.20240528-ubi as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for TiFlash"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 # install packages.
 RUN --mount=type=cache,target=/var/cache/dnf \
@@ -38,12 +38,12 @@ ENV LD ld.lld
 COPY --from=extract-conda-openssl /usr/local/opt/openssl /usr/local/opt/openssl
 
 # building and install libc++
-RUN --mount=type=cache,target=/tmp/llvm \
-    git clone https://github.com/llvm/llvm-project.git  --branch llvmorg-17.0.6 --depth 1 /tmp/llvm/llvm-project && \
-    cd /tmp/llvm/llvm-project && rm -rf build && mkdir build && \
+RUN git clone https://github.com/llvm/llvm-project.git  --branch llvmorg-17.0.6 --depth 1 /tmp/llvm-project && \
+    cd /tmp/llvm-project && rm -rf build && mkdir build && \
     cmake -G Ninja -S runtimes -B build -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" -DCMAKE_INSTALL_PREFIX=/usr -DLLVM_LIBDIR_SUFFIX=64 && \
     ninja -C build cxx cxxabi unwind && \
-    ninja -C build install-cxx install-cxxabi install-unwind
+    ninja -C build install-cxx install-cxxabi install-unwind && \
+    rm -rf /tmp/llvm-project
 
 # install rust toolchain
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none

diff --git a/dockerfiles/cd/builders/tiflow/Dockerfile b/dockerfiles/cd/builders/tiflow/Dockerfile
@@ -43,7 +43,7 @@ RUN node --version && npm --version && npm install -g yarn
 FROM builder-dm as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for TiFlow"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 ########### stage: building-cdc
 FROM builder-cdc as building-cdc

diff --git a/dockerfiles/cd/builders/tikv/Dockerfile b/dockerfiles/cd/builders/tikv/Dockerfile
@@ -9,7 +9,7 @@
 FROM quay.io/rockylinux/rockylinux:8.10.20240528-ubi as builder
 LABEL org.opencontainers.image.authors "[email protected]"
 LABEL org.opencontainers.image.description "binary builder for TiKV"
-LABEL org.opencontainers.image.source = "https://github.com/PingCAP-QE/artifacts"
+LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts"
 
 # install packages.
 # install packages.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -13,4 +13,3 @@ RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \
		# set timezone
		ENV TZ=/etc/localtime
		ENV TZDIR=/usr/share/zoneinfo