diff --git a/.github/workflows/pull-verify-packages-config.yaml b/.github/workflows/pull-verify-packages-config.yaml index 7a47e1b1..62a83ebe 100644 --- a/.github/workflows/pull-verify-packages-config.yaml +++ b/.github/workflows/pull-verify-packages-config.yaml @@ -17,23 +17,38 @@ jobs: - name: Install tools run: | - git clone https://github.com/hairyhenderson/gomplate - cd gomplate - GOBIN=/usr/local/bin go install ./cmd/gomplate - rm -rf gomplate - gomplate --version + # install gomplate + git clone https://github.com/hairyhenderson/gomplate && \ + pushd gomplate && \ + GOBIN=/usr/local/bin go install ./cmd/gomplate && + popd && + rm -rf gomplate && \ + gomplate --version + + # install shelldoc to check the README.md + git clone https://github.com/endocode/shelldoc && \ + pushd shelldoc && \ + GOBIN=/usr/local/bin go install ./cmd/shelldoc && + popd && + rm -rf shelldoc && \ + + # we need yq and jq. yq --version jq --version - name: Checkout code uses: actions/checkout@v4 + # - name: Verify the README.md + # run: | + # shelldoc run --shell `which bash` packages/README.md + - name: Test => Get builder run: | - for cm in tidb tiflow tiflash tikv pd; do - for os in linux darwin; do - for ac in amd64 arm64; do - for version in v7.5.0 v7.1.0 v6.5.0 v6.1.0; do + for version in v7.5.0 v7.1.0 v6.5.0 v6.1.0; do + for cm in tidb tiflow tiflash tikv pd; do + for os in linux darwin; do + for ac in amd64 arm64; do for profile in release; do ./packages/scripts/get-package-builder-with-config.sh $cm $os $ac $version $profile done @@ -42,12 +57,24 @@ jobs: done done + ##### others that owns theirs non-unified versions ##### + # tidb-operator + cm="tidb-operator" + os="linux" + for ac in amd64 arm64; do + for version in v1.5.0 v1.6.0; do + for profile in release; do + ./packages/scripts/get-package-builder-with-config.sh $cm $os $ac $version $profile + done + done + done + - name: Test => gen package artifacts script run: | - for cm in tidb tiflow tiflash tikv pd; do - for os in linux darwin; do - for ac in amd64 arm64; do - for version in v7.5.0 v7.1.0 v6.5.0 v6.1.0; do + for version in v7.5.0 v7.1.0 v6.5.0 v6.1.0; do + for cm in tidb tiflow tiflash tikv pd; do + for os in linux darwin; do + for ac in amd64 arm64; do for profile in release; do ./packages/scripts/gen-package-artifacts-with-config.sh $cm $os $ac $version $profile branch-xxx 123456789abcdef done @@ -56,14 +83,38 @@ jobs: done done + ##### others that owns theirs non-unified versions ##### + # tidb-operator + cm="tidb-operator" + os="linux" + for ac in amd64 arm64; do + for version in v1.5.0 v1.6.0; do + for profile in release; do + ./packages/scripts/gen-package-artifacts-with-config.sh $cm $os $ac $version $profile branch-xxx 123456789abcdef + done + done + done + - name: Test => gen package images script run: | - for cm in tidb tiflow tiflash tikv pd; do - for ac in amd64 arm64; do - for version in v7.5.0 v7.1.0 v6.5.0 v6.1.0; do + for version in v7.5.0 v7.1.0 v6.5.0 v6.1.0; do + for cm in tidb tiflow tiflash tikv pd; do + for ac in amd64 arm64; do for profile in release; do ./packages/scripts/gen-package-images-with-config.sh $cm linux $ac $version $profile branch-xxx 123456789abcdef done done done done + + ##### others that owns theirs non-unified versions ##### + # tidb-operator + cm="tidb-operator" + os="linux" + for version in v1.6.0 v1.5.0; do + for ac in amd64 arm64; do + for profile in release; do + ./packages/scripts/gen-package-images-with-config.sh $cm linux $ac $version $profile branch-xxx 123456789abcdef + done + done + done diff --git a/packages/README.md b/packages/README.md index 97721d37..70c0f34d 100644 --- a/packages/README.md +++ b/packages/README.md @@ -1,2 +1,38 @@ -`WIP` +Central declarative congfigurations for artifacts delivering. === + +> We use go template format to control them. + +## Prerequire tools + +- [gomplate](https://github.com/hairyhenderson/gomplate) + > Please install the master version. +- [yq] +- jq + +## For component binaries packages and container images + +Configuration template: [packages.yaml.tmpl](./packages.yaml.tmpl) + +### Required context + +You can get them by run: +```console +$ grep -oE "{{\s*\..*?}}" packages/packages.yaml.tmpl | grep -oE "\.\w+(\.\w+)*" | sort -u +.Git.ref +.Git.sha +.Release.arch +.Release.os +.Release.version +``` + +## For offline deploy pacakges + +### Required context + +You can get them by run: +```console +$ grep -oE "{{\s*\..*?}}" packages/offline-packages.yaml.tmpl | grep -oE "\.\w+(\.\w+)*" | sort -u +.Release.arch +.Release.version +``` diff --git a/packages/offline-packages.yaml.tmpl b/packages/offline-packages.yaml.tmpl index f4a3dc85..39c86ec6 100644 --- a/packages/offline-packages.yaml.tmpl +++ b/packages/offline-packages.yaml.tmpl @@ -1,11 +1,12 @@ +# See README.md +artifactory: + repo: hub.pingcap.net/pingcap/offline-package + tags: + - "{{ .Release.version }}" artifacts: community: desc: community offline package version: "{{ .Release.version }}" # segment version. - artifactory: - repo: hub.pingcap.net/pingcap/offline-package - tags: - - "{{ .Release.version }}" routers: # match once. - description: "Started from v7.5.0 to latest" # ref: https://github.com/Masterminds/semver#checking-version-constraints @@ -107,7 +108,7 @@ artifacts: extract: true extract_inner_path: "etcd-v3.4.21-linux-{{ .Release.arch }}/etcdctl" - description: "Started from v7.1.0 until v7.5" - # ref: https://github.com/Masterminds/semver#checking-version-constraints + # ref: https://github.com/Masterminds/semver#checking-version-constraints if: {{ semver.CheckConstraint ">=7.1.0-0, <7.5.0-0" .Release.version }} os: [linux] arch: [amd64, arm64] diff --git a/packages/packages.yaml.tmpl b/packages/packages.yaml.tmpl index a69766d7..0db78427 100644 --- a/packages/packages.yaml.tmpl +++ b/packages/packages.yaml.tmpl @@ -1,3 +1,4 @@ +# See README.md components: pd: desc: pd server component tarball @@ -859,6 +860,7 @@ components: - script: make build fips: - script: ENABLE_FIPS=1 make build + builder: ghcr.io/pingcap-qe/cd/utils/release:v20231216-14-g77d0cd2 artifacts: - name: "tidb-operator-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" files: # output files. @@ -982,6 +984,7 @@ components: - {{ strings.ReplaceAll "/" "-" .Git.ref | strings.ToLower }}-{{ strings.Trunc 7 .Git.sha }} {{- end }} - {{ strings.ReplaceAll "/" "-" .Git.ref | strings.ToLower }} + builder: ghcr.io/pingcap-qe/cd/utils/release:v20231216-14-g77d0cd2 artifacts: - name: container image type: image diff --git a/packages/scripts/get-package-builder-with-config.sh b/packages/scripts/get-package-builder-with-config.sh index 5a863667..21af001d 100755 --- a/packages/scripts/get-package-builder-with-config.sh +++ b/packages/scripts/get-package-builder-with-config.sh @@ -35,17 +35,17 @@ function main() { # fail when array length greater than 1. if yq -e 'length > 1' release-package-routes.yaml >/dev/null 2>&1; then - echo "Error: wrong package config that make me matched more than 1 routes!" + echo "Error: wrong package config that make me matched more than 1 routes!" >&2 exit 1 fi if yq -e 'length == 0' release-package-routes.yaml >/dev/null 2>&1; then - echo "No package routes matched for the target($target_info)." - exit 0 + echo "No package routes matched for the target($target_info)." >&2 + exit 1 fi # get the builder image. - yq '.[0].builder' release-package-routes.yaml | tee $out_file + yq -e '.[0].builder' release-package-routes.yaml | tee $out_file } main "$@"