diff --git a/.github/actions/build-linux-artifacts/action.yml b/.github/actions/build-linux-artifacts/action.yml index dfbe856560b7..21f1d12de7ac 100644 --- a/.github/actions/build-linux-artifacts/action.yml +++ b/.github/actions/build-linux-artifacts/action.yml @@ -17,6 +17,12 @@ inputs: description: Enable dev mode, only build standard greptime required: false default: "false" + image-namespace: + description: Image Namespace + required: true + image-registry: + description: Image Registry + required: true working-dir: description: Working directory to build the artifacts required: false @@ -31,8 +37,8 @@ runs: run: | cd ${{ inputs.working-dir }} && \ make run-it-in-container BUILD_JOBS=4 \ - IMAGE_NAMESPACE=i8k6a5e1/greptime \ - IMAGE_REGISTRY=public.ecr.aws + IMAGE_NAMESPACE=${{ inputs.image-namespace }} \ + IMAGE_REGISTRY=${{ inputs.image-registry }} - name: Upload sqlness logs if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed. @@ -51,8 +57,8 @@ runs: artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }} version: ${{ inputs.version }} working-dir: ${{ inputs.working-dir }} - image-registry: public.ecr.aws - image-namespace: i8k6a5e1/greptime + image-registry: ${{ inputs.image-registry }} + image-namespace: ${{ inputs.image-namespace }} - name: Build greptime without pyo3 if: ${{ inputs.dev-mode == 'false' }} @@ -64,8 +70,8 @@ runs: artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }} version: ${{ inputs.version }} working-dir: ${{ inputs.working-dir }} - image-registry: public.ecr.aws - image-namespace: i8k6a5e1/greptime + image-registry: ${{ inputs.image-registry }} + image-namespace: ${{ inputs.image-namespace }} - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build. shell: bash @@ -82,8 +88,8 @@ runs: artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }} version: ${{ inputs.version }} working-dir: ${{ inputs.working-dir }} - image-registry: public.ecr.aws - image-namespace: i8k6a5e1/greptime + image-registry: ${{ inputs.image-registry }} + image-namespace: ${{ inputs.image-namespace }} - name: Build greptime on android base image uses: ./.github/actions/build-greptime-binary @@ -94,5 +100,5 @@ runs: version: ${{ inputs.version }} working-dir: ${{ inputs.working-dir }} build-android-artifacts: true - image-registry: public.ecr.aws - image-namespace: i8k6a5e1/greptime + image-registry: ${{ inputs.image-registry }} + image-namespace: ${{ inputs.image-namespace }} diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 036f029f3a56..6f2be60e6733 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -177,6 +177,8 @@ jobs: disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} dev-mode: true # Only build the standard greptime binary. working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }} + image-registry: ${{ vars.ECR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }} build-linux-arm64-artifacts: name: Build linux-arm64 artifacts @@ -206,6 +208,8 @@ jobs: disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} dev-mode: true # Only build the standard greptime binary. working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }} + image-registry: ${{ vars.ECR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }} release-images-to-dockerhub: name: Build and push images to DockerHub diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 6a1a3853de01..09fcc5c26eba 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -154,6 +154,8 @@ jobs: cargo-profile: ${{ env.CARGO_PROFILE }} version: ${{ needs.allocate-runners.outputs.version }} disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + image-registry: ${{ vars.ECR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }} build-linux-arm64-artifacts: name: Build linux-arm64 artifacts @@ -173,6 +175,8 @@ jobs: cargo-profile: ${{ env.CARGO_PROFILE }} version: ${{ needs.allocate-runners.outputs.version }} disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + image-registry: ${{ vars.ECR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }} release-images-to-dockerhub: name: Build and push images to DockerHub diff --git a/.github/workflows/release-dev-builder-images.yaml b/.github/workflows/release-dev-builder-images.yaml index aa3da561dbed..283c16ec45af 100644 --- a/.github/workflows/release-dev-builder-images.yaml +++ b/.github/workflows/release-dev-builder-images.yaml @@ -3,10 +3,6 @@ name: Release dev-builder images on: workflow_dispatch: # Allows you to run this workflow manually. inputs: - version: - description: Version of the dev-builder - required: false - default: latest release_dev_builder_ubuntu_image: type: boolean description: Release dev-builder-ubuntu image @@ -28,22 +24,85 @@ jobs: name: Release dev builder images if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job. runs-on: ubuntu-20.04-16-cores + outputs: + version: ${{ steps.set-version.outputs.version }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Configure build image version + id: set-version + shell: bash + run: | + commitShortSHA=`echo ${GITHUB_SHA} | cut -c1-8` + buildTime=`date +%Y%m%d%H%M%S` + BUILD_VERSION="$commitShortSHA-$buildTime" + RUST_TOOLCHAIN_VERSION=$(cat rust_toolchain.toml | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}') + VERSION="${RUST_TOOLCHAIN_VERSION}-${BUILD_VERSION}" + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "version=VERSION" >> $GITHUB_OUTPUT + - name: Build and push dev builder images uses: ./.github/actions/build-dev-builder-images with: - version: ${{ inputs.version }} + version: ${{ env.VERSION }} dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }} build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }} build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }} build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }} + release-dev-builder-images-ecr: + name: Release dev builder images to AWS ECR + runs-on: ubuntu-20.04 + needs: [ + release-dev-builder-images + ] + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.ECR_REGION }} + + - name: Login to Amazon ECR + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + env: + AWS_REGION: ${{ vars.ECR_REGION }} + with: + registry-type: public + + - name: Push dev-builder-ubuntu image + shell: bash + if: ${{ inputs.release_dev_builder_ubuntu_image }} + run: | + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \ + docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} + + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest \ + docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest + - name: Push dev-builder-centos image + shell: bash + if: ${{ inputs.release_dev_builder_centos_image }} + run: | + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \ + docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} + + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:latest \ + docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:latest + - name: Push dev-builder-android image + shell: bash + if: ${{ inputs.release_dev_builder_android_image }} + run: | + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \ + docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} + + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:latest \ + docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:latest release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container. name: Release dev builder images to CN region runs-on: ubuntu-20.04 @@ -51,35 +110,30 @@ jobs: release-dev-builder-images ] steps: + - name: Login to AliCloud Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.ACR_IMAGE_REGISTRY }} + username: ${{ secrets.ALICLOUD_USERNAME }} + password: ${{ secrets.ALICLOUD_PASSWORD }} + - name: Push dev-builder-ubuntu image shell: bash if: ${{ inputs.release_dev_builder_ubuntu_image }} - env: - DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }} - DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }} run: | - docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} \ - --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \ + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \ docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} - name: Push dev-builder-centos image shell: bash if: ${{ inputs.release_dev_builder_centos_image }} - env: - DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }} - DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }} run: | - docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} \ - --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \ + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \ docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} - name: Push dev-builder-android image shell: bash if: ${{ inputs.release_dev_builder_android_image }} - env: - DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }} - DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }} run: | - docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} \ - --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \ + docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \ docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1c9ee999385d..7bf10408eef4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -183,6 +183,8 @@ jobs: cargo-profile: ${{ env.CARGO_PROFILE }} version: ${{ needs.allocate-runners.outputs.version }} disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + image-registry: ${{ vars.ECR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }} build-linux-arm64-artifacts: name: Build linux-arm64 artifacts @@ -202,6 +204,8 @@ jobs: cargo-profile: ${{ env.CARGO_PROFILE }} version: ${{ needs.allocate-runners.outputs.version }} disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + image-registry: ${{ vars.ECR_IMAGE_REGISTRY }} + image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }} build-macos-artifacts: name: Build macOS artifacts