nightly-build.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Nightly - Build

on:
  schedule:
    - cron: '0 16 */1 * *'  # once a day. UTC time
  workflow_dispatch:
    inputs:
      commit-id:
        required: false
        type: string
        description: 'Commit ID to build. Default is the latest commit on the default branch.'

env:
  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -DskipITs -DskipTests -Dspotless.apply.skip=true
  HUB: ghcr.io/apache/shardingsphere
  PROXY: ghcr.io/apache/shardingsphere-proxy
  PROXY_NATIVE: ghcr.io/apache/shardingsphere-proxy-native
  AGENT: ghcr.io/apache/shardingsphere-agent

jobs:
  global-environment:
    name: Import Global Environment
    uses: ./.github/workflows/required-reusable.yml
  
  upload-to-nightlies:
    if: github.repository == 'apache/shardingsphere'
    name: Upload to Nightly Builds
    needs: global-environment
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-cache-${{ github.sha }}
          restore-keys: |
            ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-cache-
            ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11
      - name: Build Project
        run: |
          ./mvnw -B clean install -Prelease
      - uses: burnett01/rsync-deployments@5.2
        with:
          switches: -avzr
          path: distribution/proxy/target/apache-shardingsphere-*.tar.gz
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/shardingsphere
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
  
  build-proxy-image:
    if: github.repository == 'apache/shardingsphere'
    name: Build Proxy Image
    needs: global-environment
    runs-on: ${{ needs.global-environment.outputs.GLOBAL_RUNS_ON }}
    permissions:
      contents: read
      packages: write
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.commit-id }}
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11
      - uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-cache-${{ github.sha }}
          restore-keys: |
            ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-cache-
            ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-
      - name: Login Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.HUB }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: Push Docker Image
        run: ./mvnw -am -pl distribution/proxy -B -Prelease,docker.buildx.push -DskipTests -Dproxy.image.repository=${{ env.PROXY }} -Dproxy.image.tag=${{ github.sha }} clean install
  
  build-proxy-native-image:
    if: github.repository == 'apache/shardingsphere'
    name: Build Proxy Native Image
    needs: global-environment
    timeout-minutes: 90
    permissions:
      contents: read
      packages: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.commit-id }}
      - uses: graalvm/setup-graalvm@v1
        with:
          java-version: '22.0.2'
          distribution: 'graalvm-community'
          github-token: ${{ secrets.GITHUB_TOKEN }}
          cache: 'maven'
          native-image-job-reports: 'true'
      - uses: docker/login-action@v3
        with:
          registry: ${{ env.HUB }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Push Docker Image
        run: |
          ./mvnw -am -pl distribution/proxy-native -Prelease.native,docker.buildx.push.native -B -T1C -DskipTests -Dproxy.native.image.repository=${{ env.PROXY_NATIVE }} -Dproxy.native.image.tag=${{ github.sha }} clean package
  build-agent-image:
    if: github.repository == 'apache/shardingsphere'
    name: Build Agent Image
    needs: global-environment
    timeout-minutes: 90
    permissions:
      contents: read
      packages: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.commit-id }}
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '11'
          cache: 'maven'
      - uses: docker/login-action@v3
        with:
          registry: ${{ env.HUB }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Push Docker Image
        run: |
          ./mvnw -am -pl distribution/agent -Prelease,docker.buildx.push -B -T1C -DskipTests -Dagent.image.repository=${{ env.AGENT }} -Dagent.image.tag=${{ github.sha }} clean package
  build-cache:
    if: ${{ needs.global-environment.outputs.GLOBAL_IS_NIGHTLY_JOB_EXECUTABLE == 'true' }}
    name: Build Project
    needs: global-environment
    runs-on: ${{ needs.global-environment.outputs.GLOBAL_RUNS_ON }}
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11
      - uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-
      - name: Build with Maven
        run: ./mvnw -B -T1C -ntp clean install -DskipITs -DskipTests
  
  generate-and-test-examples:
    name: Example-${{ matrix.feature }}-${{ matrix.framework }}-${{ matrix.mode }}-${{ matrix.transaction }}
    if: github.repository == 'apache/shardingsphere'
    needs: build-cache
    runs-on: ubuntu-latest
    timeout-minutes: 90
    strategy:
      max-parallel: 20
      fail-fast: false
      matrix:
        feature: [ sharding, readwrite-splitting, encrypt, shadow, mask ]
        framework: [ jdbc, spring-boot-starter-jdbc, spring-boot-starter-jpa, spring-boot-starter-mybatis, spring-namespace-jdbc, spring-namespace-jpa, spring-namespace-mybatis ]
        mode: [ standalone, cluster-zookeeper ]
        transaction: [ local, xa-atomikos, xa-narayana ]
        exclude:
          - feature: shadow
            framework: spring-boot-starter-jpa
          - feature: shadow
            framework: spring-namespace-jpa
          - transaction: xa-atomikos
            mode: cluster-zookeeper
          - transaction: xa-narayana
            mode: cluster-zookeeper
          - feature: sharding
            framework: spring-boot-starter-mybatis
            mode: standalone
            transaction: xa-atomikos
          - feature: sharding
            framework: spring-boot-starter-mybatis
            mode: standalone
            transaction: xa-narayana
    services:
      mysql:
        image: mysql:8.0.22
        env:
          MYSQL_ROOT_PASSWORD: 123456
        ports:
          - 3306:3306
        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
      zookeeper:
        image: zookeeper:3.6.3
        ports:
          - 2181:2181
        options: --health-cmd="echo ruok | nc localhost 2181" --health-interval=10s --health-timeout=5s --health-retries=3
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 11
      - uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-
      - name: Prepare Environments
        run: sh .github/workflows/resources/scripts/nightly-build-example/init-mysql-container.sh
      - name: Build with Maven
        run: ./mvnw -B -T1C -ntp clean install -DskipITs -DskipTests
      - name: Generate Examples
        run: ./mvnw -B clean install -f examples/shardingsphere-jdbc-example-generator/pom.xml -Pexample-generator -Dmodes=${{ matrix.mode }} -Dtransactions=${{ matrix.transaction }} -Dfeatures=${{ matrix.feature }} -Dframeworks=${{ matrix.framework }}
      - name: Test Examples
        run : ./mvnw -B test -f examples/shardingsphere-jdbc-example-generator/target/generated-sources/shardingsphere-jdbc-sample/${{ matrix.feature }}--${{ matrix.framework }}--${{ matrix.mode }}--${{ matrix.transaction }}/pom.xml -Pexample-generator -Dexec.cleanupDaemonThreads=false
      - name: Package Examples
        run: |
          cd examples/shardingsphere-jdbc-example-generator/target/generated-sources/shardingsphere-jdbc-sample/
          tar -czvf ${{ matrix.feature }}--${{ matrix.framework }}--${{ matrix.mode }}--${{ matrix.transaction }}.tar.gz ${{ matrix.feature }}--${{ matrix.framework }}--${{ matrix.mode }}--${{ matrix.transaction }}
      - uses: burnett01/rsync-deployments@5.2
        with:
          switches: -avzr
          path: examples/shardingsphere-jdbc-example-generator/target/generated-sources/shardingsphere-jdbc-sample/${{ matrix.feature }}--${{ matrix.framework }}--${{ matrix.mode }}--${{ matrix.transaction }}.tar.gz
          remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/shardingsphere/examples
          remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
          remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
          remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
          remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}