test-cloud-k3d-self.yml

name: Test Cloud On K3D Self Runner

on:
  workflow_dispatch:
    inputs:
      cloud-version:
        description: 'kb-cloud version (e.g. 0.29.0-alpha.110)'
        type: string
        required: false
        default: '0.29.0-alpha.110'
      k3s-version:
        description: 'k3s cluster version (e.g. 1.30)'
        type: string
        required: false
        default: '1.30'
  workflow_call:
    inputs:
      cloud-version:
        description: 'kb-cloud version (e.g. 0.29.0-alpha.110)'
        type: string
        required: false
        default: '0.29.0-alpha.110'
      k3s-version:
        description: 'k3s cluster version (e.g. 1.30)'
        type: string
        required: false
        default: '1.30'
    outputs:
      test-result:
        description: "e2e test result"
        value: "${{ jobs.cloud-test-k3d.outputs.test-result }}"

run-name: Test Cloud Version:${{ inputs.cloud-version }} K3s Version:${{ inputs.k3s-version }}

env:
  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
  K3D_NAME: kbcloud
  CLOUD_LICENSE: ${{ secrets.CLOUD_LICENSE }}

permissions:
  id-token: write
  contents: read

jobs:
  cloud-test-k3d:
    runs-on: [ self-hosted, k3s-runner ]
    outputs:
      deploy-result: ${{ steps.deploy_cloud.outputs.deploy-result }}
      test-result: ${{ steps.test_result.outputs.test-result }}
      test-type: ${{ steps.test_result.outputs.test-type }}
      runner-name: ${{ steps.get_runner_name.outputs.runner-name }}
    steps:
      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: apecloud-cd

      - name: setup k3d k3s
        uses: nolar/setup-k3d-k3s@v1
        with:
          k3d-name: "${{ env.K3D_NAME }}"
          version: "v${{ inputs.k3s-version }}"
          github-token: ${{ env.GITHUB_TOKEN }}
          k3d-args: -p 443:443@loadbalancer -p 80:80@loadbalancer --agents 2

      - name: update k3d coredns cm
        run: |
          bash ${{ github.workspace }}/apecloud-cd/.github/utils/utils.sh --type 37

      - name: install cert-manager
        run: |
          helm repo add jetstack https://charts.jetstack.io --force-update
          helm install \
            cert-manager jetstack/cert-manager \
            --namespace cert-manager \
            --create-namespace \
            --version v1.16.0 \
            --set crds.enabled=true

      - name: install kb cloud
        run: |
          helm repo add kb-chart https://apecloud.github.io/helm-charts
          
          helm install installer kb-chart/kb-cloud-installer \
            --version ${{ inputs.cloud-version }} \
            --set image.registry="docker.io" \
            --set version=${{ inputs.cloud-version }} \
            --set kubernetesProvider=k3d \
            --set fqdn.domain=mytest.kubeblocks.com \
            --set tls.enabled=true

      - name: check kb-cloud-installer pod Completed
        run: |
          for i in {1..360}; do
            if kubectl get pods | grep "kb-cloud-installer" | grep Completed ; then
              kubectl get pods -n kb-cloud | grep Running
              echo "KB Cloud is ready"
              break
            fi
            echo "Waiting for KB Cloud to be ready..."
            kubectl get pods -n kb-cloud
            echo ""
            sleep 5
          done

      - name: deploy kubeblocks
        run: |
          echo "get k8s config"
          k8s_cluster_name=$(kubectl config get-clusters | sed '1d')
          kubectl config set-cluster ${k8s_cluster_name} --insecure-skip-tls-verify=true
          k8s_kubeconfig_yaml=$(kubectl config view --minify -o yaml --raw)
          kubeconfig_file_name="k8s_kubeconfig_file.yaml"
          touch ${kubeconfig_file_name}
          echo "${k8s_kubeconfig_yaml}" > ${kubeconfig_file_name}
          yq e -i '.clusters[0].cluster.server = "https://kubernetes.default:443"' ${kubeconfig_file_name}
          k8s_kubeconfig=$(cat ${kubeconfig_file_name})
          echo "------------------------------------------------------------"
          echo "${k8s_kubeconfig}"
          echo "------------------------------------------------------------"
          K8S_KUBECONFIG="$(echo "${k8s_kubeconfig}" | base64)"
          
          CLOUD_LICENSE="${{ env.CLOUD_LICENSE }}"
          echo "deploy kubeblocks"
          
          helm install deployer kb-chart/kb-cloud-deployer \
            --version 0.0.1 \
            --set image.registry="docker.io" \
            --set license="${CLOUD_LICENSE}" \
            --set kubeconfig="${K8S_KUBECONFIG}"

      - name: check kb-cloud-deployer pod Completed
        run: |
          for i in {1..360}; do
            if kubectl get pods | grep "kb-cloud-deployer" | grep Completed ; then
              kubectl get pods -n kb-system | grep Running
              echo "KubeBlocks is ready"
              break
            fi
            echo "Waiting for KubeBlocks to be ready..."
            kubectl get pods -n kb-system
            echo ""
            sleep 5
          done

      - name: get runner name
        id: get_runner_name
        if: ${{ always() }}
        run: |
          echo runner-name=${RUNNER_NAME} >> $GITHUB_OUTPUT

  remove-runner:
    needs: [ cloud-test-k3d ]
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: ./

      - name: remove runner
        run: |
          bash .github/utils/utils.sh --type 25 \
              --github-token "${{ env.GITHUB_TOKEN }}" \
              --github-repo "${{ github.repository }}" \
              --runner-name ${{ needs.cloud-test-k3d.outputs.runner-name }}

      - uses: azure/setup-kubectl@v3
        with:
          version: "v1.28.3"

      - uses: aws-actions/setup-sam@v2
        if: ${{ inputs.cloud-provider == 'eks' }}
        with:
          use-installer: true

      - uses: aws-actions/configure-aws-credentials@v4
        if: ${{ inputs.cloud-provider == 'eks' && env.AWS_ACCESS_KEY_ID != '' && env.AWS_SECRET_ACCESS_KEY != '' }}
        with:
          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
          aws-region: "${{ env.AWS_REGION }}"

      - if: ${{ inputs.cloud-provider == 'eks' }}
        run: |
          aws eks --region ${{ env.AWS_REGION }} update-kubeconfig --name ${{ env.AWS_CLUSTER }} \
              --role-arn $(aws iam get-role --role-name ${{ env.AWS_CLUSTER }}-admin-role --query 'Role.Arn' --output text)

      - if: ${{ inputs.cloud-provider == 'gke' && env.GOOGLE_CREDENTIALS != '' }}
        id: 'auth'
        name: 'Authenticate to Google Cloud'
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: '${{ env.GOOGLE_CREDENTIALS }}'

      - name: setup gke-gcloud-auth-plugin
        if: ${{ inputs.cloud-provider == 'gke' }}
        uses: simenandre/setup-gke-gcloud-auth-plugin@v1

      - uses: google-github-actions/get-gke-credentials@v1
        if: ${{ inputs.cloud-provider == 'gke' }}
        with:
          cluster_name: ${{ env.CLUSTER_NAME }}
          location: ${{ env.GCP_REGION }}

      - name: delete runner pod
        env:
          RUNNER_NAME: ${{ needs.cloud-test-k3d.outputs.runner-name }}
        run: |
          runner_pod_exists=$(kubectl get pod | (grep "${{ env.RUNNER_NAME }}"||true))
          if [[ -n "${runner_pod_exists}" ]]; then
              kubectl delete pod "${{ env.RUNNER_NAME }}" --force
          fi