e2e-multicluster-ci.yml

name: E2E Multi-Cluster Fleet

on:
  schedule:
    # Run everyday day at 9:00 AM
    - cron: '0 9 * * *'
  workflow_dispatch:
    inputs:
      enable_tmate:
        description: 'Enable debugging via tmate'
        required: false
        default: "false"
  pull_request:
  push:
    branches:
    - 'release/*'

env:
  GOARCH: amd64
  CGO_ENABLED: 0
  SETUP_K3D_VERSION: 'v5.5.1'

jobs:
  e2e-fleet-mc-test:
    runs-on: ubuntu-latest

    steps:
      -
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      -
        uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
          check-latest: true
      -
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-
      -
        name: Install Ginkgo CLI
        run: go install github.com/onsi/ginkgo/v2/ginkgo
      -
        name: Build Fleet Binaries
        run: |
          go build -o bin/fleetcontroller-linux-$GOARCH ./cmd/fleetcontroller

          go build -o "bin/fleet-linux-$GOARCH" ./cmd/fleetcli
          go build -o "bin/fleetagent-linux-$GOARCH" ./cmd/fleetagent
      -
        name: Build Docker Images
        run: |
          docker build -f package/Dockerfile -t rancher/fleet:dev --build-arg="ARCH=$GOARCH" .
          docker build -f package/Dockerfile.agent -t rancher/fleet-agent:dev --build-arg="ARCH=$GOARCH" .
      -
        name: Provision k3d Cluster
        uses: AbsaOSS/k3d-action@v2
        with:
          k3d-version: ${{ env.SETUP_K3D_VERSION }}
          cluster-name: "upstream"
          args: >-
            -p "80:80@agent:0:direct"
            -p "443:443@agent:0:direct"
            --api-port 6443
            --agents 1
            --network "nw01"
      -
        name: Provision k3d Downstream Cluster
        uses: AbsaOSS/k3d-action@v2
        with:
          k3d-version: ${{ env.SETUP_K3D_VERSION }}
          cluster-name: "downstream"
          args: >-
            -p "81:80@agent:0:direct"
            -p "444:443@agent:0:direct"
            --api-port 6644
            --agents 1
            --network "nw01"
      -
        name: Import Images Into k3d
        run: |
          ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev -c upstream
          ./.github/scripts/k3d-import-retry.sh rancher/fleet-agent:dev -c downstream
      -
        name: Set Up Tmate Debug Session
        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}
        uses: mxschmitt/action-tmate@v3
        timeout-minutes: 15
        with:
          limit-access-to-actor: true
      -
        name: Deploy Fleet
        run: |
          kubectl config use-context k3d-upstream
          ./.github/scripts/deploy-fleet.sh
      -
        name: Deploy and Register Downstream Fleet
        run: |
          kubectl create ns fleet-default
          kubectl apply -f - <<EOF
          apiVersion: "fleet.cattle.io/v1alpha1"
          kind: ClusterRegistrationToken
          metadata:
            name: second-token
            namespace: fleet-default
          spec:
            ttl: 12h
          EOF
          { grep -q -m 1 "second-token"; kill $!; } < <(kubectl get secrets -n fleet-default -l "fleet.cattle.io/managed=true" -w)

          token=$(kubectl get secret -n fleet-default second-token -o go-template='{{index .data "values" | base64decode}}' | yq eval .token -)
          ca=$(kubectl get secret -n cattle-fleet-system fleet-controller-bootstrap-token -o go-template='{{index .data "ca.crt" | base64decode}}')
          apiServerIP=$(kubectl get node k3d-upstream-server-0 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')

          # agent initiated cluster registration
          helm \
            --kube-context k3d-downstream \
            -n cattle-fleet-system \
              upgrade \
                --install \
                --create-namespace \
                --wait \
              fleet-agent charts/fleet-agent \
                --set-string labels.env=test \
                --set apiServerCA="$ca" \
                --set apiServerURL="https://$apiServerIP:6443" \
                --set clusterNamespace="fleet-default" \
                --set token="$token"

          echo "waiting for downstream cluster to be registered..."
          { grep -q -m 1 "k3d-downstream"; kill $!; } < <(kubectl get cluster -n fleet-default -w)

          echo "waiting for cluster to report being ready..."
          while [ $(kubectl -n fleet-default get cluster -o jsonpath='{.items[0].status.summary.ready}') -ne 1 ]; do
            sleep 1
          done
      -
        name: E2E tests
        env:
          FLEET_E2E_NS: fleet-local
          FLEET_E2E_NS_DOWNSTREAM: fleet-default
        run: |
          kubectl config use-context k3d-upstream
          ginkgo e2e/multi-cluster
      -
        name: Acceptance Tests for Examples
        if: >
          github.event_name == 'schedule'
        env:
          FLEET_E2E_NS: fleet-local
          FLEET_E2E_NS_DOWNSTREAM: fleet-default
        run: |
          ginkgo e2e/acceptance/multi-cluster-examples
      -
        name: Dump Failed Environment
        if: failure()
        run: |
          ./.github/scripts/dump-failed-k3ds.sh
      -
        name: Upload Logs
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: gha-fleet-mc-e2e-logs-${{ github.sha }}-${{ github.run_id }}
          path: |
            tmp/*.json
            tmp/*.log
          retention-days: 2