Add test OLM upgrade path

.github/workflows/test-upgrade.yaml

@@ -5,19 +5,19 @@ on:
   workflow_dispatch:
     inputs:
       kuadrantStartVersion:
-        description: Kuadrant start version
+        description: Kuadrant start version (format X.Y.Z)
         required: true
         type: string
       clusterServer:
-        description: Cluster server URL
+        description: Cluster Server URL
         required: false
         type: string
       clusterToken:
         description: Cluster Server Bearer Token
         required: false
         type: string
       kuadrantNamespace:
-        description: Namespace where Kuadrant is installed
+        description: Namespace
         required: false
         default: kuadrant-system
         type: string
@@ -59,7 +59,7 @@ jobs:
         if: ${{ !env.LOCAL_CLUSTER }}
         uses: azure/setup-kubectl@v4
         with:
-          version: v1.25.3
+          version: v1.31.0
       - name: Mask cluster token
         if: ${{ !env.LOCAL_CLUSTER }}
         run: |
@@ -132,3 +132,177 @@ jobs:
         run: |
           kubectl wait --timeout=300s --for=condition=Ready kuadrant kuadrant -n ${{ inputs.kuadrantNamespace }}
           echo "kuadrant upgrade ✅"
+  olm-upgrade-test:
+    runs-on: ubuntu-latest
+    name: OLM Upgrade Test
+    env:
+      KIND_CLUSTER_NAME: kuadrant-test
+      K8S_USER: kuadrant-admin  # can be whatever, it does not matter.
+      CLUSTER_NAME: remote-cluster
+      LOCAL_CLUSTER: ${{ inputs.clusterServer == '' && inputs.clusterToken == '' }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Install yq tool
+        run: |
+          # following sub-shells running make target should have yq already installed
+          make yq
+      - name: Deploy local Kind cluster
+        if: ${{ env.LOCAL_CLUSTER }}
+        uses: helm/[email protected]
+        with:
+          version: v0.23.0
+          config: utils/kind-cluster.yaml
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          wait: 120s
+      - name: Install kubectl for remote cluster
+        if: ${{ !env.LOCAL_CLUSTER }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: v1.31.0
+      - name: Mask cluster token
+        if: ${{ !env.LOCAL_CLUSTER }}
+        run: |
+          CLUSTER_TOKEN=$(jq -r '.inputs.clusterToken' $GITHUB_EVENT_PATH)
+          echo ::add-mask::$CLUSTER_TOKEN
+          echo CLUSTER_TOKEN=$CLUSTER_TOKEN >> $GITHUB_ENV
+      - name: Setup kubectl for remote cluster
+        if: ${{ !env.LOCAL_CLUSTER }}
+        run: |
+          kubectl config set-credentials ${{ env.K8S_USER }}/${{ env.CLUSTER_NAME }} --token ${{ env.CLUSTER_TOKEN }}
+          kubectl config set-cluster ${{ env.CLUSTER_NAME }} --insecure-skip-tls-verify=true --server=${{ inputs.clusterServer }}
+          kubectl config set-context ${{ inputs.kuadrantNamespace }}/${{ env.CLUSTER_NAME }}/${{ env.K8S_USER }} --user=${{ env.K8S_USER }}/${{ env.CLUSTER_NAME }} --namespace=${{ inputs.kuadrantNamespace }} --cluster=${{ env.CLUSTER_NAME }}
+          kubectl config use-context ${{ inputs.kuadrantNamespace }}/${{ env.CLUSTER_NAME }}/${{ env.K8S_USER }}
+      ## makes sure cluster is up and running
+      - run: |
+          kubectl cluster-info
+          kubectl get nodes
+      - name: Deploy OLM on local Kind cluster
+        if: ${{ env.LOCAL_CLUSTER }}
+        run: |
+          make install-olm
+      - name: Deploy pre-requisites on local Kind cluster
+        if: ${{ env.LOCAL_CLUSTER }}
+        run: |
+          make install-metallb
+          make install-cert-manager
+          make envoy-gateway-install
+          make deploy-eg-gateway
+      - name: Determine versions and URLs
+        id: versions
+        run: |
+          operator_repo=`make print-operator-repo`
+          echo starting_operator_url=${operator_repo}:v${{ inputs.kuadrantStartVersion }} >> $GITHUB_OUTPUT
+          catalog_repo=`make print-catalog-repo`
+          echo starting_catalog_url=${catalog_repo}:v${{ inputs.kuadrantStartVersion }} >> $GITHUB_OUTPUT
+          upgrade_version=`make read-release-version`
+          echo upgrade_version=$upgrade_version >> $GITHUB_OUTPUT
+          upgrade_operator_url=`make print-operator-image`
+          echo upgrade_operator_url=$upgrade_operator_url >> $GITHUB_OUTPUT
+          upgrade_bundle_url=`make print-bundle-image`
+          echo upgrade_bundle_url=$upgrade_bundle_url >> $GITHUB_OUTPUT
+          upgrade_catalog_url=`make print-catalog-image`
+          echo upgrade_catalog_url=${upgrade_catalog_url} >> $GITHUB_OUTPUT
+      - name: Print summary
+        run: echo "Installing version ${{ inputs.kuadrantStartVersion }}, upgrading to version ${{ steps.versions.outputs.upgrade_version }}"
+      - name: Create namespace
+        run: |
+          kubectl create namespace ${{ inputs.kuadrantNamespace }}
+      - name: Install Kuadrant ${{ inputs.kuadrantStartVersion }}
+        run: |
+          kubectl apply -f - <<EOF
+          ---
+          apiVersion: operators.coreos.com/v1alpha2
+          kind: OperatorGroup
+          metadata:
+            name: all-namespaces-operator-group
+            namespace: ${{ inputs.kuadrantNamespace }}
+          ---
+          apiVersion: operators.coreos.com/v1alpha1
+          kind: CatalogSource
+          metadata:
+            name: kuadrant-operator-catalog-starting-version
+            namespace: ${{ inputs.kuadrantNamespace }}
+          spec:
+            sourceType: grpc
+            image: ${{ steps.versions.outputs.starting_catalog_url }}
+            displayName: Kuadrant Operators
+            publisher: grpc
+            updateStrategy:
+              registryPoll:
+                interval: 45m
+          ---
+          apiVersion: operators.coreos.com/v1alpha1
+          kind: Subscription
+          metadata:
+            name: kuadrant-operator
+            namespace: ${{ inputs.kuadrantNamespace }}
+          spec:
+            channel: stable
+            name: kuadrant-operator
+            source: kuadrant-operator-catalog-starting-version
+            sourceNamespace: ${{ inputs.kuadrantNamespace }}
+          EOF
+      - name: Wait for subscription to be ready
+        run: |
+          kubectl wait --for=jsonpath='{.status.state}'=AtLatestKnown subscription/kuadrant-operator -n ${{ inputs.kuadrantNamespace }} --timeout=600s
+          echo "kuadrant subscription ✅"
+      - name: Wait for CSV to be ready
+        run: |
+          kubectl wait --for=jsonpath='{.status.phase}'=Succeeded csv/kuadrant-operator.v${{ inputs.kuadrantStartVersion }} --timeout=300s -n ${{ inputs.kuadrantNamespace }}
+          echo "kuadrant CSV ✅"
+      - name: Verify kuadrant start version deployment image
+        run: |
+          kubectl wait --timeout=300s --for=jsonpath='{.spec.template.spec.containers[0].image}'=${{ steps.versions.outputs.starting_operator_url }} deployment kuadrant-operator-controller-manager -n ${{ inputs.kuadrantNamespace }}
+          echo "kuadrant start version deployment image ✅"
+      - name: Deploy Kuadrant
+        run: |
+          kubectl -n ${{ inputs.kuadrantNamespace }} apply -f - <<EOF
+          apiVersion: kuadrant.io/v1beta1
+          kind: Kuadrant
+          metadata:
+            name: kuadrant
+          spec: {}
+          EOF
+      - name: Verify Kuadrant installation
+        run: |
+          kubectl wait --timeout=300s --for=condition=Ready kuadrant kuadrant -n ${{ inputs.kuadrantNamespace }}
+          echo "kuadrant installation ✅"
+      - name: Note for debugging failed upgrades
+        run: |
+          echo "📝 If the upgrade fails, you can debug it by running it locally with OLM and Catalog Operator in debugging mode."
+          echo "📝 If the upgrade fails, make sure dependencies have also upgrade path available from available operator catalogs in the cluster."
+      - name: Start upgrade to kuadrant ${{ steps.versions.outputs.upgrade_version }}
+        run: |
+          kubectl apply -f - <<EOF
+          apiVersion: operators.coreos.com/v1alpha1
+          kind: CatalogSource
+          metadata:
+            name: kuadrant-operator-catalog-upgraded-version
+            namespace: ${{ inputs.kuadrantNamespace }}
+          spec:
+            sourceType: grpc
+            image: ${{ steps.versions.outputs.upgrade_catalog_url }}
+            displayName: Kuadrant Operators
+            publisher: grpc
+            updateStrategy:
+              registryPoll:
+                interval: 45m
+          EOF
+          bin/operator-sdk run bundle-upgrade \
+          --namespace ${{ inputs.kuadrantNamespace }} \
+          --skip-tls-verify \
+          --timeout 3m0s \
+          ${{ steps.versions.outputs.upgrade_bundle_url }}
+      - name: Wait for upgraded CSV to be ready
+        run: |
+          kubectl wait --for=jsonpath='{.status.phase}'=Succeeded csv/kuadrant-operator.${{ steps.versions.outputs.upgrade_version }} --timeout=300s -n ${{ inputs.kuadrantNamespace }}
+          echo "kuadrant upgraded CSV ✅"
+      - name: Verify kuadrant upgraded version deployment image
+        run: |
+          kubectl wait --timeout=300s --for=jsonpath='{.spec.template.spec.containers[0].image}'=${{ steps.versions.outputs.upgrade_operator_url }} deployment kuadrant-operator-controller-manager -n ${{ inputs.kuadrantNamespace }}
+          echo "kuadrant upgraded version deployment image ✅"
+      - name: Verify Kuadrant upgrade
+        run: |
+          kubectl wait --timeout=300s --for=condition=Ready kuadrant kuadrant -n ${{ inputs.kuadrantNamespace }}
+          echo "kuadrant upgrade ✅"

Makefile

@@ -487,9 +487,15 @@ bundle-operator-image-url: $(YQ) ## Read operator image reference URL from the m
 read-release-version: ## Reads release version
 	@echo "v$(VERSION)"
 
-print-bundle-image: ## Pring bundle images.
+print-bundle-image: ## Print bundle image
 	@echo $(BUNDLE_IMG)
 
+print-operator-repo: ## Print operator repo
+	@echo $(IMAGE_TAG_BASE)
+
+print-operator-image: ## Print operator image
+	@echo $(IMG)
+
 .PHONY: update-catalogsource
 update-catalogsource:
 	@$(YQ) e -i '.spec.image = "${CATALOG_IMG}"' config/deploy/olm/catalogsource.yaml	

make/catalog.mk

@@ -2,7 +2,8 @@
 
 
 # The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0).
-CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(IMAGE_TAG)
+CATALOG_IMG_REPO ?= $(IMAGE_TAG_BASE)-catalog
+CATALOG_IMG ?= $(CATALOG_IMG_REPO):$(IMAGE_TAG)
 
 CATALOG_FILE = $(PROJECT_PATH)/catalog/kuadrant-operator-catalog/operator.yaml
 CATALOG_DOCKERFILE = $(PROJECT_PATH)/catalog/kuadrant-operator-catalog.Dockerfile
@@ -74,3 +75,9 @@ deploy-catalog: $(KUSTOMIZE) $(YQ) ## Deploy operator to the K8s cluster specifi
 .PHONY: undeploy-catalog
 undeploy-catalog: $(KUSTOMIZE) ## Undeploy controller from the K8s cluster specified in ~/.kube/config using OLM catalog image.
 	$(KUSTOMIZE) build config/deploy/olm | kubectl delete -f -
+
+print-catalog-repo: ## Print catalog repo
+	@echo $(CATALOG_IMG_REPO)
+
+print-catalog-image: ## Print catalog image
+	@echo $(CATALOG_IMG)