test: Harden rollingupdate-kubernetes-cr-test.sh

* failing in CI * I think it should wait for ready, not only running * more error diagnostics * jdk 17 image (seems to work) * but main issue is the resource requuests with minikube - argh, how many times are we going to run into this problem? * update minikube and kubernetes versions
akka · Dec 6, 2023 · 60aaeef · 60aaeef
1 parent 2f8598c
commit 60aaeef
Show file tree

Hide file tree

Showing 13 changed files with 64 additions and 37 deletions.
diff --git a/.github/workflows/integration-tests-app-version-revision.yml b/.github/workflows/integration-tests-app-version-revision.yml
@@ -45,8 +45,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.22.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/.github/workflows/integration-tests-kube-api.yml b/.github/workflows/integration-tests-kube-api.yml
@@ -44,8 +44,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.21.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/.github/workflows/integration-tests-kube-dns.yml b/.github/workflows/integration-tests-kube-dns.yml
@@ -44,8 +44,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.21.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/.github/workflows/integration-tests-lease.yml b/.github/workflows/integration-tests-lease.yml
@@ -47,8 +47,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.21.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/.github/workflows/integration-tests-maven.yml b/.github/workflows/integration-tests-maven.yml
@@ -43,8 +43,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.21.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/.github/workflows/integration-tests-rollingupdate-cr.yml b/.github/workflows/integration-tests-rollingupdate-cr.yml
@@ -45,8 +45,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.22.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/.github/workflows/integration-tests-rollingupdate.yml b/.github/workflows/integration-tests-rollingupdate.yml
@@ -45,8 +45,8 @@ jobs:
         # v2.7.1
         uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9
         with:
-          minikube version: 'v1.21.0'
-          kubernetes version: 'v1.22.0'
+          minikube version: 'v1.32.0'
+          kubernetes version: 'v1.28.4'
           driver: docker
           start args: '--addons=ingress'
 

diff --git a/integration-test/rollingupdate-kubernetes/build.sbt b/integration-test/rollingupdate-kubernetes/build.sbt
@@ -5,5 +5,5 @@ resolvers += "Akka library repository".at("https://repo.akka.io/maven")
 version := "1.3.3.7" // we hard-code the version here, it could be anything really
 
 dockerExposedPorts := Seq(8080, 8558, 2552)
-dockerBaseImage := "docker.io/library/adoptopenjdk:11-jre-hotspot"
+dockerBaseImage := "docker.io/library/eclipse-temurin:17.0.8.1_1-jre"
 dockerUpdateLatest := true
diff --git a/integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml b/integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml
@@ -27,12 +27,6 @@ spec:
         image: integration-test-rollingupdate-kubernetes:1.3.3.7
         # Remove for a real project, the image is picked up locally for the integration test
         imagePullPolicy: Never
-        resources:
-          limits:
-            memory: "256Mi"
-          requests:
-            memory: "256Mi"
-            cpu: "300m"
         #health
         livenessProbe:
           httpGet:
@@ -64,7 +58,7 @@ spec:
         - name: REQUIRED_CONTACT_POINT_NR
           value: "3"
         - name: JAVA_TOOL_OPTIONS
-          value: "-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -Dakka.rollingupdate.kubernetes.custom-resource.enabled=on"
+          value: "-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -Xlog:gc -Dakka.rollingupdate.kubernetes.custom-resource.enabled=on"
 #deployment
 ---
 #rbac-reader

diff --git a/integration-test/scripts/app-version-revision-kubernetes-test.sh b/integration-test/scripts/app-version-revision-kubernetes-test.sh
@@ -27,7 +27,8 @@ testRevisionInPodsLog () {
   if [ $i -eq 20 ]
   then
     echo "Pods did not get ready (revision $1)"
-    kubectl -n $NAMESPACE describe deployment akka-rollingupdate-demo
+    kubectl -n $NAMESPACE events $APP_NAME
+    kubectl -n $NAMESPACE describe deployment $APP_NAME
     exit -1
   fi
 
@@ -56,7 +57,7 @@ testRevisionInPodsLog () {
 # prep
 docker images | head
 kubectl create namespace $NAMESPACE || true
-kubectl -n $NAMESPACE delete deployment akka-rollingupdate-demo || true
+kubectl -n $NAMESPACE delete deployment $APP_NAME || true
 kubectl -n $NAMESPACE apply -f $DEPLOYMENT
 
 # after the initial deployment
@@ -70,4 +71,4 @@ testRevisionInPodsLog "2"
 kubectl rollout undo deployment/$APP_NAME -n $NAMESPACE
 testRevisionInPodsLog "3"
 
-echo "Test Successful!"
+echo "Test Successful!"
diff --git a/integration-test/scripts/kubernetes-test.sh b/integration-test/scripts/kubernetes-test.sh
@@ -19,6 +19,8 @@ done
 if [ $i -eq 10 ]
 then
   echo "Pods did not get ready"
+  kubectl -n $NAMESPACE events $APP_NAME
+  kubectl -n $NAMESPACE describe deployment $APP_NAME
   exit -1
 fi
 

diff --git a/integration-test/scripts/rollingupdate-kubernetes-cr-test.sh b/integration-test/scripts/rollingupdate-kubernetes-cr-test.sh
@@ -9,21 +9,36 @@ docker images | head
 
 kubectl create namespace $NAMESPACE || true
 kubectl apply -f $CRD
-kubectl -n $NAMESPACE delete deployment akka-rollingupdate-demo || true
+kubectl -n $NAMESPACE delete deployment $APP_NAME || true
 kubectl -n $NAMESPACE apply -f $DEPLOYMENT
 
-for i in {1..10}
+for i in {1..20}
 do
   echo "Waiting for pods to get ready..."
   kubectl get pods -n $NAMESPACE
-  [ `kubectl get pods -n $NAMESPACE | grep Running | wc -l` -eq 3 ] && break
+  phase=$(kubectl get pods -o jsonpath="{.items[*].status.phase}" -n $NAMESPACE)
+  status=$(kubectl get pods -o jsonpath="{.items[*].status.containerStatuses[*].ready}" -n $NAMESPACE)
+  if [ "$phase" == "Running Running Running" ] && [ "$status" == "true true true" ]
+  then
+    break
+  fi
   sleep 4
 done
 
-if [ $i -eq 10 ]
+if [ $i -eq 20 ]
 then
   echo "Pods did not get ready"
-  kubectl -n $NAMESPACE describe deployment akka-rollingupdate-demo
+  kubectl events $APP_NAME -n $NAMESPACE
+  kubectl describe deployment $APP_NAME -n $NAMESPACE
+
+  echo ""
+  echo "Logs from all $APP_NAME containers"
+  kubectl logs -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true
+
+  echo ""
+  echo "Logs from all previous $APP_NAME containers"
+  kubectl logs -p -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true
+
   exit -1
 fi
 
@@ -37,7 +52,7 @@ do
   pod_list=$(kubectl get pods -n $NAMESPACE | grep $APP_NAME | grep Running | awk '{ print $1 }' | sort)
 
   # Get the pods in the CR
-  cr_pod_list=$(kubectl describe podcosts.akka.io akka-rollingupdate-demo -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort)
+  cr_pod_list=$(kubectl describe podcosts.akka.io $APP_NAME -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort)
 
   if [ "$pod_list" = "$cr_pod_list" ]
     then
@@ -50,7 +65,7 @@ do
   for pod_name in $pod_list
   do
     # Get the pod names from the cr
-    cr_pod_list=$(kubectl describe podcosts.akka.io akka-rollingupdate-demo -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort -z)
+    cr_pod_list=$(kubectl describe podcosts.akka.io $APP_NAME -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort -z)
 
     # Check if the annotation value is set or empty
     if ["$pod_list" == "$cr_pod_list" ]

diff --git a/integration-test/scripts/rollingupdate-kubernetes-test.sh b/integration-test/scripts/rollingupdate-kubernetes-test.sh
@@ -8,21 +8,36 @@ sbt $PROJECT_NAME/docker:publishLocal
 docker images | head
 
 kubectl create namespace $NAMESPACE || true
-kubectl -n $NAMESPACE delete deployment akka-rollingupdate-demo || true
+kubectl -n $NAMESPACE delete deployment $APP_NAME || true
 kubectl -n $NAMESPACE apply -f $DEPLOYMENT
 
-for i in {1..10}
+for i in {1..20}
 do
   echo "Waiting for pods to get ready..."
   kubectl get pods -n $NAMESPACE
-  [ `kubectl get pods -n $NAMESPACE | grep Running | wc -l` -eq 3 ] && break
+  phase=$(kubectl get pods -o jsonpath="{.items[*].status.phase}" -n $NAMESPACE)
+  status=$(kubectl get pods -o jsonpath="{.items[*].status.containerStatuses[*].ready}" -n $NAMESPACE)
+  if [ "$phase" == "Running Running Running" ] && [ "$status" == "true true true" ]
+  then
+    break
+  fi
   sleep 4
 done
 
-if [ $i -eq 10 ]
+if [ $i -eq 20 ]
 then
   echo "Pods did not get ready"
-  kubectl -n $NAMESPACE describe deployment akka-rollingupdate-demo
+  kubectl events $APP_NAME -n $NAMESPACE
+  kubectl describe deployment $APP_NAME -n $NAMESPACE
+
+  echo ""
+  echo "Logs from all $APP_NAME containers"
+  kubectl logs -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true
+
+  echo ""
+  echo "Logs from all previous $APP_NAME containers"
+  kubectl logs -p -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true
+
   exit -1
 fi