From 60aaeeffdca533692f63745aeb141773762038c5 Mon Sep 17 00:00:00 2001 From: Patrik Nordwall Date: Tue, 5 Dec 2023 14:22:48 +0100 Subject: [PATCH] test: Harden rollingupdate-kubernetes-cr-test.sh * failing in CI * I think it should wait for ready, not only running * more error diagnostics * jdk 17 image (seems to work) * but main issue is the resource requuests with minikube - argh, how many times are we going to run into this problem? * update minikube and kubernetes versions --- ...integration-tests-app-version-revision.yml | 4 +-- .../workflows/integration-tests-kube-api.yml | 4 +-- .../workflows/integration-tests-kube-dns.yml | 4 +-- .github/workflows/integration-tests-lease.yml | 4 +-- .github/workflows/integration-tests-maven.yml | 4 +-- .../integration-tests-rollingupdate-cr.yml | 4 +-- .../integration-tests-rollingupdate.yml | 4 +-- .../rollingupdate-kubernetes/build.sbt | 2 +- .../kubernetes/akka-cluster-cr.yml | 8 +---- .../app-version-revision-kubernetes-test.sh | 7 +++-- integration-test/scripts/kubernetes-test.sh | 2 ++ .../rollingupdate-kubernetes-cr-test.sh | 29 ++++++++++++++----- .../scripts/rollingupdate-kubernetes-test.sh | 25 ++++++++++++---- 13 files changed, 64 insertions(+), 37 deletions(-) diff --git a/.github/workflows/integration-tests-app-version-revision.yml b/.github/workflows/integration-tests-app-version-revision.yml index c73a33e8d..de692fcf7 100644 --- a/.github/workflows/integration-tests-app-version-revision.yml +++ b/.github/workflows/integration-tests-app-version-revision.yml @@ -45,8 +45,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.22.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/.github/workflows/integration-tests-kube-api.yml b/.github/workflows/integration-tests-kube-api.yml index 8a6bae394..5a9f94aef 100644 --- a/.github/workflows/integration-tests-kube-api.yml +++ b/.github/workflows/integration-tests-kube-api.yml @@ -44,8 +44,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.21.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/.github/workflows/integration-tests-kube-dns.yml b/.github/workflows/integration-tests-kube-dns.yml index 5ecb17632..41e426740 100644 --- a/.github/workflows/integration-tests-kube-dns.yml +++ b/.github/workflows/integration-tests-kube-dns.yml @@ -44,8 +44,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.21.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/.github/workflows/integration-tests-lease.yml b/.github/workflows/integration-tests-lease.yml index 68bfed687..414fc47ee 100644 --- a/.github/workflows/integration-tests-lease.yml +++ b/.github/workflows/integration-tests-lease.yml @@ -47,8 +47,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.21.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/.github/workflows/integration-tests-maven.yml b/.github/workflows/integration-tests-maven.yml index 93f5fbb81..a05aca3ec 100644 --- a/.github/workflows/integration-tests-maven.yml +++ b/.github/workflows/integration-tests-maven.yml @@ -43,8 +43,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.21.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/.github/workflows/integration-tests-rollingupdate-cr.yml b/.github/workflows/integration-tests-rollingupdate-cr.yml index 7b5fd7e62..d6e3d9cf8 100644 --- a/.github/workflows/integration-tests-rollingupdate-cr.yml +++ b/.github/workflows/integration-tests-rollingupdate-cr.yml @@ -45,8 +45,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.22.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/.github/workflows/integration-tests-rollingupdate.yml b/.github/workflows/integration-tests-rollingupdate.yml index 719a4a635..3e8c46c33 100644 --- a/.github/workflows/integration-tests-rollingupdate.yml +++ b/.github/workflows/integration-tests-rollingupdate.yml @@ -45,8 +45,8 @@ jobs: # v2.7.1 uses: manusa/actions-setup-minikube@4582844dcacbf482729f8d7ef696f515d2141bb9 with: - minikube version: 'v1.21.0' - kubernetes version: 'v1.22.0' + minikube version: 'v1.32.0' + kubernetes version: 'v1.28.4' driver: docker start args: '--addons=ingress' diff --git a/integration-test/rollingupdate-kubernetes/build.sbt b/integration-test/rollingupdate-kubernetes/build.sbt index a1b55cd22..fc19518e1 100644 --- a/integration-test/rollingupdate-kubernetes/build.sbt +++ b/integration-test/rollingupdate-kubernetes/build.sbt @@ -5,5 +5,5 @@ resolvers += "Akka library repository".at("https://repo.akka.io/maven") version := "1.3.3.7" // we hard-code the version here, it could be anything really dockerExposedPorts := Seq(8080, 8558, 2552) -dockerBaseImage := "docker.io/library/adoptopenjdk:11-jre-hotspot" +dockerBaseImage := "docker.io/library/eclipse-temurin:17.0.8.1_1-jre" dockerUpdateLatest := true diff --git a/integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml b/integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml index 2bdfc051e..63e0e4222 100644 --- a/integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml +++ b/integration-test/rollingupdate-kubernetes/kubernetes/akka-cluster-cr.yml @@ -27,12 +27,6 @@ spec: image: integration-test-rollingupdate-kubernetes:1.3.3.7 # Remove for a real project, the image is picked up locally for the integration test imagePullPolicy: Never - resources: - limits: - memory: "256Mi" - requests: - memory: "256Mi" - cpu: "300m" #health livenessProbe: httpGet: @@ -64,7 +58,7 @@ spec: - name: REQUIRED_CONTACT_POINT_NR value: "3" - name: JAVA_TOOL_OPTIONS - value: "-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -Dakka.rollingupdate.kubernetes.custom-resource.enabled=on" + value: "-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75 -Xlog:gc -Dakka.rollingupdate.kubernetes.custom-resource.enabled=on" #deployment --- #rbac-reader diff --git a/integration-test/scripts/app-version-revision-kubernetes-test.sh b/integration-test/scripts/app-version-revision-kubernetes-test.sh index 3a0e1bea4..4dbca9d74 100755 --- a/integration-test/scripts/app-version-revision-kubernetes-test.sh +++ b/integration-test/scripts/app-version-revision-kubernetes-test.sh @@ -27,7 +27,8 @@ testRevisionInPodsLog () { if [ $i -eq 20 ] then echo "Pods did not get ready (revision $1)" - kubectl -n $NAMESPACE describe deployment akka-rollingupdate-demo + kubectl -n $NAMESPACE events $APP_NAME + kubectl -n $NAMESPACE describe deployment $APP_NAME exit -1 fi @@ -56,7 +57,7 @@ testRevisionInPodsLog () { # prep docker images | head kubectl create namespace $NAMESPACE || true -kubectl -n $NAMESPACE delete deployment akka-rollingupdate-demo || true +kubectl -n $NAMESPACE delete deployment $APP_NAME || true kubectl -n $NAMESPACE apply -f $DEPLOYMENT # after the initial deployment @@ -70,4 +71,4 @@ testRevisionInPodsLog "2" kubectl rollout undo deployment/$APP_NAME -n $NAMESPACE testRevisionInPodsLog "3" -echo "Test Successful!" \ No newline at end of file +echo "Test Successful!" diff --git a/integration-test/scripts/kubernetes-test.sh b/integration-test/scripts/kubernetes-test.sh index 84a0a25b2..4db7e5300 100755 --- a/integration-test/scripts/kubernetes-test.sh +++ b/integration-test/scripts/kubernetes-test.sh @@ -19,6 +19,8 @@ done if [ $i -eq 10 ] then echo "Pods did not get ready" + kubectl -n $NAMESPACE events $APP_NAME + kubectl -n $NAMESPACE describe deployment $APP_NAME exit -1 fi diff --git a/integration-test/scripts/rollingupdate-kubernetes-cr-test.sh b/integration-test/scripts/rollingupdate-kubernetes-cr-test.sh index 37d518e74..058534551 100755 --- a/integration-test/scripts/rollingupdate-kubernetes-cr-test.sh +++ b/integration-test/scripts/rollingupdate-kubernetes-cr-test.sh @@ -9,21 +9,36 @@ docker images | head kubectl create namespace $NAMESPACE || true kubectl apply -f $CRD -kubectl -n $NAMESPACE delete deployment akka-rollingupdate-demo || true +kubectl -n $NAMESPACE delete deployment $APP_NAME || true kubectl -n $NAMESPACE apply -f $DEPLOYMENT -for i in {1..10} +for i in {1..20} do echo "Waiting for pods to get ready..." kubectl get pods -n $NAMESPACE - [ `kubectl get pods -n $NAMESPACE | grep Running | wc -l` -eq 3 ] && break + phase=$(kubectl get pods -o jsonpath="{.items[*].status.phase}" -n $NAMESPACE) + status=$(kubectl get pods -o jsonpath="{.items[*].status.containerStatuses[*].ready}" -n $NAMESPACE) + if [ "$phase" == "Running Running Running" ] && [ "$status" == "true true true" ] + then + break + fi sleep 4 done -if [ $i -eq 10 ] +if [ $i -eq 20 ] then echo "Pods did not get ready" - kubectl -n $NAMESPACE describe deployment akka-rollingupdate-demo + kubectl events $APP_NAME -n $NAMESPACE + kubectl describe deployment $APP_NAME -n $NAMESPACE + + echo "" + echo "Logs from all $APP_NAME containers" + kubectl logs -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true + + echo "" + echo "Logs from all previous $APP_NAME containers" + kubectl logs -p -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true + exit -1 fi @@ -37,7 +52,7 @@ do pod_list=$(kubectl get pods -n $NAMESPACE | grep $APP_NAME | grep Running | awk '{ print $1 }' | sort) # Get the pods in the CR - cr_pod_list=$(kubectl describe podcosts.akka.io akka-rollingupdate-demo -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort) + cr_pod_list=$(kubectl describe podcosts.akka.io $APP_NAME -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort) if [ "$pod_list" = "$cr_pod_list" ] then @@ -50,7 +65,7 @@ do for pod_name in $pod_list do # Get the pod names from the cr - cr_pod_list=$(kubectl describe podcosts.akka.io akka-rollingupdate-demo -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort -z) + cr_pod_list=$(kubectl describe podcosts.akka.io $APP_NAME -n $NAMESPACE | grep "Pod Name" | awk '{print $3}' | sort -z) # Check if the annotation value is set or empty if ["$pod_list" == "$cr_pod_list" ] diff --git a/integration-test/scripts/rollingupdate-kubernetes-test.sh b/integration-test/scripts/rollingupdate-kubernetes-test.sh index b5f2fdb45..96d5f02ec 100755 --- a/integration-test/scripts/rollingupdate-kubernetes-test.sh +++ b/integration-test/scripts/rollingupdate-kubernetes-test.sh @@ -8,21 +8,36 @@ sbt $PROJECT_NAME/docker:publishLocal docker images | head kubectl create namespace $NAMESPACE || true -kubectl -n $NAMESPACE delete deployment akka-rollingupdate-demo || true +kubectl -n $NAMESPACE delete deployment $APP_NAME || true kubectl -n $NAMESPACE apply -f $DEPLOYMENT -for i in {1..10} +for i in {1..20} do echo "Waiting for pods to get ready..." kubectl get pods -n $NAMESPACE - [ `kubectl get pods -n $NAMESPACE | grep Running | wc -l` -eq 3 ] && break + phase=$(kubectl get pods -o jsonpath="{.items[*].status.phase}" -n $NAMESPACE) + status=$(kubectl get pods -o jsonpath="{.items[*].status.containerStatuses[*].ready}" -n $NAMESPACE) + if [ "$phase" == "Running Running Running" ] && [ "$status" == "true true true" ] + then + break + fi sleep 4 done -if [ $i -eq 10 ] +if [ $i -eq 20 ] then echo "Pods did not get ready" - kubectl -n $NAMESPACE describe deployment akka-rollingupdate-demo + kubectl events $APP_NAME -n $NAMESPACE + kubectl describe deployment $APP_NAME -n $NAMESPACE + + echo "" + echo "Logs from all $APP_NAME containers" + kubectl logs -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true + + echo "" + echo "Logs from all previous $APP_NAME containers" + kubectl logs -p -l app=$APP_NAME --all-containers=true -n $NAMESPACE || true + exit -1 fi