From c4f6b679b4b1169d22b5aa3fb6b5c716348df433 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 25 Sep 2024 15:47:18 -0400 Subject: [PATCH 1/2] feat: E2E tests iteration #2 (#237) - Validate that the kardinal manager pods are running - Validate access to the baseline and dev flows using the CLI gateway - Validate the the dev pod terminates when the flow is deleted - Validate that the kardinal manager pods are terminated after the manager is removed --- .github/workflows/ci-e2e-tests.yml | 128 +++++++++++++++++++++++--- kardinal-cli/deployment/deployment.go | 1 + 2 files changed, 114 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-e2e-tests.yml b/.github/workflows/ci-e2e-tests.yml index 4da217f..8902f36 100644 --- a/.github/workflows/ci-e2e-tests.yml +++ b/.github/workflows/ci-e2e-tests.yml @@ -90,6 +90,25 @@ jobs: run: | KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli manager deploy local-minikube + # Check that the three kardinal manager service pods are running and ready + while [ $(kubectl get pods -n default --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep "true" | wc -l) -ne 3 ] + do + echo "Waiting for kardinal manager pods to run..." + kubectl get pods -n default -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready + ((c++)) && ((c==12)) && exit 1 + sleep 10 + done + apps=$(kubectl get pods -n default -o custom-columns=:metadata.labels.app | tr " " "\n" | sort -g | tr "\n" " " | xargs) + echo ${apps} + if [ "${apps}" != "kardinal-manager trace-router trace-router-redis" ]; then exit 1; fi + + # Check for errors in the kardinal manager logs + if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep -q "ERRO" + then + echo "Errors found in the kardinal manager logs" + kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" + fi + - name: Deploy boutique demo manifest run: | KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli deploy -k ci/obd-demo.yaml @@ -97,27 +116,48 @@ jobs: - name: Validate that Kardinal manager applied the changes run: | # Check that the four baseline service pods are running and ready - while [ $(kubectl get pods -n baseline -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep "true,true" | wc -l) -ne 4 ] + while [ $(kubectl get pods -n baseline --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep "true,true" | wc -l) -ne 4 ] do echo "Waiting for baseline pods to run..." kubectl get pods -n baseline -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready ((c++)) && ((c==12)) && exit 1 sleep 10 done - kubectl get pods -n baseline - - # Check the right services are running. apps=$(kubectl get pods -n baseline -o custom-columns=:metadata.labels.app | tr " " "\n" | sort -g | tr "\n" " " | xargs) echo ${apps} if [ "${apps}" != "cartservice frontend postgres productcatalogservice" ]; then exit 1; fi # Check for errors in the kardinal manager logs - if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" + if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep -q "ERRO" then - echo "Errors found in the kardinal manager" + echo "Errors found in the kardinal manager logs" kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" fi + - name: Start gateway to the baseline flow and validate access + run: | + KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli gateway baseline > kardinal.out & + cli_pid=$! + while ! grep "Proxy server for host" kardinal.out + do + echo "Waiting for gateway to start..." + cat kardinal.out + ((c++)) && ((c==12)) && exit 1 + sleep 10 + done + flow_url=$(grep "http" kardinal.out | awk '{print $NF}') + status_code=$(curl -s -o /dev/null -w "%{http_code}" ${flow_url}) + echo "${flow_url} returned status code ${status_code}" + if [ "${status_code}" != "200" ]; then exit 1; fi + kill ${cli_pid} + + # Check for errors in the trace router logs + if kubectl logs -n default -l app=trace-router | grep -q "ERRO" + then + echo "Errors found in the trace router logs" + kubectl logs -n default -l app=trace-router | grep "ERRO" + fi + - name: Create, validate and delete flow run: | KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli flow create frontend kurtosistech/frontend:demo-frontend > kardinal.out @@ -126,7 +166,7 @@ jobs: KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli flow ls | grep ${flow_id} # Check that the dev service pod is running and ready - while ! kubectl get pods -n baseline -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep true | grep "frontend-${flow_id}" + while ! kubectl get pods -n baseline --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep true | grep "frontend-${flow_id}" do echo "Waiting for dev frontend pod to run..." kubectl get pods -n baseline -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready @@ -134,13 +174,46 @@ jobs: sleep 10 done kubectl get pods -n baseline + + # Start gateway to the flow and validate access + KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli gateway ${flow_id} > kardinal.out & + cli_pid=$! + while ! grep "Proxy server for host" kardinal.out + do + echo "Waiting for gateway to start..." + cat kardinal.out + ((c++)) && ((c==12)) && exit 1 + sleep 10 + done + flow_url=$(grep "http" kardinal.out | awk '{print $NF}') + status_code=$(curl -s -o /dev/null -w "%{http_code}" ${flow_url}) + echo "${flow_url} returned status code ${status_code}" + if [ "${status_code}" != "200" ]; then exit 1; fi + kill ${cli_pid} + # Check for errors in the trace router logs + if kubectl logs -n default -l app=trace-router | grep -q "ERRO" + then + echo "Errors found in the trace router logs" + kubectl logs -n default -l app=trace-router | grep "ERRO" + fi + KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli flow delete ${flow_id} - + + # Check that the dev service pod is not running anymore + while kubectl get pods -n baseline | grep "frontend-${flow_id}" + do + echo "Waiting for dev frontend pod to terminate..." + kubectl get pods -n baseline + ((c++)) && ((c==12)) && exit 1 + sleep 10 + done + kubectl get pods -n baseline + # Check for errors in the kardinal manager logs - if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" + if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep -q "ERRO" then - echo "Errors found in the kardinal manager" + echo "Errors found in the kardinal manager logs" kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" fi @@ -157,7 +230,7 @@ jobs: KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli flow ls | grep ${flow_id} # Check that the dev service pod is running and ready - while ! kubectl get pods -n baseline -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep true | grep "frontend-${flow_id}" + while ! kubectl get pods -n baseline --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep true | grep "frontend-${flow_id}" do echo "Waiting for dev frontend pod to run..." kubectl get pods -n baseline -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready @@ -168,10 +241,20 @@ jobs: KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli flow delete ${flow_id} + # Check that the dev service pod is not running anymore + while kubectl get pods -n baseline | grep "frontend-${flow_id}" + do + echo "Waiting for dev frontend pod to terminate..." + kubectl get pods -n baseline + ((c++)) && ((c==12)) && exit 1 + sleep 10 + done + kubectl get pods -n baseline + # Check for errors in the kardinal manager logs - if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" + if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep -q "ERRO" then - echo "Errors found in the kardinal manager" + echo "Errors found in the kardinal manager logs" kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" fi @@ -196,8 +279,23 @@ jobs: kubectl get pods -n baseline # Check for errors in the kardinal manager logs - if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" + if kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep -q "ERRO" then - echo "Errors found in the kardinal manager" + echo "Errors found in the kardinal manager logs" kubectl logs -n default -l dev.kardinal.app-id=kardinal-manager | grep "ERRO" fi + + - name: Remove kardinal manager + run: | + KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli manager remove > kardinal.out + + # Check that the services have been terminated + while [ "$(kubectl get pods -n default 2>&1 >/dev/null)" != "No resources found in default namespace." ] + do + echo "Waiting for the services to terminate..." + kubectl get pods -n default + ((c++)) && ((c==12)) && exit 1 + sleep 10 + done + kubectl get pods -n default + \ No newline at end of file diff --git a/kardinal-cli/deployment/deployment.go b/kardinal-cli/deployment/deployment.go index 40a89ec..0d8171c 100644 --- a/kardinal-cli/deployment/deployment.go +++ b/kardinal-cli/deployment/deployment.go @@ -73,6 +73,7 @@ spec: metadata: labels: {{.KardinalAppIDLabelKey}}: {{.KardinalManagerAppIDLabelValue}} + app: kardinal-manager spec: serviceAccountName: kardinal-manager containers: From 54bdc2144d7cacab7687d0ede777dd955db10df4 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 25 Sep 2024 16:05:09 -0400 Subject: [PATCH 2/2] fix: Clarify the manager CLI command deploy kontrol service location argument (#247) Some users are confused about the kardinal kontrol service location argument. Some of them are not able to differentiate between the kardinal manager location and the kontrol service location. I myself was also confused when I saw the manager deploy command. --- .github/workflows/ci-e2e-tests.yml | 2 +- kardinal-cli/cmd/root.go | 12 +++++++----- kardinal-cli/deployment/deployment.go | 7 ++++--- kardinal-cli/kontrol/location.go | 19 +++++++++++++++---- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-e2e-tests.yml b/.github/workflows/ci-e2e-tests.yml index 8902f36..354bd45 100644 --- a/.github/workflows/ci-e2e-tests.yml +++ b/.github/workflows/ci-e2e-tests.yml @@ -88,7 +88,7 @@ jobs: - name: Deploy Kardinal manager run: | - KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli manager deploy local-minikube + KARDINAL_CLI_DEV_MODE=TRUE /tmp/kardinal-cli manager deploy local-kardinal-kontrol # Check that the three kardinal manager service pods are running and ready while [ $(kubectl get pods -n default --no-headers -o custom-columns=NAMESPACE:metadata.namespace,POD:metadata.name,PodIP:status.podIP,READY-true:status.containerStatuses[*].ready | grep "true" | wc -l) -ne 3 ] diff --git a/kardinal-cli/cmd/root.go b/kardinal-cli/cmd/root.go index 1dd94e0..fbb28c2 100644 --- a/kardinal-cli/cmd/root.go +++ b/kardinal-cli/cmd/root.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "kardinal.cli/kubernetes" "log" "net" "net/http" @@ -17,6 +16,8 @@ import ( "strings" "time" + "kardinal.cli/kubernetes" + "gopkg.in/yaml.v3" "kardinal.cli/consts" "kardinal.cli/multi_os_cmd_executor" @@ -471,9 +472,10 @@ var topologyManifestCmd = &cobra.Command{ } var deployManagerCmd = &cobra.Command{ - Use: fmt.Sprintf("deploy [kontrol location] accepted values: %s and %s ", kontrol.KontrolLocationLocalMinikube, kontrol.KontrolLocationKloudKontrol), + Use: fmt.Sprintf("deploy [kardinal-kontrol service location] accepted values: %s and %s ", kontrol.KontrolLocationLocal, kontrol.KontrolLocationKloud), Short: "Deploy Kardinal manager into the cluster", - ValidArgs: []string{kontrol.KontrolLocationLocalMinikube, kontrol.KontrolLocationKloudKontrol}, + Long: "The Kardinal Manager retrieves the latest configuration from the Kardinal Kontrol service and applies changes to the user K8S topology. The Kardinal Kontrol service can be the one running in the Kardinal Cloud or can be the one deployed locally.", + ValidArgs: []string{kontrol.KontrolLocationLocal, kontrol.KontrolLocationKloud}, Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), Run: func(cmd *cobra.Command, args []string) { kontrolLocation := args[0] @@ -1087,10 +1089,10 @@ func getKontrolBaseURLForManager() (string, error) { ) switch kontrolLocation { - case kontrol.KontrolLocationLocalMinikube: + case kontrol.KontrolLocationLocal: scheme = httpSchme host = localMinikubeKontrolAPIHost - case kontrol.KontrolLocationKloudKontrol: + case kontrol.KontrolLocationKloud: scheme = httpsScheme host = kloudKontrolAPIHost default: diff --git a/kardinal-cli/deployment/deployment.go b/kardinal-cli/deployment/deployment.go index 0d8171c..08063a2 100644 --- a/kardinal-cli/deployment/deployment.go +++ b/kardinal-cli/deployment/deployment.go @@ -3,9 +3,10 @@ package deployment import ( "bytes" "context" - "kardinal.cli/kubernetes" "text/template" + "kardinal.cli/kubernetes" + "kardinal.cli/kontrol" "github.com/kurtosis-tech/stacktrace" @@ -208,9 +209,9 @@ func DeployKardinalManagerInCluster(ctx context.Context, clusterResourcesURL str var imagePullPolicy string switch kontrolLocation { - case kontrol.KontrolLocationLocalMinikube: + case kontrol.KontrolLocationLocal: imagePullPolicy = "IfNotPresent" - case kontrol.KontrolLocationKloudKontrol: + case kontrol.KontrolLocationKloud: imagePullPolicy = "Always" default: stacktrace.NewError("invalid Kontrol location: %s", kontrolLocation) diff --git a/kardinal-cli/kontrol/location.go b/kardinal-cli/kontrol/location.go index ad3e9df..8bd2f03 100644 --- a/kardinal-cli/kontrol/location.go +++ b/kardinal-cli/kontrol/location.go @@ -1,15 +1,19 @@ package kontrol import ( + "fmt" + "os" + "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" "kardinal.cli/host_machine_directories" - "os" ) const ( - KontrolLocationLocalMinikube = "local-minikube" - KontrolLocationKloudKontrol = "kloud-kontrol" + KontrolLocationLocal = "local-kardinal-kontrol" + KontrolLocationKloud = "kloud-kardinal-kontrol" + OldKontrolLocationLocal = "local-minikube" + OldKontrolLocationKloud = "kloud-kontrol" kontrolLocationFilePermissions os.FileMode = 0644 ) @@ -33,9 +37,10 @@ func GetKontrolLocation() (string, error) { return "", stacktrace.Propagate(err, "An error occurred getting the Kontrol location filepath") } + fmt.Println(kontrolLocationFilepath) _, err = os.Stat(kontrolLocationFilepath) if err != nil { - return "", stacktrace.Propagate(err, "An error occurred getting the Kontrol location file info") + return "", stacktrace.Propagate(err, "An error occurred getting the Kontrol location file info") } kontrolLocationFileBytes, err := os.ReadFile(kontrolLocationFilepath) @@ -44,6 +49,12 @@ func GetKontrolLocation() (string, error) { } kontrolLocationFileStr := string(kontrolLocationFileBytes) + switch kontrolLocationFileStr { + case OldKontrolLocationLocal: + kontrolLocationFileStr = KontrolLocationLocal + case OldKontrolLocationKloud: + kontrolLocationFileStr = KontrolLocationKloud + } logrus.Infof("Using Kontrol location %s", kontrolLocationFileStr) return kontrolLocationFileStr, nil