From 1a0417f69e3734ab8d197b7446ab64d8a54e4a71 Mon Sep 17 00:00:00 2001 From: shahargl Date: Sat, 16 Nov 2024 10:36:17 +0200 Subject: [PATCH] feat: add tests for proxies --- .github/workflows/test-haproxy-ingress.yml | 145 ++++++++++++++++++ .github/workflows/test-nginx-ingress.yml | 166 +++++++++++++++++++++ local_test_haproxy.sh | 139 +++++++++++++++++ local_test_nginx.sh | 131 ++++++++++++++++ 4 files changed, 581 insertions(+) create mode 100644 .github/workflows/test-haproxy-ingress.yml create mode 100644 .github/workflows/test-nginx-ingress.yml create mode 100644 local_test_haproxy.sh create mode 100644 local_test_nginx.sh diff --git a/.github/workflows/test-haproxy-ingress.yml b/.github/workflows/test-haproxy-ingress.yml new file mode 100644 index 0000000..651f10d --- /dev/null +++ b/.github/workflows/test-haproxy-ingress.yml @@ -0,0 +1,145 @@ +name: Test HAProxy Ingress + +on: + pull_request: + branches: + - main + workflow_dispatch: + inputs: + debug_enabled: + description: 'Run workflow with debug logging' + required: false + default: false + type: boolean + +jobs: + test-haproxy-ingress: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install test dependencies + run: | + npm install -g wscat + sudo apt-get update && sudo apt-get install -y curl + + - name: Create kind cluster + uses: helm/kind-action@v1.9.0 + with: + wait: 600s + config: | + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + + - name: Install HAProxy Ingress Controller + run: | + helm repo add haproxy-ingress https://haproxy-ingress.github.io/charts + helm repo update + helm install haproxy-ingress haproxy-ingress/haproxy-ingress \ + --set controller.service.type=NodePort \ + --set controller.watchIngressWithoutClass=true + + - name: Wait for HAProxy Ingress + run: | + kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=haproxy-ingress \ + --timeout=90s + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.6.1 + + - name: Create test values for haproxy + run: | + cat < haproxy-values.yaml + global: + ingress: + enabled: true + className: haproxy + classType: haproxy + websocketPrefix: /websocket + backendPrefix: /v2/docs + frontendPrefix: / + frontend: + enabled: true + backend: + enabled: true + websocket: + enabled: true + EOF + + - name: Debug - Show test values + if: ${{ inputs.debug_enabled }} + run: cat haproxy-values.yaml + + - name: Run chart-testing install + run: | + ct install --debug --config ct.yaml --namespace default --values haproxy-values.yaml + + - name: Wait for the backend pod + run: | + kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=keep,keep-component=backend \ + --timeout=90s + + - name: Debug - Show pods and services + if: ${{ inputs.debug_enabled }} + run: | + kubectl get pods -A + kubectl get svc -A + kubectl get ingress -A + kubectl describe ingress -A + + - name: Test HAProxy Ingress endpoints + run: | + INGRESS_IP="127.0.0.1" + APP_NAME=$(helm list -n default -o json | jq -r '.[0].name') + + echo "Testing frontend endpoint..." + RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://$INGRESS_IP/) + echo "Frontend response code: $RESP_CODE" + [ "$RESP_CODE" -eq 200 ] || exit 1 + + echo "Testing backend endpoint..." + RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://$INGRESS_IP/v2/docs/) + echo "Backend response code: $RESP_CODE" + [ "$RESP_CODE" -eq 200 ] || exit 1 + + echo "Testing websocket endpoint..." + RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Upgrade: websocket" -H "Connection: Upgrade" http://$INGRESS_IP/websocket/) + echo "WebSocket response code: $RESP_CODE" + [ "$RESP_CODE" -eq 101 ] || [ "$RESP_CODE" -eq 400 ] || exit 1 + + - name: Debug - Show logs on failure + if: ${{ failure() && inputs.debug_enabled }} + run: | + echo "HAProxy Ingress Controller logs:" + kubectl logs -l app.kubernetes.io/instance=haproxy-ingress -n default --tail=100 + + echo "Application pods logs:" + for pod in $(kubectl get pods -n default -l app.kubernetes.io/instance=$APP_NAME -o name); do + echo "Logs for $pod:" + kubectl logs $pod -n default --tail=100 + done \ No newline at end of file diff --git a/.github/workflows/test-nginx-ingress.yml b/.github/workflows/test-nginx-ingress.yml new file mode 100644 index 0000000..ae07844 --- /dev/null +++ b/.github/workflows/test-nginx-ingress.yml @@ -0,0 +1,166 @@ +name: Test Nginx Ingress + +on: + pull_request: + branches: + - main + workflow_dispatch: + inputs: + debug_enabled: + description: 'Run workflow with debug logging' + required: false + default: false + type: boolean + +jobs: + test-nginx-ingress: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install test dependencies + run: | + npm install -g wscat + sudo apt-get update && sudo apt-get install -y curl + + - name: Create kind cluster + uses: helm/kind-action@v1.9.0 + with: + wait: 600s + config: | + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + + - name: Install Nginx Ingress Controller + run: | + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo update + helm install nginx-ingress ingress-nginx/ingress-nginx \ + --set controller.service.type=NodePort \ + --set controller.watchIngressWithoutClass=true \ + --set controller.allowSnippetAnnotations=true \ + --set controller.config.allow-snippets=true \ + --set controller.config.enable-snippets=true \ + --set controller.hostPort.enabled=true \ + --set controller.service.ports.http=80 \ + --set controller.service.ports.https=443 \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.admissionWebhooks.enabled=false + + - name: Wait for Nginx Ingress + run: | + kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=nginx-ingress \ + --timeout=90s + + - name: Create test values for nginx + run: | + mkdir -p debug + cat < debug/nginx-values.yaml + global: + ingress: + enabled: true + className: nginx + classType: nginx + websocketPrefix: /websocket + backendPrefix: /v2/docs + frontendPrefix: / + frontend: + enabled: true + backend: + enabled: true + websocket: + enabled: true + EOF + + - name: Debug - Show test values + if: ${{ inputs.debug_enabled }} + run: cat debug/nginx-values.yaml + + - name: Install Keep chart + run: | + helm install keep ./charts/keep -f debug/nginx-values.yaml + + - name: Wait for the backend pod + run: | + kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=keep,keep-component=backend \ + --timeout=90s + + - name: Debug - Show resources + if: ${{ inputs.debug_enabled }} + run: | + echo "๐Ÿ” Checking all resources..." + kubectl get pods -A + kubectl get svc -A + kubectl get ingress -A + kubectl describe ingress -A + + - name: Test endpoints + run: | + echo "๐Ÿงช Testing endpoints..." + + echo "Frontend (/) - Should redirect to /incidents:" + RESP=$(curl -s -I http://localhost/) + echo "$RESP" | grep "HTTP" + LOCATION=$(echo "$RESP" | grep -i "location") + echo "$LOCATION" + if [[ "$LOCATION" != *"/incidents"* ]]; then + echo "โŒ Redirect to /incidents not found" + exit 1 + fi + + echo -e "\nBackend (/v2/docs) - Should return API info:" + RESP=$(curl -s http://localhost/v2/docs/) + echo "Response: $RESP" + if [[ $RESP != *"Rest API powering"* ]]; then + echo "โŒ Unexpected backend response" + exit 1 + fi + + echo -e "\nWebSocket (/websocket) - Testing connection:" + wscat -c "ws://localhost/websocket/app/keepappkey?protocol=7&client=js&version=8.3.0&flash=false" & + WSCAT_PID=$! + sleep 2 + kill $WSCAT_PID 2>/dev/null + + - name: Debug - Show logs on failure + if: ${{ failure() && inputs.debug_enabled }} + run: | + echo "๐Ÿ“œ Nginx Ingress Controller logs:" + kubectl logs -l app.kubernetes.io/instance=nginx-ingress --tail=100 + + echo "Application pods logs:" + for pod in $(kubectl get pods -n default -l app.kubernetes.io/instance=keep -o name); do + echo "Logs for $pod:" + kubectl logs $pod --tail=100 + done \ No newline at end of file diff --git a/local_test_haproxy.sh b/local_test_haproxy.sh new file mode 100644 index 0000000..7d3daf7 --- /dev/null +++ b/local_test_haproxy.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +# Create a debug folder and values file +mkdir -p debug +cat < debug/haproxy-values.yaml +global: + ingress: + enabled: true + className: haproxy + annotations: + ingress.class: haproxy + websocketPrefix: /websocket + backendPrefix: /v2/docs + frontendPrefix: / +frontend: + enabled: true +backend: + enabled: true +websocket: + enabled: true +EOF + +# Create Kind cluster with ingress ports exposed +cat < debug/kind-config.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - containerPort: 30024 + hostPort: 30024 + protocol: TCP +EOF + +# Function to check service/pod status +check_status() { + echo "๐Ÿ” Checking all resources..." + kubectl get pods -A + kubectl get svc -A + kubectl get ingress -A +} + +# Create cluster +kind create cluster --config debug/kind-config.yaml + +# Add and update HAProxy helm repo +echo "๐Ÿ“ฆ Adding HAProxy helm repo..." +helm repo add haproxytech https://haproxy-ingress.github.io/charts +helm repo update + +# Install HAProxy Ingress Controller +echo "๐Ÿ”ง Installing HAProxy Ingress Controller..." +helm install haproxy haproxy-ingress/haproxy-ingress \ + --set controller.service.type=NodePort \ + --set controller.kind=DaemonSet \ + --set controller.daemonset.useHostPort=true \ + --set controller.service.ports.http=80 \ + --set controller.service.ports.https=443 \ + --set controller.service.ports.stat=30024 + +# Wait for HAProxy ingress controller +echo "โณ Waiting for HAProxy ingress controller..." +kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=haproxy \ + --timeout=90s + +# Install your chart +echo "๐Ÿ“Š Installing Keep chart..." +helm install keep ./charts/keep -f debug/haproxy-values.yaml + +# Wait for backend pod (the last one to be ready) +echo "โณ Waiting for Keep backend to be ready..." +kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=keep,keep-component=backend \ + --timeout=90s + +# Check status +check_status + +# Test endpoints +echo "๐Ÿงช Testing endpoints..." + +echo "Frontend (/) - Should redirect to /incidents:" +RESP=$(curl -s -I http://localhost/) +echo "$RESP" | grep "HTTP" +LOCATION=$(echo "$RESP" | grep -i "location") +echo "$LOCATION" +if [[ "$LOCATION" == *"/incidents"* ]]; then + echo "โœ… Redirect to /incidents confirmed" +else + echo "โŒ Redirect to /incidents not found" + exit 1 +fi + +echo -e "\nBackend (/v2/docs) - Should return API info:" +RESP=$(curl -s http://localhost/v2/docs/) +echo "Response: $RESP" +if [[ $RESP == *"Rest API powering"* ]]; then + echo "โœ… Backend response contains expected message" +else + echo "โŒ Unexpected backend response" +fi + +echo -e "\nWebSocket (/websocket) - Testing connection:" +wscat -c "ws://localhost/websocket/app/keepappkey?protocol=7&client=js&version=8.3.0&flash=false" & +WSCAT_PID=$! +sleep 2 +kill $WSCAT_PID 2>/dev/null + +# Show HAProxy stats +echo -e "\n๐Ÿ“Š HAProxy Stats:" +curl -s "http://localhost:30024/stats?stats;csv" + +# Show logs if needed +echo -e "\n๐Ÿ“œ HAProxy controller logs:" +kubectl logs -l app.kubernetes.io/instance=haproxy --tail=100 + +# Keep script running for debugging +echo -e "\n๐Ÿ”„ Debug session active. Press Ctrl+C to cleanup." +echo "๐Ÿ“ Quick commands:" +echo " - kubectl get pods -A" +echo " - kubectl describe ingress" +echo " - kubectl logs " +echo " - curl -v http://localhost/" +echo " - curl -v http://localhost/v2/docs/" +echo " - curl -v -H \"Upgrade: websocket\" -H \"Connection: Upgrade\" http://localhost/websocket/" +echo " - curl http://localhost:30024/stats" + +# Wait for user interrupt +trap "kind delete cluster" EXIT +read -r -d '' _ debug/nginx-values.yaml +global: + ingress: + enabled: true + className: nginx + classType: nginx + websocketPrefix: /websocket + backendPrefix: /v2/docs + frontendPrefix: / +frontend: + enabled: true +backend: + enabled: true +websocket: + enabled: true +EOF + +# Create Kind cluster with ingress ports exposed +cat < debug/kind-config.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP +EOF + +# Function to check service/pod status +check_status() { + echo "๐Ÿ” Checking all resources..." + kubectl get pods -A + kubectl get svc -A + kubectl get ingress -A +} + +# Create cluster +kind create cluster --config debug/kind-config.yaml + +# Install Nginx Ingress +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm install nginx-ingress ingress-nginx/ingress-nginx \ + --set controller.service.type=NodePort \ + --set controller.watchIngressWithoutClass=true \ + --set controller.allowSnippetAnnotations=true \ + --set controller.config.allow-snippets=true \ + --set controller.config.enable-snippets=true \ + --set controller.hostPort.enabled=true \ + --set controller.service.ports.http=80 \ + --set controller.service.ports.https=443 \ + --set controller.nodeSelector."kubernetes\.io/os"=linux \ + --set controller.admissionWebhooks.enabled=false + +# Wait for ingress controller +echo "โณ Waiting for Nginx ingress controller..." +kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=nginx-ingress \ + --timeout=90s + +# Install your chart +helm install keep ./charts/keep -f debug/nginx-values.yaml + +# Wait for pods +echo "โณ Waiting for all pods to be ready..." +# Wait for backend pod (the last one to be ready) +echo "โณ Waiting for Keep backend to be ready..." +kubectl wait --namespace default \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/instance=keep,keep-component=backend \ + --timeout=90s + +# Check status +check_status + +# Test endpoints +echo "๐Ÿงช Testing endpoints..." + +echo "Frontend (/) - Should redirect to /incidents:" +RESP=$(curl -s -I http://localhost/) +echo "$RESP" | grep "HTTP" +LOCATION=$(echo "$RESP" | grep -i "location") +echo "$LOCATION" +if [[ "$LOCATION" == *"/incidents"* ]]; then + echo "โœ… Redirect to /incidents confirmed" +else + echo "โŒ Redirect to /incidents not found" + exit 1 +fi + +echo -e "\nBackend (/v2/docs) - Should return API info:" +RESP=$(curl -s http://localhost/v2/docs/) +echo "Response: $RESP" +if [[ $RESP == *"Rest API powering"* ]]; then + echo "โœ… Backend response contains expected message" +else + echo "โŒ Unexpected backend response" +fi + +echo -e "\nWebSocket (/websocket) - Testing connection:" +wscat -c "ws://localhost/websocket/app/keepappkey?protocol=7&client=js&version=8.3.0&flash=false" & +WSCAT_PID=$! +sleep 2 +kill $WSCAT_PID 2>/dev/null + +# Show logs if needed +echo -e "\n๐Ÿ“œ Ingress controller logs:" +kubectl logs -l app.kubernetes.io/instance=nginx-ingress --tail=100 + +# Keep script running for debugging +echo -e "\n๐Ÿ”„ Debug session active. Press Ctrl+C to cleanup." +echo "๐Ÿ“ Quick commands:" +echo " - kubectl get pods -A" +echo " - kubectl describe ingress" +echo " - kubectl logs " +echo " - curl -v http://localhost/" +echo " - curl -v http://localhost/v2/docs/" +echo " - curl -v -H \"Upgrade: websocket\" -H \"Connection: Upgrade\" http://localhost/websocket/" + +# Wait for user interrupt +trap "kind delete cluster" EXIT +read -r -d '' _