From 8f490d2c28831ac6d1b0ac4715e0a081cb0819d3 Mon Sep 17 00:00:00 2001 From: "Christian R. Garcia" Date: Thu, 28 Sep 2023 15:18:02 -0700 Subject: [PATCH] Sturdy proxy regardless of Pods. Also small fix for Abaco and change in Pods pvc. Proxy only changed in Kube for now. --- .../roles/actors/templates/kube/api/burndown | 7 ++-- .../actors/templates/kube/api/rabbit.yml | 2 +- .../roles/pods/templates/kube/nfs-pvc.yml | 28 --------------- .../proxy/templates/kube/nginx/nginx.conf | 36 ++++++++++--------- 4 files changed, 25 insertions(+), 48 deletions(-) diff --git a/playbooks/roles/actors/templates/kube/api/burndown b/playbooks/roles/actors/templates/kube/api/burndown index 85a499bf..5a15f984 100755 --- a/playbooks/roles/actors/templates/kube/api/burndown +++ b/playbooks/roles/actors/templates/kube/api/burndown @@ -15,7 +15,8 @@ kubectl delete -f nginx.yml # storage kubectl delete -f mongo.yml -#kubectl delete -f mongo-pvc.yml -# rabbit can be killed each time. If you drop rabbit, must also kill the pvc. kubectl delete -f rabbit.yml -kubectl delete -f rabbitmq-pvc.yml + +# pvc +# kubectl delete -f mongo-pvc.yml +# kubectl delete -f rabbitmq-pvc.yml \ No newline at end of file diff --git a/playbooks/roles/actors/templates/kube/api/rabbit.yml b/playbooks/roles/actors/templates/kube/api/rabbit.yml index 61ba1574..5262785d 100644 --- a/playbooks/roles/actors/templates/kube/api/rabbit.yml +++ b/playbooks/roles/actors/templates/kube/api/rabbit.yml @@ -12,6 +12,7 @@ spec: labels: app: actors-rabbit spec: + hostname: actors-rabbit # sets static hostname rather than k8 generated. Without rabbit can't use pvc to restart. containers: - name: actors-rabbit image: {{ actors_rabbitmq_image }} @@ -39,4 +40,3 @@ spec: - name: actors-rabbitmq-data persistentVolumeClaim: claimName: {{actors_rabbit_pvc}} - diff --git a/playbooks/roles/pods/templates/kube/nfs-pvc.yml b/playbooks/roles/pods/templates/kube/nfs-pvc.yml index 3787c56e..e37a94f1 100644 --- a/playbooks/roles/pods/templates/kube/nfs-pvc.yml +++ b/playbooks/roles/pods/templates/kube/nfs-pvc.yml @@ -9,31 +9,3 @@ spec: resources: requests: storage: 5Gi - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: pods-nfs-mkdirs -spec: - ttlSecondsAfterFinished: 60 - template: - spec: - restartPolicy: Never - containers: - - name: pods-nfs-vol - image: alpine:latest - command: - - sh - - -c - - | - mkdir -p /podsnfs/volumes - mkdir -p /podsnfs/snapshots - chmod 777 /podsnfs/volumes /podsnfs/snapshots - volumeMounts: - - name: pods-nfs-data - mountPath: /podsnfs - volumes: - - name: pods-nfs-data - persistentVolumeClaim: - claimName: pods-nfs-vol diff --git a/playbooks/roles/proxy/templates/kube/nginx/nginx.conf b/playbooks/roles/proxy/templates/kube/nginx/nginx.conf index 880895e0..2176dd7d 100644 --- a/playbooks/roles/proxy/templates/kube/nginx/nginx.conf +++ b/playbooks/roles/proxy/templates/kube/nginx/nginx.conf @@ -10,45 +10,49 @@ events { } {% if "pods" in proxy_nginx_service_list %} -### Everything first goes through this stream stanza. Map matches subdomain to port to route to. -### If no map found, we route to default 8443. This directs back to HTTP stanza as normal. +# Everything routes through here. Either goes to Pods or HTTP stanza stream { - log_format stream_routing '$remote_addr [$time_local] ' 'with SNI name "$ssl_preread_server_name" ' 'proxying to "$instanceport" ' '$protocol $status $bytes_sent $bytes_received ' '$session_time'; - # 'map' maps input string to output variable. Regex works. - # Ports used are purely random. Feel free to change. + # 'map's input request SNI (domain name) to output port. + # Ports used are purely random map $ssl_preread_server_name $instanceport { - # Route TCP with following whatever.pods.whatever.tenant.tapis.io to pods-traefik - # proxy_nginx_server_name would be `*.develop.tapis.io`, `*.tapis.io`, etc. + # *.pods.tenant.environment.tapis.io routes to pods-traefik "~*.pods.{{proxy_nginx_server_name}}" 5510; - # Route non-pod things to 8443 (arbitrary port, listened to by http stanza) to follow regular nginx walkthrough + # Route non-pod requests to 8443 which is read by http stanza "~{{proxy_nginx_server_name}}" 8443; - # Else default to 5510 - default 8443; + # Else default to 5510 which routes to pods (required for Postgres) + default 5510; } - # pods_service. Route TCP to pods-traefik pod. + # 'pods_upstream' routes to traefik. Backup is HTTP stanza. + # Works even if pods_service is down. + upstream pods_upstream { + server pods-traefik:80; + server 127.0.0.1:8443 backup; + } + + # 5510 routes to pod_service traefik pod. Backup is HTTP stanza. server { listen 5510; ssl_preread off; proxy_timeout 600s; - access_log /dev/stdout stream_routing; - proxy_pass pods-traefik:80; - + #access_log /dev/stdout stream_routing; # debug log if needed + proxy_pass pods_upstream; + proxy_next_upstream on; } # Listen for all incoming requests. Preread server name (for mapping). Then pass. server { listen 443; ssl_preread on; - proxy_connect_timeout 20s; # max time to connect to pserver proxy_timeout 600s; - access_log /dev/stdout stream_routing; + proxy_connect_timeout 20s; # max time to connect to pserver + #access_log /dev/stdout stream_routing; proxy_pass 127.0.0.1:$instanceport; } }