Pass through all service spec values.

Support healthchecks for gateway configurations without http enabled. Add tests for https and NodePort service with both https and http exposed.
trinodb · Dec 11, 2024 · 10895a4 · 10895a4
1 parent 706f94c
commit 10895a4
Show file tree

Hide file tree

Showing 11 changed files with 271 additions and 39 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,3 @@
 .idea/*
-tests/trino/cert.key
-tests/trino/cert.crt
+*/*/cert.key
+*/*/cert.crt
diff --git a/charts/gateway/README.md b/charts/gateway/README.md
@@ -53,9 +53,25 @@ A Helm chart for Trino Gateway
 * `command` - list, default: `["java","-XX:MinRAMPercentage=80.0","-XX:MaxRAMPercentage=80.0","-jar","/usr/lib/trino/gateway-ha-jar-with-dependencies.jar","/etc/gateway/config.yaml"]`  
 
   Startup command for Trino Gateway process. Add additional Java options and other modifications as desired.
-* `service.type` - string, default: `"ClusterIP"`
-* `service.port` - int, default: `8080`
-* `service.annotations` - object, default: `{}`
+* `service` - object, default: `{"spec":{"ports":[{"name":"request","protocol":"TCP"}],"type":"ClusterIP"}}`  
+
+  Service for accessing the gateway. The `port` and `targetPort` of the first element of the ports list will automatically be set to the value of `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined the https port is used. In this case, an additional service for the http port must be configured manually. Additional ports, such as for JMX or a Java Agent can be configured by adding additional elements to the ports list. The selector is also automatically configured. All other values are passed through as is.
+  Example exposing both https and http:
+  ```yaml
+   service:
+     spec:
+       type: NodePort
+       ports:
+         - protocol: TCP
+           name: request
+           nodePort: 30443
+           # targetPort and port will automatically pulled from serverConfig.http-server.https.port
+         - protocol: TCP
+           name: gateway-http
+           nodePort: 30080
+           port: 8080
+           targetPort: 8080
+           # targetPort and port should be manually set to serverConfig.http-server.http.port
 * `ingress.enabled` - bool, default: `false`
 * `ingress.className` - string, default: `""`
 * `ingress.annotations` - object, default: `{}`

diff --git a/charts/gateway/templates/NOTES.txt b/charts/gateway/templates/NOTES.txt
@@ -5,24 +5,20 @@ You can get the Trino Gateway endpoints by running these commands:
   http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
   {{- end }}
 {{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export REQUEST_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "request")].nodePort}' services {{ include "trino-gateway.fullname" . }})
-  export APP_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "app")].nodePort}' services {{ include "trino-gateway.fullname" . }})
-  export ADMIN_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "admin")].nodePort}' services {{ include "trino-gateway.fullname" . }})
+{{- else if contains "NodePort" .Values.service.spec.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}' services {{ include "trino-gateway.fullname" . }})
   export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath='{.items[0].status.addresses[0].address}')
-  echo http://$NODE_IP:$REQUEST_NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.spec.type }}
      NOTE: It may take a few minutes for the LoadBalancer IP to be available.
            You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "trino-gateway.fullname" . }}'
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "trino-gateway.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
+  echo http://$SERVICE_IP:'{{ .Values.service.spec.ports | first | get "port" }}'
+{{- else if contains "ClusterIP" .Values.service.spec.type }}
   export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "trino-gateway.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export REQUEST_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "request")].containerPort}')
-  export APP_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "app")].containerPort}')
-  export ADMIN_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "admin")].containerPort}')
+  export PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[0].containerPort}')
   echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$REQUEST_PORT
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$PORT
 {{- end }}
 
 Happy Helming!
diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml
@@ -46,16 +46,22 @@ spec:
               protocol: TCP
           livenessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }}
+              {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+              scheme: HTTPS
+              {{- end }}
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
           readinessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }}
+              {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+              scheme: HTTPS
+              {{- end }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

diff --git a/charts/gateway/templates/service.yaml b/charts/gateway/templates/service.yaml
@@ -4,15 +4,20 @@ metadata:
   name: {{ include "trino-gateway.fullname" . }}
   labels:
     {{- include "trino-gateway.labels" . | nindent 4 }}
-  {{- with .Values.service.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
-      protocol: TCP
-      name: request
-  selector:
-    {{- include "trino-gateway.selectorLabels" . | nindent 4 }}
+{{- $gatewayPort := coalesce (index .Values "config" "serverConfig" "http-server.https.port")
+                    (index .Values "config" "serverConfig" "http-server.http.port") }}
+{{- if empty $gatewayPort }}
+{{- fail "Error: No port defined in serverConfig!"}}
+{{- end -}}
+{{- $portDefault := dict "port" $gatewayPort "targetPort" $gatewayPort }}
+{{- $portValues := .Values.service.spec.ports | default list | first | default $portDefault}}
+{{- $_0 := set $portValues "port" $gatewayPort}}
+{{- $_1 := set $portValues "targetPort" $gatewayPort}}
+{{- $ports := list $portValues }}
+{{- $additionalPorts := .Values.service.spec.ports | default list | rest }}
+{{- $allPorts := concat $ports $additionalPorts}}
+{{- $spec := .Values.service.spec }}
+{{- $_2 := set $spec "ports" $allPorts }}
+{{- $selectorLabels := include "trino-gateway.selectorLabels" . | fromYaml }}
+{{- $_3 := set $spec "selector" $selectorLabels }}
+spec: {{ $spec | toYaml | nindent 2}}
diff --git a/charts/gateway/templates/tests/test-connection.yaml b/charts/gateway/templates/tests/test-connection.yaml
@@ -12,6 +12,7 @@ spec:
   initContainers:
     - name: extract-persistence-sql
       image: "trinodb/trino-gateway"
+      imagePullPolicy: IfNotPresent
       command:
         - "/bin/sh"
         - "-c"
@@ -24,6 +25,7 @@ spec:
           mountPath: /etc/persistence
     - name: initialize-db
       image: bitnami/postgresql:17.1.0
+      imagePullPolicy: IfNotPresent
       command:
         - "/bin/sh"
         - "-c"
@@ -38,13 +40,18 @@ spec:
           mountPath: /etc/persistence
   containers:
     - name: wget
-      image: busybox
+      image: alpine
+      imagePullPolicy: IfNotPresent
       # Get the list of backends, which should return an empty list, "[]". For this test to pass
       # the gateway must successfully connect to an initialized backend database
       command:
         - "sh"
         - "-c"
-        - '[ "$(wget {{ include "trino-gateway.fullname" . }}:{{ .Values.service.port }}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+        - '[ "$(wget --no-check-certificate https://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.https.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        {{- else }}
+        - '[ "$(wget http://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.http.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        {{- end }}
   volumes:
     - name: persistence-sql
       emptyDir:

diff --git a/charts/gateway/templates/tests/test-nodeport.yaml b/charts/gateway/templates/tests/test-nodeport.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "trino-gateway.fullname" . }}-test-connection"
+  labels:
+    {{- include "trino-gateway.labels" . | nindent 4 }}
+    app.kubernetes.io/component: test
+    test: service
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: alpine
+      imagePullPolicy: IfNotPresent
+      # Get the list of backends, which should return an empty list, "[]". For this test to pass
+      # the gateway must successfully connect to an initialized backend database
+      command:
+        - "sh"
+        - "-c"
+        - |
+          {{- if eq .Values.service.spec.type "NodePort" }}
+          [ "$(wget --no-check-certificate https://${NODE_IP}:{{ index .Values.service.spec.ports 0 "nodePort"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ] &&
+          [ "$(wget http://${NODE_IP}:{{ index .Values.service.spec.ports 1 "nodePort"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]
+          {{- else }}
+          echo non NodePort service type
+          {{- end }}
+      envFrom:
+        - secretRef:
+            name: node-ip
+  restartPolicy: Never
diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml
@@ -58,10 +58,37 @@ command:
   - "/usr/lib/trino/gateway-ha-jar-with-dependencies.jar"
   - "/etc/gateway/config.yaml"
 
+# -- Service for accessing the gateway. The `port` and `targetPort` of the first element
+# of the ports list will automatically be set to the value of
+# `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined
+# the https port is used. In this case, an additional service for the http port must be
+# configured manually. Additional ports, such as for JMX or a Java Agent
+# can be configured by adding additional elements to the ports list. The selector is
+# also automatically configured. All other values are passed through as is.
+# @raw
+# Example exposing both https and http:
+# ```yaml
+#  service:
+#    spec:
+#      type: NodePort
+#      ports:
+#        - protocol: TCP
+#          name: request
+#          nodePort: 30443
+#          # targetPort and port will automatically pulled from serverConfig.http-server.https.port
+#        - protocol: TCP
+#          name: gateway-http
+#          nodePort: 30080
+#          port: 8080
+#          targetPort: 8080
+#          # targetPort and port should be manually set to serverConfig.http-server.http.port
+
 service:
-  type: ClusterIP
-  port: 8080
-  annotations: {}
+  spec:
+    type: ClusterIP
+    ports:
+      - protocol: TCP
+        name: request
 
 ingress:
   enabled: false

diff --git a/tests/gateway/test-https.yaml b/tests/gateway/test-https.yaml
@@ -0,0 +1,57 @@
+replicaCount: 1
+
+image:
+  # -- Repository location of the Trino Gateway image, typically `organization/imagename`
+  repository: "trinodb/trino-gateway"
+  pullPolicy: IfNotPresent
+
+command:
+  - "/bin/sh"
+  - "-c"
+  - |
+    cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \
+    java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml
+
+config:
+  serverConfig:
+    node.environment: test
+    http-server.http.enabled: false
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+  dataStore:
+    # The connection details for the backend database for Trino Gateway and Trino query history
+    jdbcUrl: jdbc:postgresql://gateway-backend-db-postgresql.postgres-gateway.svc.cluster.local:5432/gateway
+    user: gateway
+    password: pass0000
+    driver: org.postgresql.Driver
+  clusterStatsConfiguration:
+    monitorType: INFO_API
+  modules:
+    - io.trino.gateway.ha.module.HaGatewayProviderModule
+    - io.trino.gateway.ha.module.ClusterStateListenerModule
+    - io.trino.gateway.ha.module.ClusterStatsMonitorModule
+  managedApps:
+    - io.trino.gateway.ha.clustermonitor.ActiveClusterMonitor
+
+volumes:
+  - name: certificates
+    secret:
+      secretName: certificates
+  - name: scratch
+    emptyDir:
+      sizeLimit: 10Mi
+
+volumeMounts:
+  - name: certificates
+    mountPath: /etc/certificates
+  - name: scratch
+    mountPath: /etc/scratch
+
+resources:
+  limits:
+    cpu: 500m
+    memory: 256Mi
+  requests:
+    cpu: 250m
+    memory: 256Mi
diff --git a/tests/gateway/test-nodeport.yaml b/tests/gateway/test-nodeport.yaml
@@ -0,0 +1,70 @@
+replicaCount: 1
+
+image:
+  # -- Repository location of the Trino Gateway image, typically `organization/imagename`
+  repository: "trinodb/trino-gateway"
+  pullPolicy: IfNotPresent
+
+command:
+  - "/bin/sh"
+  - "-c"
+  - |
+    cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \
+    java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml
+
+config:
+  serverConfig:
+    node.environment: test
+    http-server.http.enabled: true
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+  dataStore:
+    # The connection details for the backend database for Trino Gateway and Trino query history
+    jdbcUrl: jdbc:postgresql://gateway-backend-db-postgresql.postgres-gateway.svc.cluster.local:5432/gateway
+    user: gateway
+    password: pass0000
+    driver: org.postgresql.Driver
+  clusterStatsConfiguration:
+    monitorType: INFO_API
+  modules:
+    - io.trino.gateway.ha.module.HaGatewayProviderModule
+    - io.trino.gateway.ha.module.ClusterStateListenerModule
+    - io.trino.gateway.ha.module.ClusterStatsMonitorModule
+  managedApps:
+    - io.trino.gateway.ha.clustermonitor.ActiveClusterMonitor
+
+resources:
+  limits:
+    cpu: 500m
+    memory: 256Mi
+  requests:
+    cpu: 250m
+    memory: 256Mi
+
+service:
+  spec:
+    type: NodePort
+    ports:
+      - protocol: TCP
+        name: request
+        nodePort: 30443
+      - protocol: TCP
+        name: gateway-http
+        nodePort: 30080
+        port: 8080
+        targetPort: 8080
+
+volumes:
+  - name: certificates
+    secret:
+      secretName: certificates
+  - name: scratch
+    emptyDir:
+      sizeLimit: 10Mi
+
+volumeMounts:
+  - name: certificates
+    mountPath: /etc/certificates
+  - name: scratch
+    mountPath: /etc/scratch