diff --git a/.gitignore b/.gitignore index 1d1786ef..95fa20e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .idea/* -tests/trino/cert.key -tests/trino/cert.crt +*/*/cert.key +*/*/cert.crt diff --git a/charts/gateway/README.md b/charts/gateway/README.md index 00e780f2..f5c1474f 100644 --- a/charts/gateway/README.md +++ b/charts/gateway/README.md @@ -39,6 +39,7 @@ A Helm chart for Trino Gateway ``` * `config.serverConfig."node.environment"` - string, default: `"test"` * `config.serverConfig."http-server.http.port"` - int, default: `8080` +* `config.serverConfig."http-server.http.enabled"` - bool, default: `true` * `config.dataStore.jdbcUrl` - string, default: `"jdbc:postgresql://localhost:5432/gateway"` The connection details for the backend database for Trino Gateway and Trino query history @@ -53,9 +54,23 @@ A Helm chart for Trino Gateway * `command` - list, default: `["java","-XX:MinRAMPercentage=80.0","-XX:MaxRAMPercentage=80.0","-jar","/usr/lib/trino/gateway-ha-jar-with-dependencies.jar","/etc/gateway/config.yaml"]` Startup command for Trino Gateway process. Add additional Java options and other modifications as desired. -* `service.type` - string, default: `"ClusterIP"` -* `service.port` - int, default: `8080` -* `service.annotations` - object, default: `{}` +* `service` - object, default: `{"ports":[{"name":"gateway","protocol":"TCP"}],"type":"ClusterIP"}` + + Service for accessing the gateway. The contents of this dictionary are used for the [service spec](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). `The `port` and `targetPort` of the first element of the ports list will automatically be set to the value of `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined the https port is used. In this case, an additional service for the http port must be configured manually. Additional ports, such as for JMX or a Java Agent can be configured by adding elements to the ports list. The selector is also automatically configured. All other values are passed through as is. Example configuration for exposing both https and http: + ```yaml + service: + type: NodePort + ports: + - protocol: TCP + name: request + nodePort: 30443 + # targetPort and port will automatically pulled from serverConfig.http-server.https.port + - protocol: TCP + name: gateway-http + nodePort: 30080 + port: 8080 + targetPort: 8080 + # targetPort and port should be manually set to serverConfig.http-server.http.port * `ingress.enabled` - bool, default: `false` * `ingress.className` - string, default: `""` * `ingress.annotations` - object, default: `{}` diff --git a/charts/gateway/templates/NOTES.txt b/charts/gateway/templates/NOTES.txt index 660b90f9..d0f59a71 100644 --- a/charts/gateway/templates/NOTES.txt +++ b/charts/gateway/templates/NOTES.txt @@ -6,23 +6,19 @@ You can get the Trino Gateway endpoints by running these commands: {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} - export REQUEST_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "request")].nodePort}' services {{ include "trino-gateway.fullname" . }}) - export APP_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "app")].nodePort}' services {{ include "trino-gateway.fullname" . }}) - export ADMIN_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "admin")].nodePort}' services {{ include "trino-gateway.fullname" . }}) + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}' services {{ include "trino-gateway.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath='{.items[0].status.addresses[0].address}') - echo http://$NODE_IP:$REQUEST_NODE_PORT + echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "trino-gateway.fullname" . }}' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "trino-gateway.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} + echo http://$SERVICE_IP:'{{ .Values.service.ports | first | get "port" }}' {{- else if contains "ClusterIP" .Values.service.type }} export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "trino-gateway.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export REQUEST_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "request")].containerPort}') - export APP_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "app")].containerPort}') - export ADMIN_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "admin")].containerPort}') + export PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[0].containerPort}') echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$REQUEST_PORT + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$PORT {{- end }} Happy Helming! diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml index 8ca31952..9e2feb26 100644 --- a/charts/gateway/templates/deployment.yaml +++ b/charts/gateway/templates/deployment.yaml @@ -41,21 +41,34 @@ spec: envFrom: {{- toYaml .Values.envFrom | nindent 12}} ports: - - name: request + {{- if index .Values "config" "serverConfig" "http-server.http.port" }} + - name: http containerPort: {{ index .Values "config" "serverConfig" "http-server.http.port" }} protocol: TCP + {{- end }} + {{- if index .Values "config" "serverConfig" "http-server.https.port" }} + - name: https + containerPort: {{ index .Values "config" "serverConfig" "http-server.https.port" }} + protocol: TCP + {{- end }} livenessProbe: httpGet: - path: /trino-gateway - port: {{ index .Values "config" "serverConfig" "http-server.http.port" }} + path: /trino-gateway + port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }} + {{- if index .Values "config" "serverConfig" "http-server.https.port" }} + scheme: HTTPS + {{- end }} initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} failureThreshold: {{ .Values.livenessProbe.failureThreshold }} timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} readinessProbe: httpGet: - path: /trino-gateway - port: {{ index .Values "config" "serverConfig" "http-server.http.port" }} + path: /trino-gateway + port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }} + {{- if index .Values "config" "serverConfig" "http-server.https.port" }} + scheme: HTTPS + {{- end }} initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} diff --git a/charts/gateway/templates/ingress.yaml b/charts/gateway/templates/ingress.yaml index ea8ca2e9..4e8d78a3 100644 --- a/charts/gateway/templates/ingress.yaml +++ b/charts/gateway/templates/ingress.yaml @@ -1,6 +1,7 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "trino-gateway.fullname" . -}} -{{- $svcPort := .Values.service.port -}} +{{- $svcPort := coalesce (index .Values "config" "serverConfig" "http-server.https.port") +(index .Values "config" "serverConfig" "http-server.http.port") -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} diff --git a/charts/gateway/templates/service.yaml b/charts/gateway/templates/service.yaml index 52f06a38..8eb06616 100644 --- a/charts/gateway/templates/service.yaml +++ b/charts/gateway/templates/service.yaml @@ -4,15 +4,25 @@ metadata: name: {{ include "trino-gateway.fullname" . }} labels: {{- include "trino-gateway.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ index .Values "config" "serverConfig" "http-server.http.port" }} - protocol: TCP - name: request - selector: - {{- include "trino-gateway.selectorLabels" . | nindent 4 }} +{{- $gatewayPort := "" }} +{{- if index .Values "config" "serverConfig" "http-server.http.enabled" }} + {{- $gatewayPort = index .Values "config" "serverConfig" "http-server.http.port" }} +{{- end }} +{{- if index .Values "config" "serverConfig" "http-server.https.enabled" }} +{{- $gatewayPort = index .Values "config" "serverConfig" "http-server.https.port" }} +{{- end }} +{{- if empty $gatewayPort }} +{{- fail "Error: No port defined in serverConfig!" $gatewayPort }} +{{- end}} +{{- $portDefault := dict "port" $gatewayPort "targetPort" $gatewayPort }} +{{- $portValues := .Values.service.ports | default list | first | default $portDefault}} +{{- $_0 := set $portValues "port" $gatewayPort}} +{{- $_1 := set $portValues "targetPort" $gatewayPort}} +{{- $ports := list $portValues }} +{{- $additionalPorts := .Values.service.ports | default list | rest }} +{{- $allPorts := concat $ports $additionalPorts}} +{{- $spec := .Values.service }} +{{- $_2 := set $spec "ports" $allPorts }} +{{- $selectorLabels := include "trino-gateway.selectorLabels" . | fromYaml }} +{{- $_3 := set $spec "selector" $selectorLabels }} +spec: {{ $spec | toYaml | nindent 2}} diff --git a/charts/gateway/templates/tests/test-connection.yaml b/charts/gateway/templates/tests/test-connection.yaml index 0c82340c..b675a92a 100644 --- a/charts/gateway/templates/tests/test-connection.yaml +++ b/charts/gateway/templates/tests/test-connection.yaml @@ -38,13 +38,20 @@ spec: mountPath: /etc/persistence containers: - name: wget - image: busybox + image: alpine # Get the list of backends, which should return an empty list, "[]". For this test to pass # the gateway must successfully connect to an initialized backend database command: - "sh" - "-c" - - '[ "$(wget {{ include "trino-gateway.fullname" . }}:{{ .Values.service.port }}/entity/GATEWAY_BACKEND -O -)" = "[]" ]' + - | + [ 1 = 1 ] + {{- if index .Values "config" "serverConfig" "http-server.https.enabled" -}} + && [ "$(wget --no-check-certificate https://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.https.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ] + {{- end }} + {{- if index .Values "config" "serverConfig" "http-server.http.enabled" -}} + && [ "$(wget http://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.http.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ] + {{- end }} volumes: - name: persistence-sql emptyDir: diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index 9eabc67e..4b2058cd 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -34,6 +34,7 @@ config: serverConfig: node.environment: test http-server.http.port: 8080 + http-server.http.enabled: true dataStore: # -- The connection details for the backend database for Trino Gateway and Trino query history jdbcUrl: jdbc:postgresql://localhost:5432/gateway @@ -58,10 +59,38 @@ command: - "/usr/lib/trino/gateway-ha-jar-with-dependencies.jar" - "/etc/gateway/config.yaml" +# -- Service for accessing the gateway. The contents of this dictionary are used +# for the [service spec](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). +# `The `port` and `targetPort` of the first element +# of the ports list will automatically be set to the value of +# `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined +# the https port is used. In this case, an additional service for the http port must be +# configured manually. Additional ports, such as for JMX or a Java Agent +# can be configured by adding elements to the ports list. The selector is +# also automatically configured. All other values are passed through as is. +# +# Example configuration for exposing both https and http: +# @raw +# ```yaml +# service: +# type: NodePort +# ports: +# - protocol: TCP +# name: request +# nodePort: 30443 +# # targetPort and port will automatically pulled from serverConfig.http-server.https.port +# - protocol: TCP +# name: gateway-http +# nodePort: 30080 +# port: 8080 +# targetPort: 8080 +# # targetPort and port should be manually set to serverConfig.http-server.http.port + service: type: ClusterIP - port: 8080 - annotations: {} + ports: + - protocol: TCP + name: gateway ingress: enabled: false diff --git a/tests/gateway/test-https.yaml b/tests/gateway/test-https.yaml new file mode 100644 index 00000000..824f0139 --- /dev/null +++ b/tests/gateway/test-https.yaml @@ -0,0 +1,46 @@ +command: + - "/bin/sh" + - "-c" + - | + cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \ + java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml + +config: + serverConfig: + http-server.http.enabled: false + http-server.https.enabled: true + http-server.https.port: 8443 + http-server.https.keystore.path: /etc/scratch/tls.pem + +volumes: + - name: certificates + secret: + secretName: certificates + - name: scratch + emptyDir: + sizeLimit: 10Mi + +volumeMounts: + - name: certificates + mountPath: /etc/certificates + readOnly: true + - name: scratch + mountPath: /etc/scratch + +ingress: + enabled: true + className: "" + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + cert-manager.io/cluster-issuer: acme + kubernetes.io/ingress.class: nginx + hosts: + - host: request.tg.starburst-customer-success.com + paths: + - path: / + pathType: ImplementationSpecific + tls: + - secretName: tls-secret-tg-green + hosts: + - request.tg.starburst-customer-success.com diff --git a/tests/gateway/test-nodeport.yaml b/tests/gateway/test-nodeport.yaml new file mode 100644 index 00000000..aef9bfea --- /dev/null +++ b/tests/gateway/test-nodeport.yaml @@ -0,0 +1,19 @@ +config: + serverConfig: + http-server.http.enabled: true + http-server.http.port: 8080 + http-server.https.enabled: true + http-server.https.port: 8443 + http-server.https.keystore.path: /etc/scratch/tls.pem + +service: + type: NodePort + ports: + - protocol: TCP + name: request + nodePort: 30443 + - protocol: TCP + name: gateway-http + nodePort: 30080 + port: 8080 + targetPort: 8080 diff --git a/tests/gateway/test.sh b/tests/gateway/test.sh index 5cef6755..b4dda029 100755 --- a/tests/gateway/test.sh +++ b/tests/gateway/test.sh @@ -5,13 +5,19 @@ set -euo pipefail declare -A testCases=( [complete_values]="--values test-values.yaml" [env_from]="--values test-values-with-env.yaml" + [nodeport]="--values test-values.yaml --values test-https.yaml --values test-nodeport.yaml" + [https]="--values test-values.yaml --values test-https.yaml" ) declare -A testCaseCharts=( [complete_values]="../../charts/gateway" [env_from]="../../charts/gateway" + [nodeport]="../../charts/gateway" + [https]="../../charts/gateway" ) +TEST_NAMES=(complete_values env_from nodeport https) + function join_by { local d=${1-} f=${2-} if shift 2; then @@ -24,13 +30,23 @@ NAMESPACE=trino-gateway-$(LC_ALL=C tr -dc 'a-z0-9' &2 "Generating a self-signed TLS certificate" +openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ + -subj "/O=Trino Software Foundation" \ + -addext "subjectAltName=DNS:localhost,DNS:*.$NAMESPACE,DNS:*.$NAMESPACE.svc,DNS:*.$NAMESPACE.svc.cluster.local,IP:127.0.0.1,IP:${NODE_IP}" \ + -keyout cert.key -out cert.crt +kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key --dry-run=client --output yaml | kubectl apply --filename - + HELM_EXTRA_SET_ARGS= CT_ARGS=( --skip-clean-up --helm-extra-args="--timeout 2m" ) CLEANUP_NAMESPACE=true -TEST_NAMES=(complete_values env_from) usage() { cat <&2