diff --git a/tunnel-server/deployment/k8s/.gitignore b/tunnel-server/deployment/k8s/.gitignore new file mode 100644 index 00000000..006fdeb0 --- /dev/null +++ b/tunnel-server/deployment/k8s/.gitignore @@ -0,0 +1,5 @@ +/tls.crt +/tls.key +/ssh_host_key +/cookie_secret +/config.env \ No newline at end of file diff --git a/tunnel-server/deployment/k8s/README.md b/tunnel-server/deployment/k8s/README.md new file mode 100644 index 00000000..18ee9ba4 --- /dev/null +++ b/tunnel-server/deployment/k8s/README.md @@ -0,0 +1,98 @@ +# Deploying your own instance of the Tunnel Server + +This directory contains an example deployment of the Tunnel Server on Kubernetes. + +Note that this is an advanced task which requires some networking and Kubernetes know-how. + +## Why + +Deploying a private instance of the Tunnel Server allows for fine-grained control of: + +- The URLs created for preview environments: e.g, use a custom domain. +- Geolocation of the server: reduced distance to environments can result better network performance. +- Security and privacy: deploy everything in your VPC, no traffic to 3rd parties. + +## Requirements + +- A Kubernetes cluster +- An ingress solution to make K8S Services accesible from your network (e.g, Traefik). In this example, we'll use your cloud provider's load balancer. +- A TLS certificate for your domain +- `kubectl` and `kustomize` + +## Overview + +The Tunnel Server natively listens on two ports: +- A SSH port which accepts tunneling SSH connections from environments +- A HTTP port which accepts requests from clients (browsers, etc) + +In this deployment scheme, both ports are wrapped with TLS using [`stunnel`](https://www.stunnel.org/). Both HTTP and SSH connections are accepted on a [single port](https://vadosware.io/post/stuffing-both-ssh-and-https-on-port-443-with-stunnel-ssh-and-traefik/) and routed using [`sslh`](https://github.com/yrutschle/sslh/) to the tunnel server ports. + +The `stunnel` port is then exposed using a [`LoadBalancer-type K8S Service`](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). + +## Instructions + +### 1. Setup the domain and the TLS certificate + +Make sure the certificate is for a wildcard subdomain, e.g, `*.yourdomain.example` + +Put the cert and key (in PEM format) in the files `tls.crt` and `tls.key` + +Copy `config.env.example` to `config.env` and set your domain in the `BASE_URL` variable. + +### 2. Generate a cookie secret + +The cookie secret is a simple text-based secret (like a password) in a file. + +```bash +LC_ALL=C tr -dc A-Za-z0-9 cookie_secret +``` + +### 3. Generate a SSH host key + +```bash +ssh-keygen -t ed25519 -N "" -f ssh_host_key +``` + +### 4. Generate and deploy the configuration + +Review the generated configuration: + +```bash +kustomize build . +``` + +To deploy to the K8S cluster: + +```bash +kustomize build . | kubectl apply -f - +``` + +Make sure the two deployments `tunnel-server` and `tunnel-server-stunnel` exist and that their pods are running. + +### 5. Test the SSH endpoint + +This requires OpenSSH and a recent-enough OpenSSL CLI with support for the `-quiet` option. + +To test the SSH endpoint (replace `$MY_DOMAIN` with your domain): + +```bash +MY_DOMAIN=yourdomain.example + +EXTERNAL_IP=$(kubectl get service tunnel-server-tls -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + +ssh -nT -o "ProxyCommand openssl s_client -quiet -verify_quiet -servername $MY_DOMAIN -connect %h:%p" -p 443 foo@$EXTERNAL_IP hello +``` + +### 6. Create a DNS entry for the `tunnel-server` Service external IP + +The external IP for the `tunnel-server` service is not guaranteed to be static. According to your Kubernetes provider, there could be multiple ways to define a DNS entry for it. Here are some guides: + +- Amazon AWS: [EKS](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/using-domain-names-with-elb.html) +- Google Cloud: [GKE](https://cloud.google.com/kubernetes-engine/docs/concepts/service-load-balancer) +- Azure: [AKS](https://learn.microsoft.com/en-us/azure/aks/load-balancer-standard) + +A different approach would be to use a 3rd-party ingress solution like [Traefik](https://doc.traefik.io/traefik/user-guides/crd-acme/). + +## Using your Tunnel Server instance with the Preevy CLI + +The `up` and `urls` commands accept a `-t` flag which can be used to set the Tunnel Server URL. Specify `ssh+tls://yourdomain.example` to use your instance. diff --git a/tunnel-server/deployment/k8s/config.example.env b/tunnel-server/deployment/k8s/config.example.env new file mode 100644 index 00000000..c7144600 --- /dev/null +++ b/tunnel-server/deployment/k8s/config.example.env @@ -0,0 +1 @@ +BASE_URL=https://local.livecycle.run:8044 diff --git a/tunnel-server/deployment/k8s/kustomization.yaml b/tunnel-server/deployment/k8s/kustomization.yaml new file mode 100644 index 00000000..bb234c5d --- /dev/null +++ b/tunnel-server/deployment/k8s/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- tunnel-server-stunnel.yaml +- tunnel-server.yaml +secretGenerator: +- files: + - tls.crt + - tls.key + name: tunnel-server-tls + type: kubernetes.io/tls +- files: + - ssh_host_key + name: tunnel-server-ssh + type: Opaque +- files: [cookie_secret] + name: tunnel-server-cookies + type: Opaque +configMapGenerator: +- name: tunnel-server-config + envs: [config.env] +- name: tunnel-server-sslh-config + files: [sslh.conf] +images: +- name: ghcr.io/livecycle/preevy/tunnel-server + newTag: main-5f80bc0 diff --git a/tunnel-server/deployment/k8s/sslh.conf b/tunnel-server/deployment/k8s/sslh.conf new file mode 100644 index 00000000..e1d3079f --- /dev/null +++ b/tunnel-server/deployment/k8s/sslh.conf @@ -0,0 +1,16 @@ +foreground: true; +verbose-config: 1; # print configuration at startup +verbose-config-error: 1; # print configuration errors +verbose-connections-error: 1; # connection errors +verbose-probe-error: 1; # failures and problems during probing +verbose-system-error: 1; # system call problem, i.e. malloc, fork, failing +verbose-int-error: 1; # internal errors, the kind that should never happen +listen: +( + { host: "0.0.0.0"; port: "2443"; } +); +protocols: +( + { name: "ssh"; service: "ssh"; host: "0.0.0.0"; port: "2222"; }, + { name: "http"; host: "0.0.0.0"; port: "3000"; }, +); \ No newline at end of file diff --git a/tunnel-server/deployment/k8s/tunnel-server-stunnel.yaml b/tunnel-server/deployment/k8s/tunnel-server-stunnel.yaml new file mode 100644 index 00000000..90146746 --- /dev/null +++ b/tunnel-server/deployment/k8s/tunnel-server-stunnel.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tunnel-server-stunnel + namespace: default +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: tunnel-server-stunnel + template: + metadata: + labels: + app: tunnel-server-stunnel + spec: + containers: + - env: + - name: STUNNEL_ACCEPT + value: 0.0.0.0:443 + - name: STUNNEL_SERVICE + value: tunnel-server + - name: STUNNEL_CONNECT + value: tunnel-server:443 + - name: STUNNEL_KEY + value: /etc/livecycle-ssl/tls.key + - name: STUNNEL_CRT + value: /etc/livecycle-ssl/tls.crt + - name: STUNNEL_DEBUG + value: err + image: dweomer/stunnel + imagePullPolicy: IfNotPresent + name: stunnel + ports: + - containerPort: 443 + protocol: TCP + resources: + limits: + cpu: 500m + ephemeral-storage: 1Gi + memory: 2Gi + requests: + cpu: 500m + ephemeral-storage: 1Gi + memory: 2Gi + securityContext: + capabilities: + drop: + - NET_RAW + volumeMounts: + - mountPath: /etc/livecycle-ssl + name: tls-cert + readOnly: true + restartPolicy: Always + securityContext: + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 30 + volumes: + - name: tls-cert + secret: + defaultMode: 420 + secretName: tunnel-server-tls +--- +apiVersion: v1 +kind: Service +metadata: + name: tunnel-server-tls +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + app: tunnel-server-stunnel + type: LoadBalancer diff --git a/tunnel-server/deployment/k8s/tunnel-server.yaml b/tunnel-server/deployment/k8s/tunnel-server.yaml new file mode 100644 index 00000000..02898393 --- /dev/null +++ b/tunnel-server/deployment/k8s/tunnel-server.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tunnel-server +spec: + replicas: 1 + selector: + matchLabels: + app: tunnel-server + template: + metadata: + labels: + app: tunnel-server + spec: + volumes: + - name: ssh + secret: + defaultMode: 400 + secretName: tunnel-server-ssh + - name: sslh-config + configMap: + defaultMode: 420 + name: tunnel-server-sslh-config + containers: + - env: + - name: COOKIE_SECRETS + valueFrom: + secretKeyRef: + name: tunnel-server-cookies + key: cookie_secret + - name: SSH_HOST_KEY_PATH + value: /etc/livecycle-ssh/ssh_host_key + - name: BASE_URL + valueFrom: + configMapKeyRef: + name: tunnel-server-config + key: BASE_URL + - name: DEBUG + value: "1" + - name: NODE_ENV + value: production + image: ghcr.io/livecycle/preevy/tunnel-server:main + imagePullPolicy: IfNotPresent + name: tunnel-server + ports: + - containerPort: 8888 + name: metrics + protocol: TCP + - containerPort: 2222 + name: ssh + protocol: TCP + - containerPort: 3000 + name: http + protocol: TCP + resources: + limits: + cpu: "1" + ephemeral-storage: 1Gi + memory: 2Gi + requests: + cpu: "1" + ephemeral-storage: 1Gi + memory: 2Gi + securityContext: + capabilities: + drop: + - NET_RAW + volumeMounts: + - mountPath: /etc/livecycle-ssh + name: ssh + readOnly: true + - image: oorabona/sslh:v2.0-rc1 + imagePullPolicy: IfNotPresent + name: sslh + command: [sslh-ev, --config=/etc/sslh/sslh.conf] + volumeMounts: + - mountPath: /etc/sslh + name: sslh-config + readOnly: true + ports: + - containerPort: 2443 + name: sslh + protocol: TCP + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: tunnel-server +spec: + ports: + - name: sslh + port: 443 + protocol: TCP + targetPort: 2443 + - name: metrics + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + app: tunnel-server + type: ClusterIP