to be squashed: simplify and document

dev-docs/howto/vpn/helm/README.md

@@ -16,8 +16,6 @@ This Helm chart deploys a VPN server to your Constellation cluster.
    helm install -f config.yaml vpn . 
    ```
 
-3. Follow the post-installation instructions displayed by the CLI.
-
 ## Things to try
 
 Ask CoreDNS about its own service IP:
@@ -49,8 +47,27 @@ The service IP range is handed to a transparent proxy running in the VPN fronten
 * NetworkPolicy can't be applied selectively to the on-prem ranges.
 * No connectivity from Constellation to on-prem workloads.
 
-## Alternatives
+## Bidirectional VPN Connectivity
 
 If NAT and proxy are not acceptable for your environment, there's an
-[experimental alternative mode](README.experimental.md) that establishes full
-L3 connectivity between pods and on-prem services.
+alternative mode that establishes full IP connectivity between pods and on-prem
+services, activated by the `l3.enabled` Helm value.
+
+> **WARNING**: The experimental version of this Helm chart is, well,
+> experimental. It messes with the node network configuration and has the
+> potential to break all networking. It's only tested on GCP.
+> Use at your own risk!
+
+In addition to the NAT-based resources, the frontend contains an init container
+that sets up a networking bypass around Cilium. This is necessary to circumvent
+the restrictions that Cilium applies to pod traffic (source IP enforcement, for
+example). VPN traffic is routed directly to the host network, which in turn is
+modified to forward VPN traffic correctly.
+
+A VPN operator deployment is added that configures the `CiliumEndpoint` with
+on-prem IP ranges.
+
+In case this chart causes problems with cluster networking that are not
+resolved by uninstalling it, rebooting the worker nodes to start with a fresh
+network setup should help. There's a button for this in the *Instance Group*
+view of GCP.

dev-docs/howto/vpn/helm/files/routing/experimental/all-nodes.sh → dev-docs/howto/vpn/helm/files/routing/all-nodes.sh

@@ -6,8 +6,6 @@
 # We detect which mode we're in by looking for the magic network interface that
 # the VPN frontend creates.
 
-# TODO: Check if we're accidentally going through the cloud router (can we prevent this with tc?)
-
 set -eu
 
 reconcile_vpn() {
@@ -34,6 +32,9 @@ cleanup_vpn() {
 
 reconcile_other() {
   cleanup_vpn 2> /dev/null
+
+  # Adding this route prevents VPN traffic from being SNATed by Cilium on the
+  # default route through the physical interface.
   for cidr in ${VPN_PEER_CIDRS}; do
     ip route replace "${cidr}" dev cilium_wg0
   done

dev-docs/howto/vpn/helm/files/routing/experimental/frontend-pod.sh → dev-docs/howto/vpn/helm/files/routing/pod-l3-setup.sh

@@ -10,6 +10,7 @@ fi
 # Set up a parallel veth connection to the host network namespace so that we
 # are not subject to Cilium restrictions (e.g. source IPs).
 
+rm -f /var/run/netns/root
 ip netns attach root 1
 
 ip link add vpn_upper type veth peer name "${MAGIC_INTERFACE}"
@@ -27,7 +28,9 @@ ip link set dev vpn_upper up
 table=41
 ip route replace 169.254.42.1 dev br0
 ip route replace default via 169.254.42.1 dev br0 table "${table}"
-# IPSec
-ip rule add to "${VPN_POD_CIDR}" fwmark 0x8/0x8 table "${table}" priority 10
-# Wireguard
-ip rule add to "${VPN_POD_CIDR}" iif vpn_wg0 table "${table}" priority 11
+ip rule add to "${VPN_POD_CIDR}" iif "${VPN_INTERFACE}" table "${table}" priority 11
+
+for cidr in ${VPN_PEER_CIDRS}; do
+  ip route replace "${cidr}" dev "${VPN_INTERFACE}"
+done
+

dev-docs/howto/vpn/helm/files/routing/service-proxy-setup.sh

@@ -12,13 +12,17 @@ mark=0x1/0x1
 ip rule add pref "${pref}" fwmark "${mark}" lookup "${table}"
 ip route replace local 0.0.0.0/0 dev lo table "${table}"
 
-iptables -t mangle -N VPN_PRE || iptables -t mangle -F VPN_PRE
 
+iptables -t mangle -N VPN_DIVERT || iptables -t mangle -F VPN_DIVERT
+iptables -t mangle -A VPN_DIVERT -j MARK --set-mark 1
+iptables -t mangle -A VPN_DIVERT -j ACCEPT
+iptables -t mangle -A PREROUTING -p tcp -m socket -j VPN_DIVERT
+
+iptables -t mangle -N VPN_PRE || iptables -t mangle -F VPN_PRE
 for cidr in ${VPN_PEER_CIDRS}; do
   for proto in tcp udp; do
     iptables -t mangle -A VPN_PRE -p "${proto}" -s "${cidr}" -d "${VPN_SERVICE_CIDR}" \
       -j TPROXY --tproxy-mark "${mark}" --on-port 61001
   done
 done
-
 iptables -t mangle -C PREROUTING -j VPN_PRE || iptables -t mangle -A PREROUTING -j VPN_PRE

dev-docs/howto/vpn/helm/files/strongswan/strongswan-setup.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -eu
+
+ip link add dev "${VPN_INTERFACE}" type xfrm dev eth0 if_id 0xfe
+ip link set dev "${VPN_INTERFACE}" up

dev-docs/howto/vpn/helm/files/wireguard-setup.sh

@@ -2,12 +2,6 @@
 
 set -eu
 
-dev=vpn_wg0
-
-ip link add dev "${dev}" type wireguard
-wg setconf "${dev}" /etc/wireguard/wg.conf
-ip link set dev "${dev}" up
-
-for cidr in ${VPN_PEER_CIDRS}; do
-  ip route replace "${cidr}" dev "${dev}"
-done
+ip link add dev "${VPN_INTERFACE}" type wireguard
+wg setconf "${VPN_INTERFACE}" /etc/wireguard/wg.conf
+ip link set dev "${VPN_INTERFACE}" up

dev-docs/howto/vpn/helm/templates/_helpers.tpl

@@ -39,4 +39,6 @@ app.kubernetes.io/instance: {{ .Release.Name }}
   value: {{ .Values.serviceCIDR | quote }}
 - name: MAGIC_INTERFACE
   value: cilium_c11n_vpn
+- name: VPN_INTERFACE
+  value: vpn0
 {{- end }}

dev-docs/howto/vpn/helm/templates/configmaps.yaml

@@ -1,13 +1,10 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ include "..fullname" . }}-routes
+  name: {{ include "..fullname" . }}-scripts
   labels: {{- include "..labels" . | nindent 4 }}
 data:
 {{ (.Files.Glob "files/routing/*.sh").AsConfig | indent 2 }}
-{{- if .Values.experimental.l3.enabled }}
-{{ (.Files.Glob "files/routing/experimental/*.sh").AsConfig | indent 2 }}
-{{- end }}
 {{- if .Values.wireguard.enabled }}
 ---
 apiVersion: v1

dev-docs/howto/vpn/helm/templates/operator-deployment.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.experimental.l3.enabled -}}
+{{ if .Values.l3.enabled -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -30,5 +30,5 @@ spec:
       volumes:
       - name: scripts
         configMap:
-          name: {{ include "..fullname" . }}-routes
+          name: {{ include "..fullname" . }}-scripts
 {{- end }}

dev-docs/howto/vpn/helm/templates/rbac.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.experimental.l3.enabled -}}
+{{ if .Values.l3.enabled -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

dev-docs/howto/vpn/helm/templates/routing-daemonset.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.experimental.l3.enabled }}
+{{ if .Values.l3.enabled }}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -29,11 +29,11 @@ spec:
         command: ["/bin/sh", "/script/all-nodes.sh"]
         env: {{- include "..commonEnv" . | nindent 10 }}
         volumeMounts:
-        - name: routes
+        - name: scripts
           mountPath: "/script"
           readOnly: true
       volumes:
-      - name: routes
+      - name: scripts
         configMap:
-          name: {{ include "..fullname" . }}-routes
+          name: {{ include "..fullname" . }}-scripts
 {{- end }}

dev-docs/howto/vpn/helm/templates/strongswan-secret.tpl

@@ -1,6 +1,8 @@
 {{- define "strongswan.swanctl-conf" }}
 connections {
     net-net {
+      if_id_in = 0xfe
+      if_id_out = 0xfe
       remote_addrs = {{ .Values.ipsec.peer }}
       local {
           auth = psk
@@ -13,7 +15,6 @@ connections {
               local_ts  = {{ .Values.podCIDR }},{{ .Values.serviceCIDR }}
               remote_ts = {{ join "," .Values.peerCIDRs }}
               start_action = trap
-              set_mark_in = "0x8/0x8"
           }
       }
     }

dev-docs/howto/vpn/helm/templates/strongswan-statefulset.yaml

@@ -15,19 +15,31 @@ spec:
         {{- include "..selectorLabels" . | nindent 8 }}
         component: frontend
     spec:
-      {{- if .Values.experimental.l3.enabled }}
+      {{- if .Values.l3.enabled }}
       hostPID: true
       {{- end }}
       initContainers:
-      {{- if .Values.experimental.l3.enabled }}
+      - name: strongswan-setup
+        image: "nixery.dev/shell/iproute2"
+        command: ["/bin/sh", "-x", "/entrypoint.sh"]
+        env: {{- include "..commonEnv" . | nindent 10 }}
+        securityContext:
+          capabilities:
+            add: ["NET_ADMIN"]
+        volumeMounts:
+        - name: strongswan
+          mountPath: "/entrypoint.sh"
+          subPath: "strongswan-setup.sh"
+          readOnly: true
+      {{- if .Values.l3.enabled }}
       - name: networking-setup
         image: nixery.dev/shell/iptables/iproute2
-        command: ["/bin/sh", "-x", "/scripts/frontend-pod.sh"]
+        command: ["/bin/sh", "-x", "/scripts/pod-l3-setup.sh"]
         env: {{- include "..commonEnv" . | nindent 10 }}
         securityContext:
           privileged: true
         volumeMounts:
-        - name: routes
+        - name: scripts
           mountPath: "/scripts"
           readOnly: true
         - name: netns
@@ -41,7 +53,7 @@ spec:
           capabilities:
             add: ["NET_ADMIN"]
         volumeMounts:
-        - name: routes
+        - name: scripts
           mountPath: "/scripts"
           readOnly: true
       {{- end }}
@@ -53,7 +65,7 @@ spec:
           capabilities:
             add: ["NET_ADMIN"]
         volumeMounts:
-        - name: routes
+        - name: scripts
           mountPath: "/scripts"
           readOnly: true
       containers:
@@ -86,9 +98,9 @@ spec:
       volumes:
       - name: netns
         emptyDir: {}
-      - name: routes
+      - name: scripts
         configMap:
-          name: {{ include "..fullname" . }}-routes
+          name: {{ include "..fullname" . }}-scripts
       - name: strongswan
         projected:
           sources:
@@ -104,4 +116,6 @@ spec:
                   path: entrypoint.sh
                 - key: charon-logging.conf
                   path: charon-logging.conf
+                - key: strongswan-setup.sh
+                  path: strongswan-setup.sh
 {{- end }}