Add node-to-node strict mode

This commit extends the WireGuard pod-to-pod strict mode to
also allow to secure node-to-node encryption when WireGuard
is used.

Signed-off-by: Leonard Cohnen <[email protected]>

bpf/bpf_alignchecker.c

@@ -85,3 +85,4 @@ add_type(struct tunnel_key);
 add_type(struct tunnel_value);
 add_type(struct auth_key);
 add_type(struct auth_info);
+add_type(struct strict_mode_policy);

bpf/bpf_host.c

@@ -1400,11 +1400,17 @@ int cil_to_netdev(struct __ctx_buff *ctx __maybe_unused)
 		return send_drop_notify_error(ctx, 0, ret, CTX_ACT_DROP,
 					      METRIC_EGRESS);
 
-#if defined(ENCRYPTION_STRICT_MODE)
+	/* We disable the WireGuard strict mode if the tunnel mode is enabled,
+	 * since we have a check earlier in the datapath before encapsulation.
+	 * We chose this approach so that we don't have to decapsulate the
+	 * packet to check if the packet's original destination is allowed to
+	 * be sent unencrypted.
+	 */
+#if defined(ENCRYPTION_STRICT_MODE) && (!defined(TUNNEL_MODE) || defined(ENABLE_NODE_ENCRYPTION))
 	if (!strict_allow(ctx))
 		return send_drop_notify_error(ctx, 0, DROP_UNENCRYPTED_TRAFFIC,
 					      CTX_ACT_DROP, METRIC_EGRESS);
-#endif /* ENCRYPTION_STRICT_MODE */
+#endif /* ENCRYPTION_STRICT_MODE && (!TUNNEL_MODE || ENABLE_NODE_ENCRYPTION) */
 #endif /* ENABLE_WIREGUARD */
 
 #ifdef ENABLE_HEALTH_CHECK

bpf/lib/maps.h

@@ -183,6 +183,22 @@ struct {
 	__uint(max_entries, 1);
 } ENCRYPT_MAP __section_maps_btf;
 
+struct strict_mode_policy {
+	__u8	allow;
+	__u8	pad1;
+	__be16	port1;
+	__be16	port2;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_LPM_TRIE);
+	__type(key, struct ipcache_key);
+	__type(value, struct strict_mode_policy);
+	__uint(pinning, LIBBPF_PIN_BY_NAME);
+	__uint(max_entries, STRICT_MAP_SIZE);
+	__uint(map_flags, BPF_F_NO_PREALLOC);
+} STRICT_MODE_MAP __section_maps_btf;
+
 struct node_key {
 	__u16 pad1;
 	__u8 pad2;

bpf/lib/wireguard.h

@@ -12,6 +12,8 @@
 #include "tailcall.h"
 #include "common.h"
 #include "overloadable.h"
+#include "maps.h"
+#include "eps.h"
 
 static __always_inline int
 wg_maybe_redirect_to_encrypt(struct __ctx_buff *ctx)
@@ -147,20 +149,43 @@ encrypt: __maybe_unused
 
 #ifdef ENCRYPTION_STRICT_MODE
 
+static __always_inline __maybe_unused struct strict_mode_policy *
+strict_lookup4(const void *map, __be32 addr, __u32 prefix, __u8 cluster_id)
+{
+	struct ipcache_key key = {
+		.lpm_key = { IPCACHE_PREFIX_LEN(prefix), {} },
+		.cluster_id = cluster_id,
+		.family = ENDPOINT_KEY_IPV4,
+		.ip4 = addr,
+	};
+
+	key.ip4 &= GET_PREFIX(prefix);
+	return map_lookup_elem(map, &key);
+}
+
 /* strict_allow checks whether the packet is allowed to pass through the strict mode. */
 static __always_inline bool
 strict_allow(struct __ctx_buff *ctx) {
 	struct remote_endpoint_info __maybe_unused *dest_info, __maybe_unused *src_info;
-	bool __maybe_unused in_strict_cidr = false;
+	bool __maybe_unused src_in_cidr = false;
+	bool __maybe_unused dst_in_cidr = false;
+	struct strict_mode_policy __maybe_unused *entry = NULL;
 	void *data, *data_end;
 #ifdef ENABLE_IPV4
 	struct iphdr *ip4;
+	struct tcphdr *tcph = NULL;
+	__u16 offset;
 #endif
 	__u16 proto = 0;
 
 	if (!validate_ethertype(ctx, &proto))
 		return true;
 
+#ifdef ENABLE_NODE_ENCRYPTION
+	if ((ctx->mark & MARK_MAGIC_WG_ENCRYPTED) == MARK_MAGIC_WG_ENCRYPTED)
+		return true;
+#endif /* ENABLE_NODE_ENCRYPTION */
+
 	switch (proto) {
 #ifdef ENABLE_IPV4
 	case bpf_htons(ETH_P_IP):
@@ -171,24 +196,39 @@ strict_allow(struct __ctx_buff *ctx) {
 		 * (1) When encapsulation is used and the destination is a remote pod.
 		 * (2) When the destination is a remote-node.
 		 */
+#ifndef ENABLE_NODE_ENCRYPTION
 		if (ip4->saddr == IPV4_GATEWAY || ip4->saddr == IPV4_ENCRYPT_IFACE)
 			return true;
+#endif /* ENABLE_NODE_ENCRYPTION */
+
+		if (ip4->protocol == IPPROTO_TCP) {
+			offset = sizeof(struct ethhdr) + sizeof(struct iphdr);
+			if ((data + offset + sizeof(struct tcphdr)) > data_end)
+				return true;
+			tcph = (struct tcphdr *)(data + offset);
+		}
+
+		entry = strict_lookup4(&STRICT_MODE_MAP, ip4->daddr, V4_CACHE_KEY_LEN, 0);
+		if (entry && entry->allow == 0)
+			dst_in_cidr = true;
+		if (entry && tcph && (tcph->dest == entry->port1 || tcph->dest == entry->port2))
+			return true;
+
+		entry = strict_lookup4(&STRICT_MODE_MAP, ip4->saddr, V4_CACHE_KEY_LEN, 0);
+		if (entry && entry->allow == 0)
+			src_in_cidr = true;
+		if (entry && tcph && (tcph->source == entry->port1 || tcph->source == entry->port2))
+			return true;
 
-		in_strict_cidr = ipv4_is_in_subnet(ip4->daddr,
-						   STRICT_IPV4_NET,
-						   STRICT_IPV4_NET_SIZE);
-		in_strict_cidr &= ipv4_is_in_subnet(ip4->saddr,
-						    STRICT_IPV4_NET,
-						    STRICT_IPV4_NET_SIZE);
 
-#if defined(TUNNEL_MODE) || defined(STRICT_IPV4_OVERLAPPING_CIDR)
-		/* Allow pod to remote-node communication */
+#ifdef ALLOW_REMOTE_NODE_IDENTITIES
+		/* Allow X to remote-node communication */
 		dest_info = lookup_ip4_remote_endpoint(ip4->daddr, 0);
 		if (dest_info && dest_info->sec_identity &&
 		    identity_is_node(dest_info->sec_identity))
 			return true;
-#endif /* TUNNEL_MODE || STRICT_IPV4_OVERLAPPING_CIDR */
-		return !in_strict_cidr;
+#endif /* ALLOW_REMOTE_NODE_IDENTITIES */
+		return !(src_in_cidr && dst_in_cidr);
 #endif /* ENABLE_IPV4 */
 	default:
 		return true;

bpf/node_config.h

@@ -148,6 +148,7 @@ DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x
 #define AUTH_MAP test_cilium_auth
 #define CONFIG_MAP test_cilium_runtime_config
 #define IPCACHE_MAP test_cilium_ipcache
+#define STRICT_MODE_MAP test_cilium_strict_mode_map
 #define NODE_MAP test_cilium_node_map
 #define ENCRYPT_MAP test_cilium_encrypt_state
 #define L2_RESPONDER_MAP4 test_cilium_l2_responder_v4
@@ -187,6 +188,7 @@ DEFINE_IPV6(HOST_IP, 0xbe, 0xef, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0xa, 0x
 #define POLICY_MAP_SIZE 16384
 #define AUTH_MAP_SIZE 512000
 #define CONFIG_MAP_SIZE 256
+#define STRICT_MAP_SIZE 5
 #define IPCACHE_MAP_SIZE 512000
 #define NODE_MAP_SIZE 16384
 #define EGRESS_POLICY_MAP_SIZE 16384

daemon/cmd/daemon.go

@@ -404,6 +404,10 @@ func newDaemon(ctx context.Context, cleaner *daemonCleanup, params *daemonParams
 	}
 	lbmap.Init(lbmapInitParams)
 
+	if err := setupStrictModeMap(params.LocalNodeStore); err != nil {
+		return nil, nil, fmt.Errorf("unable to setup strict map: %s", err)
+	}
+
 	params.NodeManager.Subscribe(params.Datapath.Node())
 
 	identity.IterateReservedIdentities(func(_ identity.NumericIdentity, _ *identity.Identity) {
@@ -1000,7 +1004,7 @@ func newDaemon(ctx context.Context, cleaner *daemonCleanup, params *daemonParams
 	// controller is to ensure that endpoints and host IPs entries are
 	// reinserted to the bpf maps if they are ever removed from them.
 	syncErrs := make(chan error, 1)
-	var syncHostIPsControllerGroup = controller.NewGroup("sync-host-ips")
+	syncHostIPsControllerGroup := controller.NewGroup("sync-host-ips")
 	d.controllers.UpdateController(
 		syncHostIPsController,
 		controller.ControllerParams{

daemon/cmd/daemon_main.go

@@ -402,8 +402,11 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.Bool(option.EnableEncryptionStrictMode, false, "Enable encryption strict mode")
 	option.BindEnv(vp, option.EnableEncryptionStrictMode)
 
-	flags.String(option.EncryptionStrictModeCIDR, "", "In strict-mode encryption, all unencrypted traffic coming from this CIDR and going to this same CIDR will be dropped")
-	option.BindEnv(vp, option.EncryptionStrictModeCIDR)
+	flags.StringSlice(option.EncryptionStrictModeNodeCIDRs, []string{}, "In strict-mode encryption, all unencrypted traffic coming from one of those CIDRs and going one of those CIDRs will be dropped")
+	option.BindEnv(vp, option.EncryptionStrictModeNodeCIDRs)
+
+	flags.StringSlice(option.EncryptionStrictModePodCIDRs, []string{}, "In strict-mode encryption, all unencrypted traffic coming from one of those CIDRs and going one of those CIDRs will be dropped")
+	option.BindEnv(vp, option.EncryptionStrictModePodCIDRs)
 
 	flags.Bool(option.EncryptionStrictModeAllowRemoteNodeIdentities, false, "Allows unencrypted traffic from pods to remote node identities within the strict mode CIDR. This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap.")
 	option.BindEnv(vp, option.EncryptionStrictModeAllowRemoteNodeIdentities)

daemon/cmd/datapath.go

@@ -4,6 +4,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/netip"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
+	k8sLabels "k8s.io/apimachinery/pkg/labels"
 
 	"github.com/cilium/cilium/pkg/cidr"
 	"github.com/cilium/cilium/pkg/datapath/linux/linux_defaults"
@@ -35,6 +37,7 @@ import (
 	"github.com/cilium/cilium/pkg/maps/neighborsmap"
 	"github.com/cilium/cilium/pkg/maps/policymap"
 	"github.com/cilium/cilium/pkg/maps/srv6map"
+	"github.com/cilium/cilium/pkg/maps/strictmap"
 	"github.com/cilium/cilium/pkg/maps/tunnel"
 	"github.com/cilium/cilium/pkg/maps/vtep"
 	"github.com/cilium/cilium/pkg/maps/worldcidrsmap"
@@ -80,7 +83,6 @@ func clearCiliumVeths() error {
 		}
 		return -1
 	})
-
 	if err != nil {
 		return fmt.Errorf("unable to retrieve host network interfaces: %s", err)
 	}
@@ -469,6 +471,59 @@ func (d *Daemon) initMaps() error {
 	return nil
 }
 
+func setupStrictModeMap(lns *node.LocalNodeStore) error {
+	if err := strictmap.Create(); err != nil {
+		return fmt.Errorf("initializing strict mode map: %w", err)
+	}
+
+	strictCIDRs := append(option.Config.EncryptionStrictModeNodeCIDRs, option.Config.EncryptionStrictModePodCIDRs...)
+	for _, cidr := range strictCIDRs {
+
+		ipv4Interface, ok := netip.AddrFromSlice(node.GetIPv4().To4())
+		if !ok {
+			return fmt.Errorf("unable to parse node IPv4 address %s", node.GetIPv4())
+		}
+		if cidr.Contains(ipv4Interface) && !option.Config.NodeEncryptionEnabled() {
+			if !option.Config.EncryptionStrictModeAllowRemoteNodeIdentities {
+				return fmt.Errorf(`encryption strict mode is enabled but the node's IPv4 address is within the strict CIDR range.
+				This will cause the node to drop all traffic.
+				Please either disable encryption or set --encryption-strict-mode-allow-dynamic-lookup=true`)
+			}
+		}
+
+		if err := strictmap.UpdateContext(cidr, 0, 0, 0, 0); err != nil {
+			return fmt.Errorf("updating strict mode map: %w", err)
+		}
+	}
+
+	// Add the default match to the trie map.
+	// If this prefix is matched, then the packet is allowed to pass unencrypted as indicated by the "1" as a value.
+	if err := strictmap.UpdateContext(netip.MustParsePrefix("0.0.0.0/0"), 0, 1, 0, 0); err != nil {
+		return fmt.Errorf("updating strict mode map: %w", err)
+	}
+
+	// Allow etcd ports only on control plane nodes
+	sel, err := k8sLabels.Parse("node-role.kubernetes.io/control-plane")
+	if err != nil {
+		return fmt.Errorf("unable to parse control plane label selector: %w", err)
+	}
+
+	localNode, err := lns.Get(context.Background())
+	if err != nil {
+		return fmt.Errorf("unable to get local node: %w", err)
+	}
+
+	if sel.Matches(k8sLabels.Set(localNode.Labels)) {
+		for _, nodeCIDR := range option.Config.EncryptionStrictModeNodeCIDRs {
+			if err := strictmap.UpdateContext(nodeCIDR, 0, 0, 2379, 2380); err != nil {
+				return fmt.Errorf("updating strict mode map: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
 func setupVTEPMapping() error {
 	for i, ep := range option.Config.VtepEndpoints {
 		log.WithFields(logrus.Fields{
@@ -482,7 +537,6 @@ func setupVTEPMapping() error {
 
 	}
 	return nil
-
 }
 
 func setupRouteToVtepCidr() error {

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -606,7 +606,9 @@ data:
 {{- if .Values.encryption.strictMode.enabled }}
   enable-encryption-strict-mode: {{ .Values.encryption.strictMode.enabled | quote }}
 
-  encryption-strict-mode-cidr: {{ .Values.encryption.strictMode.cidr | quote }}
+  encryption-strict-mode-node-cidrs: {{ .Values.encryption.strictMode.nodeCIDRList | join " " | quote }}
+
+  encryption-strict-mode-pod-cidrs: {{ .Values.encryption.strictMode.podCIDRList | join " " | quote }}
 
   encryption-strict-mode-allow-remote-node-identities: {{ .Values.encryption.strictMode.allowRemoteNodeIdentities | quote }}
 {{- end }}