[Experimental] Use privileged dataplane entrypoint for ingress-gateway

charts/consul/templates/_helpers.tpl

@@ -23,8 +23,33 @@ securityContext:
   capabilities:
     drop:
     - ALL
+  runAsNonRoot: true
+  seccompProfile:
+    type: RuntimeDefault
+{{- if not .Values.global.openshift.enabled -}}
+{{/*
+We must set runAsUser or else the root user will be used in some cases and
+containers will fail to start due to runAsNonRoot above (e.g.
+tls-init-cleanup). On OpenShift, runAsUser is automatically. We pick user 100
+because it is a non-root user id that exists in the consul, consul-dataplane,
+and consul-k8s-control-plane images.
+*/}}
+  runAsUser: 100
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+
+{{- define "consul.restrictedSecurityContextWithNetBindService" -}}
+{{- if not .Values.global.enablePodSecurityPolicies -}}
+securityContext:
+  allowPrivilegeEscalation: false
+  readOnlyRootFilesystem: true
+  capabilities:
     add:
     - NET_BIND_SERVICE
+    drop:
+    - ALL
   runAsNonRoot: true
   seccompProfile:
     type: RuntimeDefault

charts/consul/templates/ingress-gateways-deployment.yaml

@@ -247,7 +247,7 @@ spec:
       - name: ingress-gateway
         image: {{ $root.Values.global.imageConsulDataplane | quote }}
         {{ template "consul.imagePullPolicy" $root }}
-        {{- include "consul.restrictedSecurityContext" $ | nindent 8 }}
+        {{- include "consul.restrictedSecurityContextWithNetBindService" $ | nindent 8 }}
         {{- if (default $defaults.resources .resources) }}
         resources: {{ toYaml (default $defaults.resources .resources) | nindent 10 }}
         {{- end }}
@@ -291,9 +291,14 @@ spec:
           value: component=ingress-gateway
         - name: DP_SERVICE_NODE_NAME
           value: $(NODE_NAME)-virtual
+        {{- if $root.Values.ingressGateways.supportPrivilegedPorts }}
         command:
-        - consul-dataplane
+        - privileged-consul-dataplane
+        args:
+        - -envoy-executable-path=/usr/local/bin/privileged-envoy
+        {{- else }}
         args:
+        {{- end }}
         - -envoy-ready-bind-port=21000
         {{- if $root.Values.externalServers.enabled }}
         - -addresses={{ $root.Values.externalServers.hosts | first }}

charts/consul/templates/mesh-gateway-deployment.yaml

@@ -187,14 +187,12 @@ spec:
       - name: mesh-gateway
         image: {{ .Values.global.imageConsulDataplane | quote }}
         {{ template "consul.imagePullPolicy" . }}
+        {{ if not .Values.meshGateway.hostNetwork}}
         securityContext:
           capabilities:
-            {{ if not .Values.meshGateway.hostNetwork}}
             drop:
               - ALL
-            {{- end }}
-            add:
-              - NET_BIND_SERVICE
+        {{- end }}
         {{- if .Values.meshGateway.resources }}
         resources:
             {{- if eq (typeOf .Values.meshGateway.resources) "string" }}

charts/consul/templates/mesh-gateway-podsecuritypolicy.yaml

@@ -18,8 +18,6 @@ spec:
   # but we can provide it for defense in depth.
   requiredDropCapabilities:
     - ALL
-  defaultAddCapabilities:
-    - NET_BIND_SERVICE
   # Allow core volume types.
   volumes:
     - 'configMap'

charts/consul/templates/telemetry-collector-podsecuritypolicy.yaml

@@ -18,8 +18,6 @@ spec:
   # but we can provide it for defense in depth.
   requiredDropCapabilities:
     - ALL
-  defaultAddCapabilities:
-    - NET_BIND_SERVICE
   # Allow core volume types.
   volumes:
     - 'configMap'

charts/consul/templates/terminating-gateways-podsecuritypolicy.yaml

@@ -21,8 +21,6 @@ spec:
   # but we can provide it for defense in depth.
   requiredDropCapabilities:
     - ALL
-  defaultAddCapabilities:
-    - NET_BIND_SERVICE
   # Allow core volume types.
   volumes:
     - 'configMap'

charts/consul/test/unit/server-statefulset.bats

@@ -1383,7 +1383,6 @@ load _helpers
     "allowPrivilegeEscalation": false,
     "capabilities": {
       "drop": ["ALL"],
-      "add": ["NET_BIND_SERVICE"]
     },
     "readOnlyRootFilesystem": true,
     "runAsNonRoot": true,
@@ -1416,7 +1415,6 @@ load _helpers
     "allowPrivilegeEscalation": false,
     "capabilities": {
       "drop": ["ALL"],
-      "add": ["NET_BIND_SERVICE"]
     },
     "readOnlyRootFilesystem": true,
     "runAsNonRoot": true,

charts/consul/values.yaml

@@ -3217,6 +3217,11 @@ ingressGateways:
   # @type: string
   logLevel: ""
 
+  # When set, ingress-gateway containers use a privileged entrypoint in consul-dataplane which requires the
+  # NET_BIND_SERVICE capability at runtime. Disabling this setting will use the non-privileged entrypoint and
+  # drop the requirement for NET_BIND_SERVICE. Only disable this if you are *not* using privileged ports (< 1024).
+  supportPrivilegedPorts: true
+
   # Defaults sets default values for all gateway fields. With the exception
   # of annotations, defining any of these values in the `gateways` list
   # will override the default values provided here. Annotations will

control-plane/api-gateway/gatekeeper/dataplane.go

@@ -19,7 +19,6 @@ import (
 
 const (
 	allCapabilities              = "ALL"
-	netBindCapability            = "NET_BIND_SERVICE"
 	consulDataplaneDNSBindHost   = "127.0.0.1"
 	consulDataplaneDNSBindPort   = 8600
 	defaultEnvoyProxyConcurrency = 1
@@ -114,9 +113,7 @@ func consulDataplaneContainer(metrics common.MetricsConfig, config common.HelmCo
 	// otherwise, allow the user to be assigned by OpenShift.
 	container.SecurityContext = &corev1.SecurityContext{
 		ReadOnlyRootFilesystem: ptr.To(true),
-		// Drop any Linux capabilities you'd get as root other than NET_BIND_SERVICE.
 		Capabilities: &corev1.Capabilities{
-			Add:  []corev1.Capability{netBindCapability},
 			Drop: []corev1.Capability{allCapabilities},
 		},
 	}

control-plane/api-gateway/gatekeeper/gatekeeper_test.go

@@ -1506,8 +1506,7 @@ func validateResourcesExist(t *testing.T, client client.Client, helmConfig commo
 		}
 		assert.True(t, hasInitContainer)
 
-		// Ensure there is a consul-dataplane container dropping ALL capabilities, adding
-		// back the NET_BIND_SERVICE capability, and establishing a read-only root filesystem
+		// Ensure there is a consul-dataplane container dropping ALL capabilities and establishing a read-only root filesystem
 		hasDataplaneContainer := false
 		for _, container := range actual.Spec.Template.Spec.Containers {
 			if container.Image == dataplaneImage {
@@ -1516,7 +1515,6 @@ func validateResourcesExist(t *testing.T, client client.Client, helmConfig commo
 				require.NotNil(t, container.SecurityContext.Capabilities)
 				require.NotNil(t, container.SecurityContext.ReadOnlyRootFilesystem)
 				assert.True(t, *container.SecurityContext.ReadOnlyRootFilesystem)
-				assert.Equal(t, []corev1.Capability{netBindCapability}, container.SecurityContext.Capabilities.Add)
 				assert.Equal(t, []corev1.Capability{allCapabilities}, container.SecurityContext.Capabilities.Drop)
 			}
 		}

control-plane/connect-inject/webhook/consul_dataplane_sidecar.go

@@ -10,12 +10,13 @@ import (
 	"strings"
 
 	"github.com/google/shlex"
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/common"
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
+
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/common"
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
 )
 
 const (
@@ -264,12 +265,7 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		RunAsGroup:               ptr.To(group),
 		RunAsNonRoot:             ptr.To(true),
 		AllowPrivilegeEscalation: ptr.To(false),
-		// consul-dataplane requires the NET_BIND_SERVICE capability regardless of binding port #.
-		// See https://developer.hashicorp.com/consul/docs/connect/dataplane#technical-constraints
-		Capabilities: &corev1.Capabilities{
-			Add: []corev1.Capability{"NET_BIND_SERVICE"},
-		},
-		ReadOnlyRootFilesystem: ptr.To(true),
+		ReadOnlyRootFilesystem:   ptr.To(true),
 	}
 	return container, nil
 }

control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go

@@ -807,9 +807,6 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				RunAsNonRoot:             ptr.To(true),
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
-				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
-				},
 			},
 		},
 		"tproxy enabled; openshift disabled": {
@@ -821,9 +818,6 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				RunAsNonRoot:             ptr.To(true),
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
-				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
-				},
 			},
 		},
 		"tproxy disabled; openshift enabled": {
@@ -835,9 +829,6 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				RunAsNonRoot:             ptr.To(true),
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
-				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
-				},
 			},
 		},
 		"tproxy enabled; openshift enabled": {
@@ -849,9 +840,6 @@ func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) {
 				RunAsNonRoot:             ptr.To(true),
 				ReadOnlyRootFilesystem:   ptr.To(true),
 				AllowPrivilegeEscalation: ptr.To(false),
-				Capabilities: &corev1.Capabilities{
-					Add: []corev1.Capability{"NET_BIND_SERVICE"},
-				},
 			},
 		},
 	}