service-mesh: open envoy admin endpoint

docs/docs/architecture/observability.md

@@ -35,3 +35,21 @@ The current manifest generation is exposed as a
 [gauge](https://prometheus.io/docs/concepts/metric_types/#gauge) with the metric
 name `coordinator_manifest_generation`. If no manifest is set at the
 Coordinator, this counter will be zero.
+
+## Service Mesh metrics
+
+The [Service Mesh](../components/service-mesh.md) can be configured to expose
+metrics via its [Envoy admin
+interface](https://www.envoyproxy.io/docs/envoy/latest/operations/admin). Be
+aware that the admin interface can expose private information and allows
+destructive operations to be performed. To enable the admin interface for the
+Service Mesh, set the environment variable `EDG_ADMIN_PORT` in the configuration
+of the Service Mesh sidecar. If this variable is set, the admin interface will
+be started on this port.
+
+To access the admin interface, the Service Mesh sidecar container needs to have
+a corresponding container port and the ingress settings of the Proxy have to be
+configured to allow access to the specified port (see [Configuring the
+Proxy](../components/service-mesh#configuring-the-proxy)). All metrics will be
+exposed under the `/stats` endpoint. Metrics in Prometheus format can be scraped
+from the `/stats/prometheus` endpoint.

e2e/internal/contrasttest/contrasttest.go

@@ -95,6 +95,7 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) {
 	// Prepare resources
 	resources = kuberesource.PatchImages(resources, ct.ImageReplacements)
 	resources = kuberesource.PatchNamespaces(resources, ct.Namespace)
+	resources = kuberesource.PatchServiceMeshAdminInterface(resources, 9901)
 	resources = kuberesource.AddLogging(resources, "debug")
 	unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources)
 	require.NoError(err)

e2e/servicemesh/servicemesh_test.go

@@ -12,6 +12,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	"os"
 	"testing"
@@ -114,6 +115,27 @@ func TestIngressEgress(t *testing.T) {
 		stdout, stderr, err = c.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv)
 		require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr)
 	})
+
+	t.Run("admin interface is available", func(t *testing.T) {
+		require := require.New(t)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+		defer cancel()
+
+		c := kubeclient.NewForTest(t)
+
+		backendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, "emoji")
+		require.NoError(err)
+		require.Len(backendPods, 1, "pod not found: %s/%s", ct.Namespace, "emoji")
+
+		frontendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, "web")
+		require.NoError(err)
+		require.Len(frontendPods, 1, "pod not found: %s/%s", ct.Namespace, "web")
+
+		argv := []string{"curl", "-fsS", net.JoinHostPort(frontendPods[0].Status.PodIP, "9901") + "/stats/prometheus"}
+		stdout, stderr, err := c.Exec(ctx, ct.Namespace, backendPods[0].Name, argv)
+		require.NoError(err, "Expected Service Mesh admin interface to be reachable.\nstdout: %s\nstderr: %q", stdout, stderr)
+	})
 }
 
 func TestMain(m *testing.M) {

internal/kuberesource/sets.go

@@ -5,6 +5,7 @@ package kuberesource
 
 import (
 	"fmt"
+	"strings"
 
 	"k8s.io/apimachinery/pkg/util/intstr"
 	applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
@@ -600,3 +601,40 @@ func PatchNamespaces(resources []any, namespace string) []any {
 	}
 	return resources
 }
+
+// PatchServiceMeshAdminInterface activates the admin interface on the
+// specified port for all Service Mesh components in a set of resources.
+func PatchServiceMeshAdminInterface(resources []any, port int32) []any {
+	for _, resource := range resources {
+		switch r := resource.(type) {
+		case *applyappsv1.DeploymentApplyConfiguration:
+			for i := 0; i < len(r.Spec.Template.Spec.InitContainers); i++ {
+				// TODO(davidweisse): find service mesh containers by unique name as specified in RFC 005.
+				if strings.Contains(*r.Spec.Template.Spec.InitContainers[i].Image, "service-mesh-proxy") {
+					r.Spec.Template.Spec.InitContainers[i] = *r.Spec.Template.Spec.InitContainers[i].
+						WithEnv(NewEnvVar("EDG_ADMIN_PORT", fmt.Sprint(port))).
+						WithPorts(
+							ContainerPort().
+								WithName("admin-interface").
+								WithContainerPort(port),
+						)
+					ingressProxyConfig := false
+					for j, env := range r.Spec.Template.Spec.InitContainers[i].Env {
+						if *env.Name == "EDG_INGRESS_PROXY_CONFIG" {
+							ingressProxyConfig = true
+							env.WithValue(fmt.Sprintf("%s##admin#%d#true", *env.Value, port))
+							r.Spec.Template.Spec.InitContainers[i].Env[j] = env
+							break
+						}
+					}
+					if !ingressProxyConfig {
+						r.Spec.Template.Spec.InitContainers[i].WithEnv(
+							NewEnvVar("EDG_INGRESS_PROXY_CONFIG", fmt.Sprintf("admin#%d#true", port)),
+						)
+					}
+				}
+			}
+		}
+	}
+	return resources
+}

packages/by-name/service-mesh/package.nix

@@ -26,7 +26,7 @@ buildGoModule rec {
     };
 
   proxyVendor = true;
-  vendorHash = "sha256-hKoHRgcji48ZnWWoeOuYkvSX2jhUqDTfsIl1ENSlA7E=";
+  vendorHash = "sha256-UbUoj6SzUaTASfMawVQnMNqE//csvsM3hSbXTntP3gQ=";
 
   subPackages = [ "." ];
 

service-mesh/config.go

@@ -27,8 +27,9 @@ var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8")
 
 // ProxyConfig represents the configuration for the proxy.
 type ProxyConfig struct {
-	egress  []egressConfigEntry
-	ingress []ingressConfigEntry
+	egress    []egressConfigEntry
+	ingress   []ingressConfigEntry
+	adminPort uint32
 }
 type egressConfigEntry struct {
 	name         string
@@ -52,8 +53,8 @@ type ingressConfigEntry struct {
 // Example:
 //
 //	emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080
-func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) {
-	if ingressConfig == "" && egressConfig == "" {
+func ParseProxyConfig(ingressConfig, egressConfig, adminPort string) (ProxyConfig, error) {
+	if ingressConfig == "" && egressConfig == "" && adminPort == "" {
 		return ProxyConfig{}, nil
 	}
 
@@ -118,6 +119,14 @@ func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) {
 
 	}
 
+	if adminPort != "" {
+		adminPortInt, err := strconv.Atoi(adminPort)
+		if err != nil {
+			return ProxyConfig{}, fmt.Errorf("invalid admin port: %s", adminPort)
+		}
+		cfg.adminPort = uint32(adminPortInt)
+	}
+
 	return cfg, nil
 }
 
@@ -127,6 +136,21 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) {
 	config := &envoyConfigBootstrapV3.Bootstrap{
 		StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{},
 	}
+	if c.adminPort != 0 {
+		config.Admin = &envoyConfigBootstrapV3.Admin{
+			Address: &envoyCoreV3.Address{
+				Address: &envoyCoreV3.Address_SocketAddress{
+					SocketAddress: &envoyCoreV3.SocketAddress{
+						Address: "0.0.0.0",
+						PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{
+							PortValue: c.adminPort,
+						},
+					},
+				},
+			},
+		}
+	}
+
 	listeners := make([]*envoyConfigListenerV3.Listener, 0)
 	clusters := make([]*envoyConfigClusterV3.Cluster, 0)
 

service-mesh/go.mod

@@ -1,11 +1,11 @@
 module github.com/edgelesssys/contrast/service-mesh
 
-go 1.21
+go 1.22.0
 
 require (
 	github.com/coreos/go-iptables v0.7.0
 	github.com/envoyproxy/go-control-plane v0.12.0
-	google.golang.org/protobuf v1.34.1
+	google.golang.org/protobuf v1.34.0
 )
 
 require (

service-mesh/go.sum

@@ -60,7 +60,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4=
+google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

service-mesh/main.go

@@ -14,6 +14,7 @@ import (
 const (
 	egressProxyConfigEnvVar  = "EDG_EGRESS_PROXY_CONFIG"
 	ingressProxyConfigEnvVar = "EDG_INGRESS_PROXY_CONFIG"
+	adminPortEnvVar          = "EDG_ADMIN_PORT"
 	envoyConfigFile          = "/envoy-config.yml"
 )
 
@@ -35,7 +36,10 @@ func run() (retErr error) {
 	ingressProxyConfig := os.Getenv(ingressProxyConfigEnvVar)
 	log.Println("Egress Proxy configuration:", ingressProxyConfig)
 
-	pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig)
+	adminPort := os.Getenv(adminPortEnvVar)
+	log.Println("Port for Envoy admin interface:", adminPort)
+
+	pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig, adminPort)
 	if err != nil {
 		return err
 	}