From 03b2cc4084556f1f29aa5c20aa5e9fa46df4127b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Wei=C3=9Fe?= Date: Tue, 14 May 2024 15:17:33 +0200 Subject: [PATCH] service-mesh: open envoy admin endpoint --- docs/docs/architecture/observability.md | 18 +++++++++++++ e2e/internal/contrasttest/contrasttest.go | 1 + e2e/servicemesh/servicemesh_test.go | 21 +++++++++++++++ internal/kuberesource/sets.go | 25 +++++++++++++++++- packages/by-name/service-mesh/package.nix | 2 +- service-mesh/config.go | 32 ++++++++++++++++++++--- service-mesh/go.mod | 4 +-- service-mesh/go.sum | 4 +-- service-mesh/main.go | 6 ++++- 9 files changed, 102 insertions(+), 11 deletions(-) diff --git a/docs/docs/architecture/observability.md b/docs/docs/architecture/observability.md index 21296afbb3..52344f5b2b 100644 --- a/docs/docs/architecture/observability.md +++ b/docs/docs/architecture/observability.md @@ -30,3 +30,21 @@ For the mesh API, the metric names are prefixed with `meshapi_grpc_server_`. The metrics include similar data to the user API for the method `NewMeshCert` which gets called by the [Initializer](../components#the-initializer) when starting a new workload. + +## Service Mesh metrics + +The [Service Mesh](../components/service-mesh.md) can be configured to expose +metrics via its [Envoy admin +interface](https://www.envoyproxy.io/docs/envoy/latest/operations/admin). Be +aware that the admin interface can expose private information and allows +destructive operations to be performed. To enable the admin interface for the +Service Mesh, set the environment variable `EDG_ADMIN_PORT` in the configuration +of the Service Mesh sidecar. If this variable is set, the admin interface will +be started on this port. + +To access the admin interface, the Service Mesh sidecar container needs to have +a corresponding container port and the ingress settings of the Proxy have to be +configured to allow access to the specified port (see [Configuring the +Proxy](../components/service-mesh#configuring-the-proxy)). All metrics will be +exposed under the `/stats` endpoint. Metrics in Prometheus format can be scraped +from the `/stats/prometheus` endpoint. diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index 14deba0e71..629e8eda73 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -95,6 +95,7 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) { // Prepare resources resources = kuberesource.PatchImages(resources, ct.ImageReplacements) resources = kuberesource.PatchNamespaces(resources, ct.Namespace) + resources = kuberesource.PatchServiceMeshAdminInterface(resources, 9901) resources = kuberesource.AddLogging(resources, "debug") unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources) require.NoError(err) diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go index 175d2e773e..e41b158045 100644 --- a/e2e/servicemesh/servicemesh_test.go +++ b/e2e/servicemesh/servicemesh_test.go @@ -114,6 +114,27 @@ func TestIngressEgress(t *testing.T) { stdout, stderr, err = c.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv) require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr) }) + + t.Run("admin interface is available", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + c := kubeclient.NewForTest(t) + + backendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, "emoji") + require.NoError(err) + require.Len(backendPods, 1, "pod not found: %s/%s", ct.Namespace, "emoji") + + frontendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, "web") + require.NoError(err) + require.Len(frontendPods, 1, "pod not found: %s/%s", ct.Namespace, "web") + + argv := []string{"curl", "-sS", frontendPods[0].Status.PodIP + ":9901"} + stdout, stderr, err := c.Exec(ctx, ct.Namespace, backendPods[0].Name, argv) + require.NoError(err, "Expected Service Mesh admin interface to be reachable.\nstdout: %s\nstderr: %q", stdout, stderr) + }) } func TestMain(m *testing.M) { diff --git a/internal/kuberesource/sets.go b/internal/kuberesource/sets.go index f328a6d78a..baacb81642 100644 --- a/internal/kuberesource/sets.go +++ b/internal/kuberesource/sets.go @@ -5,6 +5,7 @@ package kuberesource import ( "fmt" + "strings" "k8s.io/apimachinery/pkg/util/intstr" applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1" @@ -220,7 +221,7 @@ func Emojivoto(smMode serviceMeshMode) ([]any, error) { smProxyWeb = smProxyWeb. WithEnv(EnvVar(). WithName("EDG_INGRESS_PROXY_CONFIG"). - WithValue("web#8080#false"), + WithValue("web#8080#false##admin#9901#true"), ). WithEnv(EnvVar(). WithName("EDG_EGRESS_PROXY_CONFIG"). @@ -600,3 +601,25 @@ func PatchNamespaces(resources []any, namespace string) []any { } return resources } + +// PatchServiceMeshAdminInterface activates the admin interface on the +// specified port for all Service Mesh components in a set of resources. +func PatchServiceMeshAdminInterface(resources []any, port int32) []any { + for _, resource := range resources { + switch r := resource.(type) { + case *applyappsv1.DeploymentApplyConfiguration: + for i := 0; i < len(r.Spec.Template.Spec.InitContainers); i++ { + if strings.Contains(*r.Spec.Template.Spec.InitContainers[i].Image, "service-mesh-proxy") { + r.Spec.Template.Spec.InitContainers[i] = *r.Spec.Template.Spec.InitContainers[i]. + WithEnv(NewEnvVar("EDG_ADMIN_PORT", fmt.Sprint(port))). + WithPorts( + ContainerPort(). + WithName("admin-interface"). + WithContainerPort(port), + ) + } + } + } + } + return resources +} diff --git a/packages/by-name/service-mesh/package.nix b/packages/by-name/service-mesh/package.nix index 292780f9cf..49459dffcc 100644 --- a/packages/by-name/service-mesh/package.nix +++ b/packages/by-name/service-mesh/package.nix @@ -26,7 +26,7 @@ buildGoModule rec { }; proxyVendor = true; - vendorHash = "sha256-hKoHRgcji48ZnWWoeOuYkvSX2jhUqDTfsIl1ENSlA7E="; + vendorHash = "sha256-UbUoj6SzUaTASfMawVQnMNqE//csvsM3hSbXTntP3gQ="; subPackages = [ "." ]; diff --git a/service-mesh/config.go b/service-mesh/config.go index 9712514b96..55b1d0fede 100644 --- a/service-mesh/config.go +++ b/service-mesh/config.go @@ -27,8 +27,9 @@ var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") // ProxyConfig represents the configuration for the proxy. type ProxyConfig struct { - egress []egressConfigEntry - ingress []ingressConfigEntry + egress []egressConfigEntry + ingress []ingressConfigEntry + adminPort uint32 } type egressConfigEntry struct { name string @@ -52,8 +53,8 @@ type ingressConfigEntry struct { // Example: // // emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080 -func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) { - if ingressConfig == "" && egressConfig == "" { +func ParseProxyConfig(ingressConfig, egressConfig, adminPort string) (ProxyConfig, error) { + if ingressConfig == "" && egressConfig == "" && adminPort == "" { return ProxyConfig{}, nil } @@ -118,6 +119,14 @@ func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) { } + if adminPort != "" { + adminPortInt, err := strconv.Atoi(adminPort) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid admin port: %s", adminPort) + } + cfg.adminPort = uint32(adminPortInt) + } + return cfg, nil } @@ -127,6 +136,21 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { config := &envoyConfigBootstrapV3.Bootstrap{ StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{}, } + if c.adminPort != 0 { + config.Admin = &envoyConfigBootstrapV3.Admin{ + Address: &envoyCoreV3.Address{ + Address: &envoyCoreV3.Address_SocketAddress{ + SocketAddress: &envoyCoreV3.SocketAddress{ + Address: "0.0.0.0", + PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{ + PortValue: c.adminPort, + }, + }, + }, + }, + } + } + listeners := make([]*envoyConfigListenerV3.Listener, 0) clusters := make([]*envoyConfigClusterV3.Cluster, 0) diff --git a/service-mesh/go.mod b/service-mesh/go.mod index ce753096af..d94601cc3d 100644 --- a/service-mesh/go.mod +++ b/service-mesh/go.mod @@ -1,11 +1,11 @@ module github.com/edgelesssys/contrast/service-mesh -go 1.21 +go 1.22.0 require ( github.com/coreos/go-iptables v0.7.0 github.com/envoyproxy/go-control-plane v0.12.0 - google.golang.org/protobuf v1.34.1 + google.golang.org/protobuf v1.34.0 ) require ( diff --git a/service-mesh/go.sum b/service-mesh/go.sum index cf21a7e769..a5a6bd81a9 100644 --- a/service-mesh/go.sum +++ b/service-mesh/go.sum @@ -60,7 +60,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/service-mesh/main.go b/service-mesh/main.go index f1c282d782..d21065a840 100644 --- a/service-mesh/main.go +++ b/service-mesh/main.go @@ -14,6 +14,7 @@ import ( const ( egressProxyConfigEnvVar = "EDG_EGRESS_PROXY_CONFIG" ingressProxyConfigEnvVar = "EDG_INGRESS_PROXY_CONFIG" + adminPortEnvVar = "EDG_ADMIN_PORT" envoyConfigFile = "/envoy-config.yml" ) @@ -35,7 +36,10 @@ func run() (retErr error) { ingressProxyConfig := os.Getenv(ingressProxyConfigEnvVar) log.Println("Egress Proxy configuration:", ingressProxyConfig) - pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig) + adminPort := os.Getenv(adminPortEnvVar) + log.Println("Port for Envoy admin interface:", adminPort) + + pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig, adminPort) if err != nil { return err }