From 9734e17cd33dd4e36353a8a984e352874d24e940 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Tue, 13 Feb 2024 11:54:11 +0100 Subject: [PATCH 1/2] service-mesh: add egress implementation --- .gitignore | 1 + go.work | 6 + justfile | 5 +- packages/by-name/nunki/package.nix | 4 +- packages/by-name/service-mesh/package.nix | 48 +++++ packages/containers.nix | 10 + packages/scripts.nix | 5 +- service-mesh/config.go | 233 ++++++++++++++++++++++ service-mesh/go.mod | 17 ++ service-mesh/go.sum | 64 ++++++ service-mesh/main.go | 55 +++++ 11 files changed, 445 insertions(+), 3 deletions(-) create mode 100644 go.work create mode 100644 packages/by-name/service-mesh/package.nix create mode 100644 service-mesh/config.go create mode 100644 service-mesh/go.mod create mode 100644 service-mesh/go.sum create mode 100644 service-mesh/main.go diff --git a/.gitignore b/.gitignore index 219c4e13d8..6df321ff8b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ justfile.env workspace workspace.cache .direnv/ +go.work.sum diff --git a/go.work b/go.work new file mode 100644 index 0000000000..6dd3847131 --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.21 + +use ( + . + ./service-mesh +) diff --git a/justfile b/justfile index 6f6f961b51..e731cdcf25 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,5 @@ # Undeploy, rebuild, deploy. -default target=default_deploy_target cli=default_cli: undeploy coordinator initializer openssl port-forwarder (deploy target cli) set verify (wait-for-workload target) +default target=default_deploy_target cli=default_cli: undeploy coordinator initializer openssl port-forwarder service-mesh-proxy (deploy target cli) set verify (wait-for-workload target) # Build the coordinator, containerize and push it. coordinator: @@ -13,6 +13,9 @@ openssl: port-forwarder: nix run .#containers.push-port-forwarder -- "$container_registry/nunki/port-forwarder" +service-mesh-proxy: + nix run .#containers.push-service-mesh-proxy -- "$container_registry/nunki/service-mesh-proxy" + # Build the initializer, containerize and push it. initializer: nix run .#containers.push-initializer -- "$container_registry/nunki/initializer" diff --git a/packages/by-name/nunki/package.nix b/packages/by-name/nunki/package.nix index 8a88dcc93e..dca3580e6b 100644 --- a/packages/by-name/nunki/package.nix +++ b/packages/by-name/nunki/package.nix @@ -22,7 +22,9 @@ buildGoModule rec { fileset = fileset.unions [ (path.append root "go.mod") (path.append root "go.sum") - (fileset.fileFilter (file: hasSuffix ".go" file.name) root) + (lib.fileset.difference + (lib.fileset.fileFilter (file: lib.hasSuffix ".go" file.name) root) + (path.append root "service-mesh")) ]; }; diff --git a/packages/by-name/service-mesh/package.nix b/packages/by-name/service-mesh/package.nix new file mode 100644 index 0000000000..a4699f8606 --- /dev/null +++ b/packages/by-name/service-mesh/package.nix @@ -0,0 +1,48 @@ +{ lib +, buildGoModule +}: + +buildGoModule rec { + pname = "service-mesh"; + version = builtins.readFile ../../../version.txt; + + # The source of the main module of this repo. We filter for Go files so that + # changes in the other parts of this repo don't trigger a rebuild. + src = + let + inherit (lib) fileset path hasSuffix; + root = ../../../service-mesh; + in + fileset.toSource { + inherit root; + fileset = fileset.unions [ + (path.append root "go.mod") + (path.append root "go.sum") + (lib.fileset.fileFilter (file: lib.hasSuffix ".go" file.name) root) + ]; + }; + + proxyVendor = true; + vendorHash = "sha256-uNd8HOd1HXhTvksoVLFsoIof/3VNnBZDLeEraYs5i3s="; + + subPackages = [ "." ]; + + CGO_ENABLED = 0; + ldflags = [ + "-s" + "-w" + "-X main.version=v${version}" + ]; + + preCheck = '' + export CGO_ENABLED=1 + ''; + + checkPhase = '' + runHook preCheck + go test -race ./... + runHook postCheck + ''; + + meta.mainProgram = "service-mesh"; +} diff --git a/packages/containers.nix b/packages/containers.nix index 4123c76260..d0bae24962 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -59,6 +59,16 @@ let tag = "v${nunki.version}"; copyToRoot = [ bash socat ]; }; + + service-mesh-proxy = dockerTools.buildImage { + name = "service-mesh-proxy"; + tag = "v${service-mesh.version}"; + copyToRoot = [ envoy ]; + config = { + Cmd = [ "${service-mesh}/bin/service-mesh" ]; + Env = [ "PATH=/bin" ]; # This is only here for policy generation. + }; + }; }; in containers // (lib.concatMapAttrs (name: container: { "push-${name}" = pushContainer container; }) containers) diff --git a/packages/scripts.nix b/packages/scripts.nix index 2e7287e5aa..2ef2e7b4cb 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -63,17 +63,20 @@ with pkgs; gunzip < "${containers.initializer}" > "$tmpdir/initializer.tar" gunzip < "${containers.openssl}" > "$tmpdir/openssl.tar" gunzip < "${containers.port-forwarder}" > "$tmpdir/port-forwarder.tar" + gunzip < "${containers.service-mesh-proxy}" > "$tmpdir/service-mesh-proxy.tar" coordHash=$(crane digest --tarball "$tmpdir/coordinator.tar") initHash=$(crane digest --tarball "$tmpdir/initializer.tar") opensslHash=$(crane digest --tarball "$tmpdir/openssl.tar") forwarderHash=$(crane digest --tarball "$tmpdir/port-forwarder.tar") + serviceMeshProxyHash=$(crane digest --tarball "$tmpdir/service-mesh-proxy.tar") kypatch images "$targetPath" \ --replace "nunki/coordinator:latest" "nunki/coordinator@$coordHash" \ --replace "nunki/initializer:latest" "nunki/initializer@$initHash" \ --replace "nunki/openssl:latest" "nunki/openssl@$opensslHash" \ - --replace "nunki/port-forwarder:latest" "nunki/port-forwarder@$forwarderHash" + --replace "nunki/port-forwarder:latest" "nunki/port-forwarder@$forwarderHash" \ + --replace "nunki/service-mesh-proxy:latest" "nunki/service-mesh-proxy@$serviceMeshProxyHash" ''; }; diff --git a/service-mesh/config.go b/service-mesh/config.go new file mode 100644 index 0000000000..60d7f3d00a --- /dev/null +++ b/service-mesh/config.go @@ -0,0 +1,233 @@ +package main + +import ( + "fmt" + "net" + "net/netip" + "strconv" + "strings" + + envoyConfigBootstrapV3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" + envoyConfigClusterV3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoyCoreV3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + endpointV3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + envoyConfigListenerV3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoyConfigTCPProxyV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" + envoyTLSV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/anypb" +) + +var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") + +// ProxyConfig represents the configuration for the proxy. +type ProxyConfig []configEntry + +type configEntry struct { + name string + listenAddr netip.Addr + listenPort uint16 + remoteDomain string + remotePort uint16 +} + +// ParseProxyConfig parses the proxy configuration from the given string. +// The configuration is expected to be in the following format: +// +// #:#:###:#:... +// +// Example: +// +// emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080 +func ParseProxyConfig(data string) (ProxyConfig, error) { + entries := strings.Split(data, "##") + var cfg ProxyConfig + for _, entry := range entries { + parts := strings.Split(entry, "#") + if len(parts) != 3 { + return nil, fmt.Errorf("invalid entry: %s", entry) + } + listenAddrPort, err := netip.ParseAddrPort(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid listen address: %s", parts[1]) + } + + if !loopbackCIDR.Contains(listenAddrPort.Addr()) { + return nil, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) + } + remoteDomain := parts[2] + remoteDomain, remotePort, err := net.SplitHostPort(remoteDomain) + if err != nil { + return nil, fmt.Errorf("invalid remote domain: %s", remoteDomain) + } + remotePortInt, err := strconv.Atoi(remotePort) + if err != nil { + return nil, fmt.Errorf("invalid remote port: %s", remotePort) + } + cfg = append(cfg, configEntry{ + name: parts[0], + listenAddr: listenAddrPort.Addr(), + listenPort: listenAddrPort.Port(), + remotePort: uint16(remotePortInt), + remoteDomain: remoteDomain, + }) + } + return cfg, nil +} + +// ToEnvoyConfig converts the proxy configuration to an Envoy configuration. +// Reference: https://github.com/solo-io/envoy-operator/blob/master/pkg/kube/config.go +func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { + config := &envoyConfigBootstrapV3.Bootstrap{ + StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{}, + } + listeners := make([]*envoyConfigListenerV3.Listener, 0, len(c)) + clusters := make([]*envoyConfigClusterV3.Cluster, 0, len(c)) + for _, entry := range c { + listener, err := listener(entry) + if err != nil { + return nil, err + } + listeners = append(listeners, listener) + cluster, err := cluster(entry) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) + } + config.StaticResources.Listeners = listeners + config.StaticResources.Clusters = clusters + + if err := config.ValidateAll(); err != nil { + return nil, err + } + + configBytes, err := protojson.Marshal(config) + if err != nil { + return nil, err + } + + return configBytes, nil +} + +func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { + proxy := &envoyConfigTCPProxyV3.TcpProxy{ + StatPrefix: entry.name, + ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{ + Cluster: entry.name, + }, + } + + proxyAny, err := anypb.New(proxy) + if err != nil { + return nil, err + } + + return &envoyConfigListenerV3.Listener{ + Name: entry.name, + Address: &envoyCoreV3.Address{ + Address: &envoyCoreV3.Address_SocketAddress{ + SocketAddress: &envoyCoreV3.SocketAddress{ + Address: entry.listenAddr.String(), + PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{ + PortValue: uint32(entry.listenPort), + }, + }, + }, + }, + FilterChains: []*envoyConfigListenerV3.FilterChain{ + { + Filters: []*envoyConfigListenerV3.Filter{ + { + Name: "envoy.filters.network.tcp_proxy", + ConfigType: &envoyConfigListenerV3.Filter_TypedConfig{ + TypedConfig: proxyAny, + }, + }, + }, + }, + }, + }, nil +} + +func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { + socket, err := tlsTransportSocket() + if err != nil { + return nil, err + } + + return &envoyConfigClusterV3.Cluster{ + Name: entry.name, + ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{ + Type: envoyConfigClusterV3.Cluster_LOGICAL_DNS, + }, + DnsLookupFamily: envoyConfigClusterV3.Cluster_V4_ONLY, + LoadAssignment: &endpointV3.ClusterLoadAssignment{ + ClusterName: entry.name, + Endpoints: []*endpointV3.LocalityLbEndpoints{ + { + LbEndpoints: []*endpointV3.LbEndpoint{ + { + HostIdentifier: &endpointV3.LbEndpoint_Endpoint{ + Endpoint: &endpointV3.Endpoint{ + Address: &envoyCoreV3.Address{ + Address: &envoyCoreV3.Address_SocketAddress{ + SocketAddress: &envoyCoreV3.SocketAddress{ + Address: entry.remoteDomain, + PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{ + PortValue: uint32(entry.remotePort), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + TransportSocket: socket, + }, nil +} + +func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) { + tls := &envoyTLSV3.UpstreamTlsContext{ + CommonTlsContext: &envoyTLSV3.CommonTlsContext{ + TlsCertificates: []*envoyTLSV3.TlsCertificate{ + { + PrivateKey: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/key.pem", + }, + }, + CertificateChain: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/certChain.pem", + }, + }, + }, + }, + ValidationContextType: &envoyTLSV3.CommonTlsContext_ValidationContext{ + ValidationContext: &envoyTLSV3.CertificateValidationContext{ + TrustedCa: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/MeshCACert.pem", + }, + }, + }, + }, + }, + } + tlsAny, err := anypb.New(tls) + if err != nil { + return nil, err + } + + return &envoyCoreV3.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &envoyCoreV3.TransportSocket_TypedConfig{ + TypedConfig: tlsAny, + }, + }, nil +} diff --git a/service-mesh/go.mod b/service-mesh/go.mod new file mode 100644 index 0000000000..48ed11a335 --- /dev/null +++ b/service-mesh/go.mod @@ -0,0 +1,17 @@ +module github.com/edgelesssys/nunki/service-mesh + +go 1.21 + +require ( + github.com/envoyproxy/go-control-plane v0.12.0 + google.golang.org/protobuf v1.32.0 +) + +require ( + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect +) diff --git a/service-mesh/go.sum b/service-mesh/go.sum new file mode 100644 index 0000000000..3b5621ff9b --- /dev/null +++ b/service-mesh/go.sum @@ -0,0 +1,64 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/service-mesh/main.go b/service-mesh/main.go new file mode 100644 index 0000000000..64b21402e2 --- /dev/null +++ b/service-mesh/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/exec" + "syscall" +) + +const proxyConfigEnvVar = "EDG_PROXY_CONFIG" + +var version = "0.0.0-dev" + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func run() (retErr error) { + log.Printf("service-mesh version %s\n", version) + + proxyConfig := os.Getenv(proxyConfigEnvVar) + if proxyConfig == "" { + return fmt.Errorf("no proxy configuration found in environment") + } + + pconfig, err := ParseProxyConfig(proxyConfig) + if err != nil { + return err + } + + envoyConfig, err := pconfig.ToEnvoyConfig() + if err != nil { + return err + } + + log.Printf("Using envoy configuration:\n%s\n", envoyConfig) + + if err := os.WriteFile("/envoy-config.yaml", envoyConfig, 0o644); err != nil { + return err + } + + // execute the envoy binary + envoyBin, err := exec.LookPath("envoy") + if err != nil { + return err + } + + log.Println("Starting envoy") + + return syscall.Exec(envoyBin, []string{"envoy", "-c", "/envoy-config.yaml"}, os.Environ()) +} From 461ab1a80e6e8d9d3f70b88d14652b6838679c42 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Tue, 13 Feb 2024 11:54:24 +0100 Subject: [PATCH 2/2] service-mesh: add emojivoto example --- .../emojivoto-sm-egress/coordinator.yml | 48 +++++++++ deployments/emojivoto-sm-egress/emoji.yml | 90 ++++++++++++++++ deployments/emojivoto-sm-egress/ns.yml | 4 + .../emojivoto-sm-egress/portforwarder.yml | 59 +++++++++++ deployments/emojivoto-sm-egress/vote-bot.yml | 35 ++++++ deployments/emojivoto-sm-egress/voting.yml | 90 ++++++++++++++++ deployments/emojivoto-sm-egress/web.yml | 100 ++++++++++++++++++ justfile | 2 +- 8 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 deployments/emojivoto-sm-egress/coordinator.yml create mode 100644 deployments/emojivoto-sm-egress/emoji.yml create mode 100644 deployments/emojivoto-sm-egress/ns.yml create mode 100644 deployments/emojivoto-sm-egress/portforwarder.yml create mode 100644 deployments/emojivoto-sm-egress/vote-bot.yml create mode 100644 deployments/emojivoto-sm-egress/voting.yml create mode 100644 deployments/emojivoto-sm-egress/web.yml diff --git a/deployments/emojivoto-sm-egress/coordinator.yml b/deployments/emojivoto-sm-egress/coordinator.yml new file mode 100644 index 0000000000..2b698f4142 --- /dev/null +++ b/deployments/emojivoto-sm-egress/coordinator.yml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coordinator + namespace: edg-default +spec: + selector: + matchLabels: + app.kubernetes.io/name: coordinator + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: coordinator + annotations: + nunki.edgeless.systems/pod-role: coordinator + spec: + runtimeClassName: kata-cc-isolation + containers: + - name: coordinator + image: "ghcr.io/edgelesssys/nunki/coordinator:latest" + ports: + - containerPort: 7777 + - containerPort: 1313 + env: + - name: NUNKI_LOG_LEVEL + value: "debug" + resources: + requests: + memory: 100Mi + limits: + memory: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: coordinator + namespace: edg-default +spec: + ports: + - name: intercom + port: 7777 + protocol: TCP + - name: coordapi + port: 1313 + protocol: TCP + selector: + app.kubernetes.io/name: coordinator diff --git a/deployments/emojivoto-sm-egress/emoji.yml b/deployments/emojivoto-sm-egress/emoji.yml new file mode 100644 index 0000000000..8da5e46731 --- /dev/null +++ b/deployments/emojivoto-sm-egress/emoji.yml @@ -0,0 +1,90 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: emoji + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emoji + namespace: edg-default + labels: + app.kubernetes.io/name: emoji + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: emoji-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: emoji-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi + serviceAccountName: emoji + containers: + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: ghcr.io/3u13r/emojivoto-emoji-svc:coco-1 + name: emoji-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: emoji-svc + namespace: edg-default +spec: + selector: + app.kubernetes.io/name: emoji-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto-sm-egress/ns.yml b/deployments/emojivoto-sm-egress/ns.yml new file mode 100644 index 0000000000..ed2712cc89 --- /dev/null +++ b/deployments/emojivoto-sm-egress/ns.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: edg-default diff --git a/deployments/emojivoto-sm-egress/portforwarder.yml b/deployments/emojivoto-sm-egress/portforwarder.yml new file mode 100644 index 0000000000..a5adcf9c36 --- /dev/null +++ b/deployments/emojivoto-sm-egress/portforwarder.yml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-coordinator + namespace: edg-default + labels: + app.kubernetes.io/name: port-forwarder-coordinator +spec: + containers: + - name: port-forwarder + image: "ghcr.io/edgelesssys/nunki/port-forwarder:latest" + env: + - name: LISTEN_PORT + value: "1313" + - name: FORWARD_HOST + value: coordinator + - name: FORWARD_PORT + value: "1313" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 1313 + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi +--- +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-emojivoto-web + namespace: edg-default + labels: + app.kubernetes.io/name: port-forwarder-emojivoto-web +spec: + containers: + - name: port-forwarder + image: "ghcr.io/edgelesssys/nunki/port-forwarder:latest" + env: + - name: LISTEN_PORT + value: "8080" + - name: FORWARD_HOST + value: web-svc + - name: FORWARD_PORT + value: "443" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 8080 + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi diff --git a/deployments/emojivoto-sm-egress/vote-bot.yml b/deployments/emojivoto-sm-egress/vote-bot.yml new file mode 100644 index 0000000000..2149d7b470 --- /dev/null +++ b/deployments/emojivoto-sm-egress/vote-bot.yml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vote-bot + namespace: edg-default + labels: + app.kubernetes.io/name: vote-bot + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: vote-bot + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: vote-bot + version: v11 + spec: + containers: + - command: + - emojivoto-vote-bot + env: + - name: WEB_HOST + value: web-svc:443 + image: docker.l5d.io/buoyantio/emojivoto-web:v11 + name: vote-bot + resources: + requests: + cpu: 10m + memory: 25Mi + limits: + memory: 25Mi diff --git a/deployments/emojivoto-sm-egress/voting.yml b/deployments/emojivoto-sm-egress/voting.yml new file mode 100644 index 0000000000..a87963a857 --- /dev/null +++ b/deployments/emojivoto-sm-egress/voting.yml @@ -0,0 +1,90 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: voting + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: voting + namespace: edg-default + labels: + app.kubernetes.io/name: voting + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: voting-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: voting-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi + serviceAccountName: voting + containers: + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: ghcr.io/3u13r/emojivoto-voting-svc:coco-1 + name: voting-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: voting-svc + namespace: edg-default +spec: + selector: + app.kubernetes.io/name: voting-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto-sm-egress/web.yml b/deployments/emojivoto-sm-egress/web.yml new file mode 100644 index 0000000000..35e35a2a7a --- /dev/null +++ b/deployments/emojivoto-sm-egress/web.yml @@ -0,0 +1,100 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: web + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: edg-default + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: web-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: web-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + serviceAccountName: web + containers: + - name: sidecar + image: "ghcr.io/edgelesssys/nunki/service-mesh-proxy:latest" + volumeMounts: + - name: tls-certs + mountPath: /tls-config + env: + - name: EDG_PROXY_CONFIG + value: "emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080" + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + - env: + - name: WEB_PORT + value: "8080" + - name: EMOJISVC_HOST + value: 127.137.0.1:8081 + - name: VOTINGSVC_HOST + value: 127.137.0.2:8081 + - name: INDEX_BUNDLE + value: dist/index_bundle.js + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + - name: EDG_DISABLE_CLIENT_AUTH + value: "true" + image: docker.l5d.io/buoyantio/emojivoto-web:v11 + name: web-svc + ports: + - containerPort: 8080 + name: https + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: web-svc + namespace: edg-default +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: web-svc + ports: + - name: https + port: 443 + targetPort: 8080 diff --git a/justfile b/justfile index e731cdcf25..784010b8c6 100644 --- a/justfile +++ b/justfile @@ -127,7 +127,7 @@ wait-for-workload target=default_deploy_target: nix run .#scripts.kubectl-wait-ready -- $ns openssl-client nix run .#scripts.kubectl-wait-ready -- $ns openssl-frontend ;; - "emojivoto") + "emojivoto" | "emojivoto-sm-egress") nix run .#scripts.kubectl-wait-ready -- $ns emoji-svc nix run .#scripts.kubectl-wait-ready -- $ns vote-bot nix run .#scripts.kubectl-wait-ready -- $ns voting-svc