diff --git a/deployments/emojivoto-sm-ingress/web.yml b/deployments/emojivoto-sm-ingress/web.yml index 40909eb17..10d90b3b9 100644 --- a/deployments/emojivoto-sm-ingress/web.yml +++ b/deployments/emojivoto-sm-ingress/web.yml @@ -37,13 +37,29 @@ spec: mountPath: /tls-config serviceAccountName: web containers: + - name: sidecar + image: "ghcr.io/edgelesssys/nunki/service-mesh-proxy:latest" + volumeMounts: + - name: tls-certs + mountPath: /tls-config + env: + - name: EDG_INGRESS_PROXY_CONFIG + value: "web#8080#true" + - name: EDG_EGRESS_PROXY_CONFIG + value: "emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080" + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + - NET_RAW - env: - name: WEB_PORT value: "8080" - name: EMOJISVC_HOST - value: emoji-svc:8080 + value: 127.137.0.1:8081 - name: VOTINGSVC_HOST - value: voting-svc:8080 + value: 127.137.0.2:8081 - name: INDEX_BUNDLE value: dist/index_bundle.js - name: EDG_CERT_PATH @@ -54,7 +70,7 @@ spec: value: /tls-config/key.pem - name: EDG_DISABLE_CLIENT_AUTH value: "true" - image: ghcr.io/3u13r/emojivoto-web:coco-1 + image: docker.l5d.io/buoyantio/emojivoto-web:v11 name: web-svc ports: - containerPort: 8080 diff --git a/service-mesh/config.go b/service-mesh/config.go index 80bac40a8..8eefa451e 100644 --- a/service-mesh/config.go +++ b/service-mesh/config.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "net" "net/netip" "strconv" @@ -23,15 +24,23 @@ import ( var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") // ProxyConfig represents the configuration for the proxy. -type ProxyConfig []configEntry - -type configEntry struct { +type ProxyConfig struct { + egress []egressConfigEntry + ingress []ingressConfigEntry +} +type egressConfigEntry struct { name string + clusterName string listenAddr netip.Addr listenPort uint16 remoteDomain string remotePort uint16 } +type ingressConfigEntry struct { + name string + listenPort uint16 + disableClientCertificate bool +} // ParseProxyConfig parses the proxy configuration from the given string. // The configuration is expected to be in the following format: @@ -41,43 +50,68 @@ type configEntry struct { // Example: // // emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080 -func ParseProxyConfig(data string) (ProxyConfig, error) { - if data == "" { +func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) { + if ingressConfig == "" && egressConfig == "" { return ProxyConfig{}, nil } - entries := strings.Split(data, "##") + entries := strings.Split(egressConfig, "##") var cfg ProxyConfig for _, entry := range entries { parts := strings.Split(entry, "#") if len(parts) != 3 { - return nil, fmt.Errorf("invalid entry: %s", entry) + log.Printf("Invalid entry: %s\n", entry) + continue } listenAddrPort, err := netip.ParseAddrPort(parts[1]) if err != nil { - return nil, fmt.Errorf("invalid listen address: %s", parts[1]) + return ProxyConfig{}, fmt.Errorf("invalid listen address: %s", parts[1]) } if !loopbackCIDR.Contains(listenAddrPort.Addr()) { - return nil, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) + return ProxyConfig{}, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) } remoteDomain := parts[2] remoteDomain, remotePort, err := net.SplitHostPort(remoteDomain) if err != nil { - return nil, fmt.Errorf("invalid remote domain: %s", remoteDomain) + return ProxyConfig{}, fmt.Errorf("invalid remote domain: %s", remoteDomain) } remotePortInt, err := strconv.Atoi(remotePort) if err != nil { - return nil, fmt.Errorf("invalid remote port: %s", remotePort) + return ProxyConfig{}, fmt.Errorf("invalid remote port: %s", remotePort) } - cfg = append(cfg, configEntry{ + cfg.egress = append(cfg.egress, egressConfigEntry{ name: parts[0], + clusterName: parts[0], listenAddr: listenAddrPort.Addr(), listenPort: listenAddrPort.Port(), remotePort: uint16(remotePortInt), remoteDomain: remoteDomain, }) } + + for _, entry := range strings.Split(ingressConfig, "##") { + parts := strings.Split(entry, "#") + if len(parts) != 3 { + log.Printf("Invalid entry: %s\n", entry) + continue + } + listenPort, err := strconv.Atoi(parts[1]) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid listen port: %s", parts[1]) + } + disableClientCertificate, err := strconv.ParseBool(parts[2]) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid disable client certificate: %s", parts[2]) + } + cfg.ingress = append(cfg.ingress, ingressConfigEntry{ + name: parts[0], + listenPort: uint16(listenPort), + disableClientCertificate: disableClientCertificate, + }) + + } + return cfg, nil } @@ -87,9 +121,9 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { config := &envoyConfigBootstrapV3.Bootstrap{ StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{}, } - listeners := make([]*envoyConfigListenerV3.Listener, 0, len(c)) - clusters := make([]*envoyConfigClusterV3.Cluster, 0, len(c)) - for _, entry := range c { + listeners := make([]*envoyConfigListenerV3.Listener, 0) + clusters := make([]*envoyConfigClusterV3.Cluster, 0) + for _, entry := range c.egress { listener, err := listener(entry) if err != nil { return nil, err @@ -102,31 +136,14 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { clusters = append(clusters, cluster) } - ingressListener, err := listener(configEntry{ - name: "ingress", - listenAddr: netip.MustParseAddr("0.0.0.0"), - listenPort: EnvoyIngressPort, - }) + ingrListenerClientAuth, err := ingressListener("ingress", 15006, true) if err != nil { return nil, err } - ingressListener.Transparent = &wrapperspb.BoolValue{Value: true} - originalDstConfig := &envoyOrigDstV3.OriginalDst{} - originalDstAny, err := anypb.New(originalDstConfig) + ingrListenerNoClientAuth, err := ingressListener("ingressWithoutClientAuth", 15007, false) if err != nil { return nil, err } - ingressListener.ListenerFilters = []*envoyConfigListenerV3.ListenerFilter{ - { - Name: "envoy.filters.listener.original_dst", - ConfigType: &envoyConfigListenerV3.ListenerFilter_TypedConfig{TypedConfig: originalDstAny}, - }, - } - tlsSock, err := downstreamTLSTransportSocket() - if err != nil { - return nil, err - } - ingressListener.FilterChains[0].TransportSocket = tlsSock ingressCluster := &envoyConfigClusterV3.Cluster{ Name: "ingress", @@ -136,9 +153,9 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { CircuitBreakers: &envoyConfigClusterV3.CircuitBreakers{ Thresholds: []*envoyConfigClusterV3.CircuitBreakers_Thresholds{{ MaxRetries: &wrapperspb.UInt32Value{Value: 1}, - MaxConnections: &wrapperspb.UInt32Value{Value: 2}, - MaxRequests: &wrapperspb.UInt32Value{Value: 1}, - MaxPendingRequests: &wrapperspb.UInt32Value{Value: 1}, + MaxConnections: &wrapperspb.UInt32Value{Value: 5}, + MaxRequests: &wrapperspb.UInt32Value{Value: 5}, + MaxPendingRequests: &wrapperspb.UInt32Value{Value: 5}, }}, }, UpstreamBindConfig: &envoyCoreV3.BindConfig{ @@ -151,7 +168,8 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { }, } - listeners = append(listeners, ingressListener) + listeners = append(listeners, ingrListenerClientAuth) + listeners = append(listeners, ingrListenerNoClientAuth) clusters = append(clusters, ingressCluster) config.StaticResources.Listeners = listeners @@ -183,11 +201,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { return configBytes, nil } -func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { +func listener(entry egressConfigEntry) (*envoyConfigListenerV3.Listener, error) { proxy := &envoyConfigTCPProxyV3.TcpProxy{ StatPrefix: entry.name, ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{ - Cluster: entry.name, + Cluster: entry.clusterName, }, } @@ -223,7 +241,7 @@ func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { }, nil } -func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { +func cluster(entry egressConfigEntry) (*envoyConfigClusterV3.Cluster, error) { socket, err := upstreamTLSTransportSocket() if err != nil { return nil, err @@ -264,6 +282,36 @@ func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { }, nil } +func ingressListener(name string, listenPort uint16, requireClientCertificate bool) (*envoyConfigListenerV3.Listener, error) { + ingressListener, err := listener(egressConfigEntry{ + name: name, + clusterName: "ingress", + listenAddr: netip.MustParseAddr("0.0.0.0"), + listenPort: listenPort, + }) + if err != nil { + return nil, err + } + ingressListener.Transparent = &wrapperspb.BoolValue{Value: true} + originalDstConfig := &envoyOrigDstV3.OriginalDst{} + originalDstAny, err := anypb.New(originalDstConfig) + if err != nil { + return nil, err + } + ingressListener.ListenerFilters = []*envoyConfigListenerV3.ListenerFilter{ + { + Name: "envoy.filters.listener.original_dst", + ConfigType: &envoyConfigListenerV3.ListenerFilter_TypedConfig{TypedConfig: originalDstAny}, + }, + } + tlsSock, err := downstreamTLSTransportSocket(requireClientCertificate) + if err != nil { + return nil, err + } + ingressListener.FilterChains[0].TransportSocket = tlsSock + return ingressListener, nil +} + func upstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { tls := &envoyTLSV3.UpstreamTlsContext{ CommonTlsContext: &envoyTLSV3.CommonTlsContext{ @@ -305,7 +353,7 @@ func upstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { }, nil } -func downstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { +func downstreamTLSTransportSocket(requireClientCertificate bool) (*envoyCoreV3.TransportSocket, error) { tls := &envoyTLSV3.DownstreamTlsContext{ CommonTlsContext: &envoyTLSV3.CommonTlsContext{ TlsCertificates: []*envoyTLSV3.TlsCertificate{ @@ -332,7 +380,7 @@ func downstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { }, }, }, - RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + RequireClientCertificate: &wrapperspb.BoolValue{Value: requireClientCertificate}, } tlsAny, err := anypb.New(tls) if err != nil { diff --git a/service-mesh/iptables.go b/service-mesh/iptables.go index 754729df9..bdeb85c6e 100644 --- a/service-mesh/iptables.go +++ b/service-mesh/iptables.go @@ -11,7 +11,7 @@ import ( const EnvoyIngressPort = 15006 // IngressIPTableRules sets up the iptables rules for the ingress proxy. -func IngressIPTableRules() error { +func IngressIPTableRules(ingressEntries []ingressConfigEntry) error { if err := os.Mkdir("/run", 0o755); err != nil { if !os.IsExist(err) { return fmt.Errorf("failed to create /run directory: %w", err) @@ -57,6 +57,14 @@ func IngressIPTableRules() error { return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to EDG_INBOUND chain: %w", err) } + for _, entry := range ingressEntries { + if entry.disableClientCertificate { + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "!", "-d", "127.0.0.1/32", "-p", "tcp", "--dport", fmt.Sprintf("%d", entry.listenPort), "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", 15007)); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_IN_REDIRECT chain: %w", err) + } + } + } + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "!", "-d", "127.0.0.1/32", "-p", "tcp", "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPort)); err != nil { return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to TPROXY chain: %w", err) } diff --git a/service-mesh/main.go b/service-mesh/main.go index 6cccde2b9..81998f18e 100644 --- a/service-mesh/main.go +++ b/service-mesh/main.go @@ -9,7 +9,10 @@ import ( "time" ) -const proxyConfigEnvVar = "EDG_PROXY_CONFIG" +const ( + egressProxyConfigEnvVar = "EDG_EGRESS_PROXY_CONFIG" + ingressProxyConfigEnvVar = "EDG_INGRESS_PROXY_CONFIG" +) var version = "0.0.0-dev" @@ -24,10 +27,13 @@ func main() { func run() (retErr error) { log.Printf("service-mesh version %s\n", version) - proxyConfig := os.Getenv(proxyConfigEnvVar) - log.Println("Proxy configuration:", proxyConfig) + egressProxyConfig := os.Getenv(egressProxyConfigEnvVar) + log.Println("Ingress Proxy configuration:", egressProxyConfig) + + ingressProxyConfig := os.Getenv(ingressProxyConfigEnvVar) + log.Println("Egress Proxy configuration:", ingressProxyConfig) - pconfig, err := ParseProxyConfig(proxyConfig) + pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig) if err != nil { return err } @@ -43,7 +49,7 @@ func run() (retErr error) { return err } - if err := IngressIPTableRules(); err != nil { + if err := IngressIPTableRules(pconfig.ingress); err != nil { return fmt.Errorf("failed to set up iptables rules: %w", err) } @@ -55,5 +61,5 @@ func run() (retErr error) { log.Println("Starting envoy") - return syscall.Exec(envoyBin, []string{"envoy", "-l", "debug", "-c", "/envoy-config.yaml"}, os.Environ()) + return syscall.Exec(envoyBin, []string{"envoy", "-c", "/envoy-config.yaml"}, os.Environ()) }