diff --git a/deployments/emojivoto-sm-ingress/web.yml b/deployments/emojivoto-sm-ingress/web.yml index c6beba1bcb..ccb86ca304 100644 --- a/deployments/emojivoto-sm-ingress/web.yml +++ b/deployments/emojivoto-sm-ingress/web.yml @@ -35,15 +35,32 @@ spec: volumeMounts: - name: tls-certs mountPath: /tls-config + - name: sidecar + image: "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest" + restartPolicy: Always + volumeMounts: + - name: tls-certs + mountPath: /tls-config + env: + - name: EDG_INGRESS_PROXY_CONFIG + value: "web#8080#true" + - name: EDG_EGRESS_PROXY_CONFIG + value: "emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080" + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + - NET_RAW serviceAccountName: web containers: - env: - name: WEB_PORT value: "8080" - name: EMOJISVC_HOST - value: emoji-svc:8080 + value: 127.137.0.1:8081 - name: VOTINGSVC_HOST - value: voting-svc:8080 + value: 127.137.0.2:8081 - name: INDEX_BUNDLE value: dist/index_bundle.js - name: EDG_CERT_PATH @@ -54,7 +71,7 @@ spec: value: /tls-config/key.pem - name: EDG_DISABLE_CLIENT_AUTH value: "true" - image: ghcr.io/3u13r/emojivoto-web:coco-1 + image: docker.l5d.io/buoyantio/emojivoto-web:v11 name: web-svc ports: - containerPort: 8080 diff --git a/service-mesh/config.go b/service-mesh/config.go index e03efef3e2..9538e72cad 100644 --- a/service-mesh/config.go +++ b/service-mesh/config.go @@ -23,15 +23,23 @@ import ( var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") // ProxyConfig represents the configuration for the proxy. -type ProxyConfig []configEntry - -type configEntry struct { +type ProxyConfig struct { + egress []egressConfigEntry + ingress []ingressConfigEntry +} +type egressConfigEntry struct { name string + clusterName string listenAddr netip.Addr listenPort uint16 remoteDomain string remotePort uint16 } +type ingressConfigEntry struct { + name string + listenPort uint16 + disableClientCertificate bool +} // ParseProxyConfig parses the proxy configuration from the given string. // The configuration is expected to be in the following format: @@ -41,8 +49,12 @@ type configEntry struct { // Example: // // emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080 -func ParseProxyConfig(data string) (ProxyConfig, error) { - entries := strings.Split(data, "##") +func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) { + if ingressConfig == "" && egressConfig == "" { + return ProxyConfig{}, nil + } + + entries := strings.Split(egressConfig, "##") var cfg ProxyConfig for _, entry := range entries { if entry == "" { @@ -50,33 +62,59 @@ func ParseProxyConfig(data string) (ProxyConfig, error) { } parts := strings.Split(entry, "#") if len(parts) != 3 { - return nil, fmt.Errorf("invalid entry: %s", entry) + return ProxyConfig{}, fmt.Errorf("invalid entry: %s", entry) } listenAddrPort, err := netip.ParseAddrPort(parts[1]) if err != nil { - return nil, fmt.Errorf("invalid listen address: %s", parts[1]) + return ProxyConfig{}, fmt.Errorf("invalid listen address: %s", parts[1]) } if !loopbackCIDR.Contains(listenAddrPort.Addr()) { - return nil, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) + return ProxyConfig{}, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) } remoteDomain := parts[2] remoteDomain, remotePort, err := net.SplitHostPort(remoteDomain) if err != nil { - return nil, fmt.Errorf("invalid remote domain: %s", remoteDomain) + return ProxyConfig{}, fmt.Errorf("invalid remote domain: %s", remoteDomain) } remotePortInt, err := strconv.Atoi(remotePort) if err != nil { - return nil, fmt.Errorf("invalid remote port: %s", remotePort) + return ProxyConfig{}, fmt.Errorf("invalid remote port: %s", remotePort) } - cfg = append(cfg, configEntry{ + cfg.egress = append(cfg.egress, egressConfigEntry{ name: parts[0], + clusterName: parts[0], listenAddr: listenAddrPort.Addr(), listenPort: listenAddrPort.Port(), remotePort: uint16(remotePortInt), remoteDomain: remoteDomain, }) } + + for _, entry := range strings.Split(ingressConfig, "##") { + if entry == "" { + continue + } + parts := strings.Split(entry, "#") + if len(parts) != 3 { + return ProxyConfig{}, fmt.Errorf("invalid entry: %s", entry) + } + listenPort, err := strconv.Atoi(parts[1]) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid listen port: %s", parts[1]) + } + disableClientCertificate, err := strconv.ParseBool(parts[2]) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid disable client certificate: %s", parts[2]) + } + cfg.ingress = append(cfg.ingress, ingressConfigEntry{ + name: parts[0], + listenPort: uint16(listenPort), + disableClientCertificate: disableClientCertificate, + }) + + } + return cfg, nil } @@ -86,9 +124,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { config := &envoyConfigBootstrapV3.Bootstrap{ StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{}, } - listeners := make([]*envoyConfigListenerV3.Listener, 0, len(c)) - clusters := make([]*envoyConfigClusterV3.Cluster, 0, len(c)) - for _, entry := range c { + listeners := make([]*envoyConfigListenerV3.Listener, 0) + clusters := make([]*envoyConfigClusterV3.Cluster, 0) + + // Create listeners and clusters for egress traffic. + for _, entry := range c.egress { listener, err := listener(entry) if err != nil { return nil, err @@ -106,6 +146,10 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { if err != nil { return nil, err } + ingrListenerNoClientAuth, err := ingressListener("ingressWithoutClientAuth", 15007, false) + if err != nil { + return nil, err + } ingressCluster := &envoyConfigClusterV3.Cluster{ Name: "ingress", @@ -123,6 +167,7 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { } listeners = append(listeners, ingrListenerClientAuth) + listeners = append(listeners, ingrListenerNoClientAuth) clusters = append(clusters, ingressCluster) config.StaticResources.Listeners = listeners @@ -140,11 +185,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { return configBytes, nil } -func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { +func listener(entry egressConfigEntry) (*envoyConfigListenerV3.Listener, error) { proxy := &envoyConfigTCPProxyV3.TcpProxy{ StatPrefix: entry.name, ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{ - Cluster: entry.name, + Cluster: entry.clusterName, }, } @@ -180,7 +225,7 @@ func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { }, nil } -func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { +func cluster(entry egressConfigEntry) (*envoyConfigClusterV3.Cluster, error) { socket, err := upstreamTLSTransportSocket() if err != nil { return nil, err @@ -222,10 +267,11 @@ func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { } func ingressListener(name string, listenPort uint16, requireClientCertificate bool) (*envoyConfigListenerV3.Listener, error) { - ingressListener, err := listener(configEntry{ - name: name, - listenAddr: netip.MustParseAddr("0.0.0.0"), - listenPort: listenPort, + ingressListener, err := listener(egressConfigEntry{ + name: name, + clusterName: "ingress", + listenAddr: netip.MustParseAddr("0.0.0.0"), + listenPort: listenPort, }) if err != nil { return nil, err diff --git a/service-mesh/iptables.go b/service-mesh/iptables.go index 30bae98bdc..92f1b2ee29 100644 --- a/service-mesh/iptables.go +++ b/service-mesh/iptables.go @@ -11,7 +11,7 @@ import ( const EnvoyIngressPort = 15006 // IngressIPTableRules sets up the iptables rules for the ingress proxy. -func IngressIPTableRules() error { +func IngressIPTableRules(ingressEntries []ingressConfigEntry) error { // Create missing `/run/xtables.lock` file. if err := os.Mkdir("/run", 0o755); err != nil { if !os.IsExist(err) { @@ -63,6 +63,14 @@ func IngressIPTableRules() error { return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to EDG_INBOUND chain: %w", err) } + for _, entry := range ingressEntries { + if entry.disableClientCertificate { + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "!", "-d", "127.0.0.1/32", "-p", "tcp", "--dport", fmt.Sprintf("%d", entry.listenPort), "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", 15007)); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_IN_REDIRECT chain: %w", err) + } + } + } + // Route all traffic not destined for 127.0.0.1 to the envoy proxy on its // port that requires client authentication. if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "!", "-d", "127.0.0.1/32", "-p", "tcp", "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPort)); err != nil { diff --git a/service-mesh/main.go b/service-mesh/main.go index 5296e26c1c..8c0f02d888 100644 --- a/service-mesh/main.go +++ b/service-mesh/main.go @@ -9,8 +9,9 @@ import ( ) const ( - proxyConfigEnvVar = "EDG_PROXY_CONFIG" - envoyConfigFile = "/envoy-config.yml" + egressProxyConfigEnvVar = "EDG_EGRESS_PROXY_CONFIG" + ingressProxyConfigEnvVar = "EDG_INGRESS_PROXY_CONFIG" + envoyConfigFile = "/envoy-config.yml" ) var version = "0.0.0-dev" @@ -25,9 +26,13 @@ func main() { func run() (retErr error) { log.Printf("service-mesh version %s\n", version) - proxyConfig := os.Getenv(proxyConfigEnvVar) + egressProxyConfig := os.Getenv(egressProxyConfigEnvVar) + log.Println("Ingress Proxy configuration:", egressProxyConfig) - pconfig, err := ParseProxyConfig(proxyConfig) + ingressProxyConfig := os.Getenv(ingressProxyConfigEnvVar) + log.Println("Egress Proxy configuration:", ingressProxyConfig) + + pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig) if err != nil { return err } @@ -43,7 +48,7 @@ func run() (retErr error) { return err } - if err := IngressIPTableRules(); err != nil { + if err := IngressIPTableRules(pconfig.ingress); err != nil { return fmt.Errorf("failed to set up iptables rules: %w", err) }