From 2ce4cbfa22fdc7601033a74ff66298889b8a8410 Mon Sep 17 00:00:00 2001 From: Leonard Cohnen Date: Fri, 1 Mar 2024 03:44:34 +0100 Subject: [PATCH] wip: first working draft --- .../emojivoto-sm-ingress/coordinator.yml | 48 +++++++ deployments/emojivoto-sm-ingress/emoji.yml | 104 +++++++++++++++ deployments/emojivoto-sm-ingress/ns.yml | 4 + .../emojivoto-sm-ingress/portforwarder.yml | 59 +++++++++ deployments/emojivoto-sm-ingress/vote-bot.yml | 35 ++++++ deployments/emojivoto-sm-ingress/voting.yml | 104 +++++++++++++++ deployments/emojivoto-sm-ingress/web.yml | 87 +++++++++++++ justfile | 2 +- packages/by-name/service-mesh/package.nix | 2 +- packages/containers.nix | 3 +- service-mesh/config.go | 119 +++++++++++++++++- service-mesh/go.mod | 1 + service-mesh/go.sum | 2 + service-mesh/iptables.go | 72 +++++++++++ service-mesh/main.go | 12 +- 15 files changed, 645 insertions(+), 9 deletions(-) create mode 100644 deployments/emojivoto-sm-ingress/coordinator.yml create mode 100644 deployments/emojivoto-sm-ingress/emoji.yml create mode 100644 deployments/emojivoto-sm-ingress/ns.yml create mode 100644 deployments/emojivoto-sm-ingress/portforwarder.yml create mode 100644 deployments/emojivoto-sm-ingress/vote-bot.yml create mode 100644 deployments/emojivoto-sm-ingress/voting.yml create mode 100644 deployments/emojivoto-sm-ingress/web.yml create mode 100644 service-mesh/iptables.go diff --git a/deployments/emojivoto-sm-ingress/coordinator.yml b/deployments/emojivoto-sm-ingress/coordinator.yml new file mode 100644 index 0000000000..2b698f4142 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/coordinator.yml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coordinator + namespace: edg-default +spec: + selector: + matchLabels: + app.kubernetes.io/name: coordinator + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: coordinator + annotations: + nunki.edgeless.systems/pod-role: coordinator + spec: + runtimeClassName: kata-cc-isolation + containers: + - name: coordinator + image: "ghcr.io/edgelesssys/nunki/coordinator:latest" + ports: + - containerPort: 7777 + - containerPort: 1313 + env: + - name: NUNKI_LOG_LEVEL + value: "debug" + resources: + requests: + memory: 100Mi + limits: + memory: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: coordinator + namespace: edg-default +spec: + ports: + - name: intercom + port: 7777 + protocol: TCP + - name: coordapi + port: 1313 + protocol: TCP + selector: + app.kubernetes.io/name: coordinator diff --git a/deployments/emojivoto-sm-ingress/emoji.yml b/deployments/emojivoto-sm-ingress/emoji.yml new file mode 100644 index 0000000000..d6cf6b9504 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/emoji.yml @@ -0,0 +1,104 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: emoji + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emoji + namespace: edg-default + labels: + app.kubernetes.io/name: emoji + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: emoji-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: emoji-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi + serviceAccountName: emoji + containers: + - name: sidecar + image: "ghcr.io/edgelesssys/nunki/service-mesh-proxy:latest" + volumeMounts: + - name: tls-certs + mountPath: /tls-config + env: + - name: EDG_PROXY_CONFIG + value: "" + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + - NET_RAW + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: docker.l5d.io/buoyantio/emojivoto-emoji-svc:v11 + name: emoji-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: emoji-svc + namespace: edg-default +spec: + selector: + app.kubernetes.io/name: emoji-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto-sm-ingress/ns.yml b/deployments/emojivoto-sm-ingress/ns.yml new file mode 100644 index 0000000000..ed2712cc89 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/ns.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: edg-default diff --git a/deployments/emojivoto-sm-ingress/portforwarder.yml b/deployments/emojivoto-sm-ingress/portforwarder.yml new file mode 100644 index 0000000000..a5adcf9c36 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/portforwarder.yml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-coordinator + namespace: edg-default + labels: + app.kubernetes.io/name: port-forwarder-coordinator +spec: + containers: + - name: port-forwarder + image: "ghcr.io/edgelesssys/nunki/port-forwarder:latest" + env: + - name: LISTEN_PORT + value: "1313" + - name: FORWARD_HOST + value: coordinator + - name: FORWARD_PORT + value: "1313" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 1313 + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi +--- +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-emojivoto-web + namespace: edg-default + labels: + app.kubernetes.io/name: port-forwarder-emojivoto-web +spec: + containers: + - name: port-forwarder + image: "ghcr.io/edgelesssys/nunki/port-forwarder:latest" + env: + - name: LISTEN_PORT + value: "8080" + - name: FORWARD_HOST + value: web-svc + - name: FORWARD_PORT + value: "443" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 8080 + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi diff --git a/deployments/emojivoto-sm-ingress/vote-bot.yml b/deployments/emojivoto-sm-ingress/vote-bot.yml new file mode 100644 index 0000000000..be4802217a --- /dev/null +++ b/deployments/emojivoto-sm-ingress/vote-bot.yml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vote-bot + namespace: edg-default + labels: + app.kubernetes.io/name: vote-bot + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: vote-bot + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: vote-bot + version: v11 + spec: + containers: + - command: + - emojivoto-vote-bot + env: + - name: WEB_HOST + value: web-svc:443 + image: ghcr.io/3u13r/emojivoto-web:coco-1 + name: vote-bot + resources: + requests: + cpu: 10m + memory: 25Mi + limits: + memory: 25Mi diff --git a/deployments/emojivoto-sm-ingress/voting.yml b/deployments/emojivoto-sm-ingress/voting.yml new file mode 100644 index 0000000000..8fa98aa437 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/voting.yml @@ -0,0 +1,104 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: voting + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: voting + namespace: edg-default + labels: + app.kubernetes.io/name: voting + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: voting-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: voting-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi + serviceAccountName: voting + containers: + - name: sidecar + image: "ghcr.io/edgelesssys/nunki/service-mesh-proxy:latest" + volumeMounts: + - name: tls-certs + mountPath: /tls-config + env: + - name: EDG_PROXY_CONFIG + value: "" + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + - NET_RAW + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: docker.l5d.io/buoyantio/emojivoto-voting-svc:v11 + name: voting-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: voting-svc + namespace: edg-default +spec: + selector: + app.kubernetes.io/name: voting-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto-sm-ingress/web.yml b/deployments/emojivoto-sm-ingress/web.yml new file mode 100644 index 0000000000..40909eb179 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/web.yml @@ -0,0 +1,87 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: web + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: edg-default + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: web-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: web-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + serviceAccountName: web + containers: + - env: + - name: WEB_PORT + value: "8080" + - name: EMOJISVC_HOST + value: emoji-svc:8080 + - name: VOTINGSVC_HOST + value: voting-svc:8080 + - name: INDEX_BUNDLE + value: dist/index_bundle.js + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + - name: EDG_DISABLE_CLIENT_AUTH + value: "true" + image: ghcr.io/3u13r/emojivoto-web:coco-1 + name: web-svc + ports: + - containerPort: 8080 + name: https + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: web-svc + namespace: edg-default +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: web-svc + ports: + - name: https + port: 443 + targetPort: 8080 diff --git a/justfile b/justfile index 784010b8c6..0c6577cf17 100644 --- a/justfile +++ b/justfile @@ -127,7 +127,7 @@ wait-for-workload target=default_deploy_target: nix run .#scripts.kubectl-wait-ready -- $ns openssl-client nix run .#scripts.kubectl-wait-ready -- $ns openssl-frontend ;; - "emojivoto" | "emojivoto-sm-egress") + "emojivoto" | "emojivoto-sm-egress" | "emojivoto-sm-ingress") nix run .#scripts.kubectl-wait-ready -- $ns emoji-svc nix run .#scripts.kubectl-wait-ready -- $ns vote-bot nix run .#scripts.kubectl-wait-ready -- $ns voting-svc diff --git a/packages/by-name/service-mesh/package.nix b/packages/by-name/service-mesh/package.nix index a4699f8606..db45b2727d 100644 --- a/packages/by-name/service-mesh/package.nix +++ b/packages/by-name/service-mesh/package.nix @@ -23,7 +23,7 @@ buildGoModule rec { }; proxyVendor = true; - vendorHash = "sha256-uNd8HOd1HXhTvksoVLFsoIof/3VNnBZDLeEraYs5i3s="; + vendorHash = "sha256-pQjQF8wYrb0qlxzY4tTjbRuHp6wWPQNEDAOdRzJji84="; subPackages = [ "." ]; diff --git a/packages/containers.nix b/packages/containers.nix index d0bae24962..160a210613 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -63,7 +63,8 @@ let service-mesh-proxy = dockerTools.buildImage { name = "service-mesh-proxy"; tag = "v${service-mesh.version}"; - copyToRoot = [ envoy ]; + # copyToRoot = [ envoy ]; + copyToRoot = [ bash coreutils ncurses bashInteractive vim procps envoy tcpdump wget kmod gzip gnugrep iproute2 python3 iptables-legacy jq util-linux curl openssl strace grpcurl pwru service-mesh ]; config = { Cmd = [ "${service-mesh}/bin/service-mesh" ]; Env = [ "PATH=/bin" ]; # This is only here for policy generation. diff --git a/service-mesh/config.go b/service-mesh/config.go index 60d7f3d00a..80bac40a8c 100644 --- a/service-mesh/config.go +++ b/service-mesh/config.go @@ -12,10 +12,12 @@ import ( envoyCoreV3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" endpointV3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoyConfigListenerV3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoyOrigDstV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/original_dst/v3" envoyConfigTCPProxyV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" envoyTLSV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" ) var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") @@ -40,6 +42,10 @@ type configEntry struct { // // emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080 func ParseProxyConfig(data string) (ProxyConfig, error) { + if data == "" { + return ProxyConfig{}, nil + } + entries := strings.Split(data, "##") var cfg ProxyConfig for _, entry := range entries { @@ -95,9 +101,76 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { } clusters = append(clusters, cluster) } + + ingressListener, err := listener(configEntry{ + name: "ingress", + listenAddr: netip.MustParseAddr("0.0.0.0"), + listenPort: EnvoyIngressPort, + }) + if err != nil { + return nil, err + } + ingressListener.Transparent = &wrapperspb.BoolValue{Value: true} + originalDstConfig := &envoyOrigDstV3.OriginalDst{} + originalDstAny, err := anypb.New(originalDstConfig) + if err != nil { + return nil, err + } + ingressListener.ListenerFilters = []*envoyConfigListenerV3.ListenerFilter{ + { + Name: "envoy.filters.listener.original_dst", + ConfigType: &envoyConfigListenerV3.ListenerFilter_TypedConfig{TypedConfig: originalDstAny}, + }, + } + tlsSock, err := downstreamTLSTransportSocket() + if err != nil { + return nil, err + } + ingressListener.FilterChains[0].TransportSocket = tlsSock + + ingressCluster := &envoyConfigClusterV3.Cluster{ + Name: "ingress", + ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{Type: envoyConfigClusterV3.Cluster_ORIGINAL_DST}, + DnsLookupFamily: envoyConfigClusterV3.Cluster_V4_ONLY, + LbPolicy: envoyConfigClusterV3.Cluster_CLUSTER_PROVIDED, + CircuitBreakers: &envoyConfigClusterV3.CircuitBreakers{ + Thresholds: []*envoyConfigClusterV3.CircuitBreakers_Thresholds{{ + MaxRetries: &wrapperspb.UInt32Value{Value: 1}, + MaxConnections: &wrapperspb.UInt32Value{Value: 2}, + MaxRequests: &wrapperspb.UInt32Value{Value: 1}, + MaxPendingRequests: &wrapperspb.UInt32Value{Value: 1}, + }}, + }, + UpstreamBindConfig: &envoyCoreV3.BindConfig{ + SourceAddress: &envoyCoreV3.SocketAddress{ + Address: "127.0.0.6", + PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{ + PortValue: 0, + }, + }, + }, + } + + listeners = append(listeners, ingressListener) + clusters = append(clusters, ingressCluster) + config.StaticResources.Listeners = listeners config.StaticResources.Clusters = clusters + config.Admin = &envoyConfigBootstrapV3.Admin{ + AccessLogPath: "/dev/null", + Address: &envoyCoreV3.Address{ + Address: &envoyCoreV3.Address_SocketAddress{ + SocketAddress: &envoyCoreV3.SocketAddress{ + Address: "127.0.0.1", + PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{ + PortValue: 9901, + }, + }, + }, + }, + } + if err := config.ValidateAll(); err != nil { return nil, err } @@ -151,7 +224,7 @@ func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { } func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { - socket, err := tlsTransportSocket() + socket, err := upstreamTLSTransportSocket() if err != nil { return nil, err } @@ -191,7 +264,7 @@ func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { }, nil } -func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) { +func upstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { tls := &envoyTLSV3.UpstreamTlsContext{ CommonTlsContext: &envoyTLSV3.CommonTlsContext{ TlsCertificates: []*envoyTLSV3.TlsCertificate{ @@ -231,3 +304,45 @@ func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) { }, }, nil } + +func downstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { + tls := &envoyTLSV3.DownstreamTlsContext{ + CommonTlsContext: &envoyTLSV3.CommonTlsContext{ + TlsCertificates: []*envoyTLSV3.TlsCertificate{ + { + PrivateKey: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/key.pem", + }, + }, + CertificateChain: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/certChain.pem", + }, + }, + }, + }, + ValidationContextType: &envoyTLSV3.CommonTlsContext_ValidationContext{ + ValidationContext: &envoyTLSV3.CertificateValidationContext{ + TrustedCa: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/MeshCACert.pem", + }, + }, + }, + }, + }, + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + } + tlsAny, err := anypb.New(tls) + if err != nil { + return nil, err + } + + return &envoyCoreV3.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &envoyCoreV3.TransportSocket_TypedConfig{ + TypedConfig: tlsAny, + }, + }, nil +} diff --git a/service-mesh/go.mod b/service-mesh/go.mod index 48ed11a335..62a94e2675 100644 --- a/service-mesh/go.mod +++ b/service-mesh/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/envoyproxy/go-control-plane v0.12.0 google.golang.org/protobuf v1.32.0 + github.com/coreos/go-iptables v0.7.0 ) require ( diff --git a/service-mesh/go.sum b/service-mesh/go.sum index 3b5621ff9b..9e90f7cda8 100644 --- a/service-mesh/go.sum +++ b/service-mesh/go.sum @@ -6,6 +6,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= +github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= diff --git a/service-mesh/iptables.go b/service-mesh/iptables.go new file mode 100644 index 0000000000..754729df95 --- /dev/null +++ b/service-mesh/iptables.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "os" + + "github.com/coreos/go-iptables/iptables" +) + +// EnvoyIngressPort is the port that the envoy proxy listens on for incoming traffic. +const EnvoyIngressPort = 15006 + +// IngressIPTableRules sets up the iptables rules for the ingress proxy. +func IngressIPTableRules() error { + if err := os.Mkdir("/run", 0o755); err != nil { + if !os.IsExist(err) { + return fmt.Errorf("failed to create /run directory: %w", err) + } + } + file, err := os.Create("/run/xtables.lock") + if err != nil { + return fmt.Errorf("failed to create /run/xtables.lock: %w", err) + } + _ = file.Close() + + iptablesExec, err := iptables.New() + if err != nil { + return fmt.Errorf("failed to create iptables client: %w", err) + } + + if err := iptablesExec.ClearChain("mangle", "EDG_INBOUND"); err != nil { + return fmt.Errorf("failed to clear EDG_INBOUND chain: %w", err) + } + + if err := iptablesExec.ClearChain("mangle", "EDG_IN_REDIRECT"); err != nil { + return fmt.Errorf("failed to clear EDG_IN_REDIRECT chain: %w", err) + } + + if err := iptablesExec.ClearChain("mangle", "EDG_OUTPUT"); err != nil { + return fmt.Errorf("failed to clear EDG_IN_REDIRECT chain: %w", err) + } + + if err := iptablesExec.AppendUnique("mangle", "PREROUTING", "-p", "tcp", "-j", "EDG_INBOUND"); err != nil { + return fmt.Errorf("failed to append EDG_INBOUND chain to PREROUTING chain: %w", err) + } + + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-i", "lo", "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err) + } + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-s", "127.0.0.6", "-i", "lo", "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err) + } + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err) + } + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-j", "EDG_IN_REDIRECT"); err != nil { + return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to EDG_INBOUND chain: %w", err) + } + + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "!", "-d", "127.0.0.1/32", "-p", "tcp", "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPort)); err != nil { + return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to TPROXY chain: %w", err) + } + + if err := iptablesExec.AppendUnique("mangle", "OUTPUT", "-p", "tcp", "-j", "EDG_OUTPUT"); err != nil { + return fmt.Errorf("failed to append EDG_OUTPUT chain to OUTPUT chain: %w", err) + } + if err := iptablesExec.AppendUnique("mangle", "EDG_OUTPUT", "-s", "127.0.0.6", "-o", "lo", "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append exception to EDG_OUTPUT chain: %w", err) + } + + return nil +} diff --git a/service-mesh/main.go b/service-mesh/main.go index 64b21402e2..6cccde2b9f 100644 --- a/service-mesh/main.go +++ b/service-mesh/main.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "syscall" + "time" ) const proxyConfigEnvVar = "EDG_PROXY_CONFIG" @@ -15,6 +16,7 @@ var version = "0.0.0-dev" func main() { if err := run(); err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) + time.Sleep(15 * time.Second) os.Exit(1) } } @@ -23,9 +25,7 @@ func run() (retErr error) { log.Printf("service-mesh version %s\n", version) proxyConfig := os.Getenv(proxyConfigEnvVar) - if proxyConfig == "" { - return fmt.Errorf("no proxy configuration found in environment") - } + log.Println("Proxy configuration:", proxyConfig) pconfig, err := ParseProxyConfig(proxyConfig) if err != nil { @@ -43,6 +43,10 @@ func run() (retErr error) { return err } + if err := IngressIPTableRules(); err != nil { + return fmt.Errorf("failed to set up iptables rules: %w", err) + } + // execute the envoy binary envoyBin, err := exec.LookPath("envoy") if err != nil { @@ -51,5 +55,5 @@ func run() (retErr error) { log.Println("Starting envoy") - return syscall.Exec(envoyBin, []string{"envoy", "-c", "/envoy-config.yaml"}, os.Environ()) + return syscall.Exec(envoyBin, []string{"envoy", "-l", "debug", "-c", "/envoy-config.yaml"}, os.Environ()) }