Skip to content

Commit

Permalink
service-mesh: first ingress implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
3u13r committed Mar 28, 2024
1 parent 8cad7a7 commit 518433e
Show file tree
Hide file tree
Showing 7 changed files with 182 additions and 7 deletions.
2 changes: 1 addition & 1 deletion packages/by-name/service-mesh/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ buildGoModule rec {
};

proxyVendor = true;
vendorHash = "sha256-PyqMDkayG7yMNN5X+RqutqTX/yBh209V3vz7qw7dzS8=";
vendorHash = "sha256-p0vkQqe6Q11N0pSP28faXwnMszJyLCUhjeMBTabZWCI=";

subPackages = [ "." ];

Expand Down
3 changes: 2 additions & 1 deletion packages/containers.nix
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ let
service-mesh-proxy = dockerTools.buildImage {
name = "service-mesh-proxy";
tag = "v${service-mesh.version}";
copyToRoot = [ envoy ];
#copyToRoot = [ envoy iptables-legacy ];
copyToRoot = [ bash coreutils ncurses bashInteractive vim procps envoy tcpdump wget kmod gzip gnugrep iproute2 python3 iptables-legacy jq util-linux curl openssl strace grpcurl pwru service-mesh ];
config = {
Cmd = [ "${service-mesh}/bin/service-mesh" ];
Env = [ "PATH=/bin" ]; # This is only here for policy generation.
Expand Down
97 changes: 95 additions & 2 deletions service-mesh/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@ import (
envoyCoreV3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
endpointV3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoyConfigListenerV3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoyOrigDstV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/original_dst/v3"
envoyConfigTCPProxyV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
envoyTLSV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/wrapperspb"
)

var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8")
Expand Down Expand Up @@ -43,6 +45,9 @@ func ParseProxyConfig(data string) (ProxyConfig, error) {
entries := strings.Split(data, "##")
var cfg ProxyConfig
for _, entry := range entries {
if entry == "" {
continue
}
parts := strings.Split(entry, "#")
if len(parts) != 3 {
return nil, fmt.Errorf("invalid entry: %s", entry)
Expand Down Expand Up @@ -95,6 +100,23 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) {
}
clusters = append(clusters, cluster)
}

// Create listeners and clusters for ingress traffic.
ingrListenerClientAuth, err := ingressListener("ingress", 15006, true)
if err != nil {
return nil, err
}

ingressCluster := &envoyConfigClusterV3.Cluster{
Name: "ingress",
ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{Type: envoyConfigClusterV3.Cluster_ORIGINAL_DST},
DnsLookupFamily: envoyConfigClusterV3.Cluster_V4_ONLY,
LbPolicy: envoyConfigClusterV3.Cluster_CLUSTER_PROVIDED,
}

listeners = append(listeners, ingrListenerClientAuth)
clusters = append(clusters, ingressCluster)

config.StaticResources.Listeners = listeners
config.StaticResources.Clusters = clusters

Expand Down Expand Up @@ -151,7 +173,7 @@ func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) {
}

func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) {
socket, err := tlsTransportSocket()
socket, err := upstreamTLSTransportSocket()
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -191,7 +213,36 @@ func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) {
}, nil
}

func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) {
func ingressListener(name string, listenPort uint16, requireClientCertificate bool) (*envoyConfigListenerV3.Listener, error) {
ingressListener, err := listener(configEntry{
name: name,
listenAddr: netip.MustParseAddr("0.0.0.0"),
listenPort: listenPort,
})
if err != nil {
return nil, err
}
ingressListener.Transparent = &wrapperspb.BoolValue{Value: true}
originalDstConfig := &envoyOrigDstV3.OriginalDst{}
originalDstAny, err := anypb.New(originalDstConfig)
if err != nil {
return nil, err
}
ingressListener.ListenerFilters = []*envoyConfigListenerV3.ListenerFilter{
{
Name: "envoy.filters.listener.original_dst",
ConfigType: &envoyConfigListenerV3.ListenerFilter_TypedConfig{TypedConfig: originalDstAny},
},
}
tlsSock, err := downstreamTLSTransportSocket(requireClientCertificate)
if err != nil {
return nil, err
}
ingressListener.FilterChains[0].TransportSocket = tlsSock
return ingressListener, nil
}

func upstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) {
tls := &envoyTLSV3.UpstreamTlsContext{
CommonTlsContext: &envoyTLSV3.CommonTlsContext{
TlsCertificates: []*envoyTLSV3.TlsCertificate{
Expand Down Expand Up @@ -231,3 +282,45 @@ func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) {
},
}, nil
}

func downstreamTLSTransportSocket(requireClientCertificate bool) (*envoyCoreV3.TransportSocket, error) {
tls := &envoyTLSV3.DownstreamTlsContext{
CommonTlsContext: &envoyTLSV3.CommonTlsContext{
TlsCertificates: []*envoyTLSV3.TlsCertificate{
{
PrivateKey: &envoyCoreV3.DataSource{
Specifier: &envoyCoreV3.DataSource_Filename{
Filename: "/tls-config/key.pem",
},
},
CertificateChain: &envoyCoreV3.DataSource{
Specifier: &envoyCoreV3.DataSource_Filename{
Filename: "/tls-config/certChain.pem",
},
},
},
},
ValidationContextType: &envoyTLSV3.CommonTlsContext_ValidationContext{
ValidationContext: &envoyTLSV3.CertificateValidationContext{
TrustedCa: &envoyCoreV3.DataSource{
Specifier: &envoyCoreV3.DataSource_Filename{
Filename: "/tls-config/MeshCACert.pem",
},
},
},
},
},
RequireClientCertificate: &wrapperspb.BoolValue{Value: requireClientCertificate},
}
tlsAny, err := anypb.New(tls)
if err != nil {
return nil, err
}

return &envoyCoreV3.TransportSocket{
Name: "envoy.transport_sockets.tls",
ConfigType: &envoyCoreV3.TransportSocket_TypedConfig{
TypedConfig: tlsAny,
},
}, nil
}
1 change: 1 addition & 0 deletions service-mesh/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21
require (
github.com/envoyproxy/go-control-plane v0.12.0
google.golang.org/protobuf v1.33.0
github.com/coreos/go-iptables v0.7.0
)

require (
Expand Down
2 changes: 2 additions & 0 deletions service-mesh/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8=
github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI=
github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0=
Expand Down
77 changes: 77 additions & 0 deletions service-mesh/iptables.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package main

import (
"fmt"
"os"

"github.com/coreos/go-iptables/iptables"
)

// EnvoyIngressPort is the port that the envoy proxy listens on for incoming traffic.
const EnvoyIngressPort = 15006

// EnvoyIngressPortNoClientCert is the port that the envoy proxy listens on for
// incoming traffic without requiring a client certificate.
const EnvoyIngressPortNoClientCert = 15007

// IngressIPTableRules sets up the iptables rules for the ingress proxy.
func IngressIPTableRules() error {
// Create missing `/run/xtables.lock` file.
if err := os.Mkdir("/run", 0o755); err != nil {
if !os.IsExist(err) {
return fmt.Errorf("failed to create /run directory: %w", err)
}
}
file, err := os.Create("/run/xtables.lock")
if err != nil {
return fmt.Errorf("failed to create /run/xtables.lock: %w", err)
}
_ = file.Close()

iptablesExec, err := iptables.New()
if err != nil {
return fmt.Errorf("failed to create iptables client: %w", err)
}

// Reconcile to clean iptables chains.
if err := iptablesExec.ClearChain("mangle", "EDG_INBOUND"); err != nil {
return fmt.Errorf("failed to clear EDG_INBOUND chain: %w", err)
}

if err := iptablesExec.ClearChain("mangle", "EDG_IN_REDIRECT"); err != nil {
return fmt.Errorf("failed to clear EDG_IN_REDIRECT chain: %w", err)
}

// Route all TCP traffic to the EDG_INBOUND chain.
if err := iptablesExec.AppendUnique("mangle", "PREROUTING", "-p", "tcp", "-j", "EDG_INBOUND"); err != nil {
return fmt.Errorf("failed to append EDG_INBOUND chain to PREROUTING chain: %w", err)
}

// RETURN all local traffic.
if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-i", "lo", "-j", "RETURN"); err != nil {
return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err)
}
// RETURN all related and established traffic.
// Since the mangle table executes on every packet and not just before the
// connection is established, as the nat table does, we need to explicitly
// return established traffic. Then using tproxy is similar to a REDIRECT
// rule in the nat table but without the nat overhead.
// We use "conntrack" instead of "-m socket" as stated in the official
// documentation because we might not have a kernel with the "xt_socket"
// module (see: https://github.com/istio/istio/pull/22527)
if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "RETURN"); err != nil {
return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err)
}
// Route all other traffic to the EDG_IN_REDIRECT chain.
if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-j", "EDG_IN_REDIRECT"); err != nil {
return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to EDG_INBOUND chain: %w", err)
}

// Route all traffic not destined for 127.0.0.1 to the envoy proxy on its
// port that requires client authentication.
if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPort)); err != nil {
return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to TPROXY chain: %w", err)
}

return nil
}
7 changes: 4 additions & 3 deletions service-mesh/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,6 @@ func run() (retErr error) {
log.Printf("service-mesh version %s\n", version)

proxyConfig := os.Getenv(proxyConfigEnvVar)
if proxyConfig == "" {
return fmt.Errorf("no proxy configuration found in environment")
}

pconfig, err := ParseProxyConfig(proxyConfig)
if err != nil {
Expand All @@ -46,6 +43,10 @@ func run() (retErr error) {
return err
}

if err := IngressIPTableRules(); err != nil {
return fmt.Errorf("failed to set up iptables rules: %w", err)
}

// execute the envoy binary
envoyBin, err := exec.LookPath("envoy")
if err != nil {
Expand Down

0 comments on commit 518433e

Please sign in to comment.