Skip to content

Commit

Permalink
service-mesh: add egress implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
3u13r committed Feb 14, 2024
1 parent 6bf3ea6 commit 0c824c9
Show file tree
Hide file tree
Showing 8 changed files with 825 additions and 3 deletions.
6 changes: 6 additions & 0 deletions go.work
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
go 1.21

use (
.
./service-mesh
)
404 changes: 404 additions & 0 deletions go.work.sum

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion justfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Undeploy, rebuild, deploy.
default target=default_deploy_target: undeploy coordinator initializer openssl port-forwarder (deploy target) set verify (wait-for-workload target)
default target=default_deploy_target: undeploy coordinator initializer openssl port-forwarder service-mesh-proxy (deploy target) set verify (wait-for-workload target)

# Build the coordinator, containerize and push it.
coordinator:
Expand All @@ -13,6 +13,9 @@ openssl:
port-forwarder:
nix run .#push-port-forwarder -- "$container_registry/nunki/port-forwarder"

service-mesh-proxy:
nix run .#push-service-mesh-proxy -- "$container_registry/nunki/service-mesh-proxy"

# Build the initializer, containerize and push it.
initializer:
nix run .#push-initializer -- "$container_registry/nunki/initializer"
Expand Down
64 changes: 62 additions & 2 deletions packages/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,18 @@ let
fileset = lib.fileset.unions [
../go.mod
../go.sum
(lib.fileset.fileFilter (file: lib.hasSuffix ".go" file.name) ../.)
(lib.fileset.difference
(lib.fileset.fileFilter (file: lib.hasSuffix ".go" file.name) ../.)
../service-mesh)
];
};

goServiceMeshFiles = lib.fileset.toSource {
root = ../service-mesh;
fileset = lib.fileset.unions [
../service-mesh/go.mod
../service-mesh/go.sum
(lib.fileset.fileFilter (file: lib.hasSuffix ".go" file.name) ../service-mesh)
];
};

Expand Down Expand Up @@ -67,6 +78,7 @@ rec {
runHook postCheck
'';

# TODO(@katexochen): Allow subPackages with "-" in their names
postInstall = ''
for sub in ${builtins.concatStringsSep " " subPackages}; do
install -Dm755 "$out/bin/$sub" "''${!sub}/bin/$sub"
Expand All @@ -77,6 +89,38 @@ rec {
};
inherit (nunki) cli;

service-mesh =
lib.makeOverridable buildGoModule {
inherit version;
name = "service-mesh";

src = goServiceMeshFiles;
proxyVendor = true;
vendorHash = "sha256-uNd8HOd1HXhTvksoVLFsoIof/3VNnBZDLeEraYs5i3s=";

subPackages = [ "." ];

CGO_ENABLED = 0;
ldflags = [
"-s"
"-w"
"-X main.version=v${version}"
];

preCheck = ''
export CGO_ENABLED=1
'';

checkPhase = ''
runHook preCheck
go test -race ./...
runHook postCheck
'';


meta.mainProgram = "service-mesh";
};

cli-release = (nunki.override (prevArgs: {
ldflags = prevArgs.ldflags ++ [
"-X main.DefaultCoordinatorPolicyHash=${builtins.readFile ../cli/assets/coordinator-policy-hash}"
Expand All @@ -100,6 +144,15 @@ rec {
};
};

serviceMeshProxy = dockerTools.buildImage {
name = "service-mesh-proxy";
tag = "v${version}";
copyToRoot = [ envoy ];
config = {
Cmd = [ "${service-mesh}/bin/service-mesh" ];
};
};

opensslContainer = dockerTools.buildImage {
name = "openssl";
tag = "v${version}";
Expand All @@ -117,6 +170,8 @@ rec {
push-coordinator = pushContainer coordinator;
push-initializer = pushContainer initializer;

push-service-mesh-proxy = pushContainer serviceMeshProxy;

push-openssl = pushContainer opensslContainer;
push-port-forwarder = pushContainer port-forwarder;

Expand Down Expand Up @@ -150,6 +205,7 @@ rec {
# we only need to update one of them to update the vendorHash
# of the builder.
nix-update --version=skip --flake cli
nix-update --version=skip --flake service-mesh
'';
};

Expand Down Expand Up @@ -178,6 +234,7 @@ rec {
kypatch
opensslContainer
port-forwarder
serviceMeshProxy
];
text = ''
targetPath=$1
Expand All @@ -189,17 +246,20 @@ rec {
gunzip < "${initializer}" > "$tmpdir/initializer.tar"
gunzip < "${opensslContainer}" > "$tmpdir/openssl.tar"
gunzip < "${port-forwarder}" > "$tmpdir/port-forwarder.tar"
gunzip < "${serviceMeshProxy}" > "$tmpdir/service-mesh-proxy.tar"
coordHash=$(crane digest --tarball "$tmpdir/coordinator.tar")
initHash=$(crane digest --tarball "$tmpdir/initializer.tar")
opensslHash=$(crane digest --tarball "$tmpdir/openssl.tar")
forwarderHash=$(crane digest --tarball "$tmpdir/port-forwarder.tar")
serviceMeshProxyHash=$(crane digest --tarball "$tmpdir/service-mesh-proxy.tar")
kypatch images "$targetPath" \
--replace "nunki/coordinator:latest" "nunki/coordinator@$coordHash" \
--replace "nunki/initializer:latest" "nunki/initializer@$initHash" \
--replace "nunki/openssl:latest" "nunki/openssl@$opensslHash" \
--replace "nunki/port-forwarder:latest" "nunki/port-forwarder@$forwarderHash"
--replace "nunki/port-forwarder:latest" "nunki/port-forwarder@$forwarderHash" \
--replace "nunki/service-mesh-proxy:latest" "nunki/service-mesh-proxy@$serviceMeshProxyHash"
'';
};

Expand Down
220 changes: 220 additions & 0 deletions service-mesh/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
package main

import (
"fmt"
"net/netip"
"strings"

envoyConfigBootstrapV3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
envoyConfigClusterV3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoyCoreV3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoyConfigListenerV3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
envoyConfigTCPProxyV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
envoyTLSV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/anypb"
)

var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8")

type proxyConfig []configEntry

type configEntry struct {
name string
listenAddr netip.Addr
port uint16
remoteDomain string
}

// ParseProxyConfig parses the proxy configuration from the given string.
// The configuration is expected to be in the following format:
//
// <name>#<listen-address>:<port>#<remote-domain>##<name>#<listen-address>:<port>#<remote-domain>...
//
// Example:
//
// emoji#127.137.0.1:8080#emoji-svc##voting#127.137.0.2:8080#voting-svc
func ParseProxyConfig(data string) (proxyConfig, error) {
entries := strings.Split(data, "##")
var cfg proxyConfig
for _, entry := range entries {
parts := strings.Split(entry, "#")
if len(parts) != 3 {
return nil, fmt.Errorf("invalid entry: %s", entry)
}
listenAddrPort, err := netip.ParseAddrPort(parts[1])
if err != nil {
return nil, fmt.Errorf("invalid listen address: %s", parts[1])
}

if !loopbackCIDR.Contains(listenAddrPort.Addr()) {
return nil, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR)
}
remoteDomain := parts[2]
cfg = append(cfg, configEntry{
name: parts[0],
listenAddr: listenAddrPort.Addr(),
port: listenAddrPort.Port(),
remoteDomain: remoteDomain,
})
}
return cfg, nil
}

// ToEnvoyConfig converts the proxy configuration to an Envoy configuration.
// Reference: https://github.com/solo-io/envoy-operator/blob/master/pkg/kube/config.go
func (c proxyConfig) ToEnvoyConfig() ([]byte, error) {
config := &envoyConfigBootstrapV3.Bootstrap{
StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{},
}
listeners := make([]*envoyConfigListenerV3.Listener, 0, len(c))
clusters := make([]*envoyConfigClusterV3.Cluster, 0, len(c))
for _, entry := range c {
listener, err := listener(entry)
if err != nil {
return nil, err
}
listeners = append(listeners, listener)
cluster, err := cluster(entry)
if err != nil {
return nil, err
}
clusters = append(clusters, cluster)
}
config.StaticResources.Listeners = listeners
config.StaticResources.Clusters = clusters

if err := config.ValidateAll(); err != nil {
return nil, err
}

configBytes, err := protojson.Marshal(config)
if err != nil {
return nil, err
}

return configBytes, nil
}

func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) {
proxy := &envoyConfigTCPProxyV3.TcpProxy{
StatPrefix: entry.name,
ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{
Cluster: entry.name,
},
}

proxyAny, err := anypb.New(proxy)
if err != nil {
return nil, err
}

return &envoyConfigListenerV3.Listener{
Name: entry.name,
Address: &envoyCoreV3.Address{
Address: &envoyCoreV3.Address_SocketAddress{
SocketAddress: &envoyCoreV3.SocketAddress{
Address: entry.listenAddr.String(),
PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{
PortValue: uint32(entry.port),
},
},
},
},
FilterChains: []*envoyConfigListenerV3.FilterChain{
{
Filters: []*envoyConfigListenerV3.Filter{
{
Name: "envoy.filters.network.tcp_proxy",
ConfigType: &envoyConfigListenerV3.Filter_TypedConfig{
TypedConfig: proxyAny,
},
},
},
},
},
}, nil
}

func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) {
socket, err := tlsTransportSocket()
if err != nil {
return nil, err
}

return &envoyConfigClusterV3.Cluster{
Name: entry.name,
ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{
Type: envoyConfigClusterV3.Cluster_LOGICAL_DNS,
},
DnsLookupFamily: envoyConfigClusterV3.Cluster_V4_ONLY,
LoadAssignment: &endpointv3.ClusterLoadAssignment{
ClusterName: entry.name,
Endpoints: []*endpointv3.LocalityLbEndpoints{
{
LbEndpoints: []*endpointv3.LbEndpoint{
{
HostIdentifier: &endpointv3.LbEndpoint_Endpoint{
Endpoint: &endpointv3.Endpoint{
Address: &envoyCoreV3.Address{
Address: &envoyCoreV3.Address_SocketAddress{
SocketAddress: &envoyCoreV3.SocketAddress{
Address: entry.remoteDomain,
PortSpecifier: &envoyCoreV3.SocketAddress_PortValue{
PortValue: uint32(entry.port),
},
},
},
},
},
},
},
},
},
},
},
TransportSocket: socket,
}, nil
}

func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) {
tls := &envoyTLSV3.UpstreamTlsContext{
CommonTlsContext: &envoyTLSV3.CommonTlsContext{
TlsCertificates: []*envoyTLSV3.TlsCertificate{
{
PrivateKey: &envoyCoreV3.DataSource{
Specifier: &envoyCoreV3.DataSource_Filename{
Filename: "/tls-config/key.pem",
},
},
CertificateChain: &envoyCoreV3.DataSource{
Specifier: &envoyCoreV3.DataSource_Filename{
Filename: "/tls-config/certChain.pem",
},
},
},
},
ValidationContextType: &envoyTLSV3.CommonTlsContext_ValidationContext{
ValidationContext: &envoyTLSV3.CertificateValidationContext{
TrustedCa: &envoyCoreV3.DataSource{
Specifier: &envoyCoreV3.DataSource_Filename{
Filename: "/tls-config/MeshCACert.pem",
},
},
},
},
},
}
tlsAny, err := anypb.New(tls)
if err != nil {
return nil, err
}

return &envoyCoreV3.TransportSocket{
Name: "envoy.transport_sockets.tls",
ConfigType: &envoyCoreV3.TransportSocket_TypedConfig{
TypedConfig: tlsAny,
},
}, nil
}
17 changes: 17 additions & 0 deletions service-mesh/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
module github.com/edgelesssys/nunki/service-mesh

go 1.21

require (
github.com/envoyproxy/go-control-plane v0.12.0
google.golang.org/protobuf v1.32.0
)

require (
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
)
Loading

0 comments on commit 0c824c9

Please sign in to comment.