Skip to content

Commit

Permalink
service-mesh: make clientAuth ingress configurable
Browse files Browse the repository at this point in the history
  • Loading branch information
3u13r committed Mar 28, 2024
1 parent 72394bc commit f362879
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 30 deletions.
22 changes: 19 additions & 3 deletions deployments/emojivoto-sm-ingress/web.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,31 @@ spec:
volumeMounts:
- name: tls-certs
mountPath: /tls-config
- name: sidecar
image: "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest"
restartPolicy: Always
volumeMounts:
- name: tls-certs
mountPath: /tls-config
env:
- name: EDG_INGRESS_PROXY_CONFIG
value: "web#8080#false"
- name: EDG_EGRESS_PROXY_CONFIG
value: "emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080"
securityContext:
privileged: true
capabilities:
add:
- NET_ADMIN
serviceAccountName: web
containers:
- env:
- name: WEB_PORT
value: "8080"
- name: EMOJISVC_HOST
value: emoji-svc:8080
value: 127.137.0.1:8081
- name: VOTINGSVC_HOST
value: voting-svc:8080
value: 127.137.0.2:8081
- name: INDEX_BUNDLE
value: dist/index_bundle.js
- name: EDG_CERT_PATH
Expand All @@ -54,7 +70,7 @@ spec:
value: /tls-config/key.pem
- name: EDG_DISABLE_CLIENT_AUTH
value: "true"
image: ghcr.io/3u13r/emojivoto-web:coco-1
image: docker.l5d.io/buoyantio/emojivoto-web:v11
name: web-svc
ports:
- containerPort: 8080
Expand Down
88 changes: 67 additions & 21 deletions service-mesh/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,23 @@ import (
var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8")

// ProxyConfig represents the configuration for the proxy.
type ProxyConfig []configEntry

type configEntry struct {
type ProxyConfig struct {
egress []egressConfigEntry
ingress []ingressConfigEntry
}
type egressConfigEntry struct {
name string
clusterName string
listenAddr netip.Addr
listenPort uint16
remoteDomain string
remotePort uint16
}
type ingressConfigEntry struct {
name string
listenPort uint16
disableTLS bool
}

// ParseProxyConfig parses the proxy configuration from the given string.
// The configuration is expected to be in the following format:
Expand All @@ -41,42 +49,72 @@ type configEntry struct {
// Example:
//
// emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080
func ParseProxyConfig(data string) (ProxyConfig, error) {
entries := strings.Split(data, "##")
func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) {
if ingressConfig == "" && egressConfig == "" {
return ProxyConfig{}, nil
}

entries := strings.Split(egressConfig, "##")
var cfg ProxyConfig
for _, entry := range entries {
if entry == "" {
continue
}
parts := strings.Split(entry, "#")
if len(parts) != 3 {
return nil, fmt.Errorf("invalid entry: %s", entry)
return ProxyConfig{}, fmt.Errorf("invalid entry: %s", entry)
}
listenAddrPort, err := netip.ParseAddrPort(parts[1])
if err != nil {
return nil, fmt.Errorf("invalid listen address: %s", parts[1])
return ProxyConfig{}, fmt.Errorf("invalid listen address: %s", parts[1])
}

if !loopbackCIDR.Contains(listenAddrPort.Addr()) {
return nil, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR)
return ProxyConfig{}, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR)
}
remoteDomain := parts[2]
remoteDomain, remotePort, err := net.SplitHostPort(remoteDomain)
if err != nil {
return nil, fmt.Errorf("invalid remote domain: %s", remoteDomain)
return ProxyConfig{}, fmt.Errorf("invalid remote domain: %s", remoteDomain)
}
remotePortInt, err := strconv.Atoi(remotePort)
if err != nil {
return nil, fmt.Errorf("invalid remote port: %s", remotePort)
return ProxyConfig{}, fmt.Errorf("invalid remote port: %s", remotePort)
}
cfg = append(cfg, configEntry{
cfg.egress = append(cfg.egress, egressConfigEntry{
name: parts[0],
clusterName: parts[0],
listenAddr: listenAddrPort.Addr(),
listenPort: listenAddrPort.Port(),
remotePort: uint16(remotePortInt),
remoteDomain: remoteDomain,
})
}

for _, entry := range strings.Split(ingressConfig, "##") {
if entry == "" {
continue
}
parts := strings.Split(entry, "#")
if len(parts) != 3 {
return ProxyConfig{}, fmt.Errorf("invalid entry: %s", entry)
}
listenPort, err := strconv.Atoi(parts[1])
if err != nil {
return ProxyConfig{}, fmt.Errorf("invalid listen port: %s", parts[1])
}
disableTLS, err := strconv.ParseBool(parts[2])
if err != nil {
return ProxyConfig{}, fmt.Errorf("invalid disable TLS: %s", parts[2])
}
cfg.ingress = append(cfg.ingress, ingressConfigEntry{
name: parts[0],
listenPort: uint16(listenPort),
disableTLS: disableTLS,
})

}

return cfg, nil
}

Expand All @@ -86,9 +124,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) {
config := &envoyConfigBootstrapV3.Bootstrap{
StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{},
}
listeners := make([]*envoyConfigListenerV3.Listener, 0, len(c))
clusters := make([]*envoyConfigClusterV3.Cluster, 0, len(c))
for _, entry := range c {
listeners := make([]*envoyConfigListenerV3.Listener, 0)
clusters := make([]*envoyConfigClusterV3.Cluster, 0)

// Create listeners and clusters for egress traffic.
for _, entry := range c.egress {
listener, err := listener(entry)
if err != nil {
return nil, err
Expand All @@ -106,6 +146,10 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) {
if err != nil {
return nil, err
}
ingrListenerNoClientAuth, err := ingressListener("ingressWithoutClientAuth", 15007, false)
if err != nil {
return nil, err
}

ingressCluster := &envoyConfigClusterV3.Cluster{
Name: "ingress",
Expand All @@ -115,6 +159,7 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) {
}

listeners = append(listeners, ingrListenerClientAuth)
listeners = append(listeners, ingrListenerNoClientAuth)
clusters = append(clusters, ingressCluster)

config.StaticResources.Listeners = listeners
Expand All @@ -132,11 +177,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) {
return configBytes, nil
}

func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) {
func listener(entry egressConfigEntry) (*envoyConfigListenerV3.Listener, error) {
proxy := &envoyConfigTCPProxyV3.TcpProxy{
StatPrefix: entry.name,
ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{
Cluster: entry.name,
Cluster: entry.clusterName,
},
}

Expand Down Expand Up @@ -172,7 +217,7 @@ func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) {
}, nil
}

func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) {
func cluster(entry egressConfigEntry) (*envoyConfigClusterV3.Cluster, error) {
socket, err := upstreamTLSTransportSocket()
if err != nil {
return nil, err
Expand Down Expand Up @@ -214,10 +259,11 @@ func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) {
}

func ingressListener(name string, listenPort uint16, requireClientCertificate bool) (*envoyConfigListenerV3.Listener, error) {
ingressListener, err := listener(configEntry{
name: name,
listenAddr: netip.MustParseAddr("0.0.0.0"),
listenPort: listenPort,
ingressListener, err := listener(egressConfigEntry{
name: name,
clusterName: "ingress",
listenAddr: netip.MustParseAddr("0.0.0.0"),
listenPort: listenPort,
})
if err != nil {
return nil, err
Expand Down
14 changes: 13 additions & 1 deletion service-mesh/iptables.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const EnvoyIngressPort = 15006
const EnvoyIngressPortNoClientCert = 15007

// IngressIPTableRules sets up the iptables rules for the ingress proxy.
func IngressIPTableRules() error {
func IngressIPTableRules(ingressEntries []ingressConfigEntry) error {
// Create missing `/run/xtables.lock` file.
if err := os.Mkdir("/run", 0o755); err != nil {
if !os.IsExist(err) {
Expand Down Expand Up @@ -67,6 +67,18 @@ func IngressIPTableRules() error {
return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to EDG_INBOUND chain: %w", err)
}

for _, entry := range ingressEntries {
if entry.disableTLS {
if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "--dport", fmt.Sprintf("%d", entry.listenPort), "-j", "RETURN"); err != nil {
return fmt.Errorf("failed to append dport exception to EDG_IN_REDIRECT chain to disable TLS: %w", err)
}
} else {
if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "--dport", fmt.Sprintf("%d", entry.listenPort), "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPortNoClientCert)); err != nil {
return fmt.Errorf("failed to append dport exception to EDG_IN_REDIRECT chain to disable client auth: %w", err)
}
}
}

// Route all traffic not destined for 127.0.0.1 to the envoy proxy on its
// port that requires client authentication.
if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPort)); err != nil {
Expand Down
15 changes: 10 additions & 5 deletions service-mesh/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@ import (
)

const (
proxyConfigEnvVar = "EDG_PROXY_CONFIG"
envoyConfigFile = "/envoy-config.yml"
egressProxyConfigEnvVar = "EDG_EGRESS_PROXY_CONFIG"
ingressProxyConfigEnvVar = "EDG_INGRESS_PROXY_CONFIG"
envoyConfigFile = "/envoy-config.yml"
)

var version = "0.0.0-dev"
Expand All @@ -25,9 +26,13 @@ func main() {
func run() (retErr error) {
log.Printf("service-mesh version %s\n", version)

proxyConfig := os.Getenv(proxyConfigEnvVar)
egressProxyConfig := os.Getenv(egressProxyConfigEnvVar)
log.Println("Ingress Proxy configuration:", egressProxyConfig)

pconfig, err := ParseProxyConfig(proxyConfig)
ingressProxyConfig := os.Getenv(ingressProxyConfigEnvVar)
log.Println("Egress Proxy configuration:", ingressProxyConfig)

pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig)
if err != nil {
return err
}
Expand All @@ -43,7 +48,7 @@ func run() (retErr error) {
return err
}

if err := IngressIPTableRules(); err != nil {
if err := IngressIPTableRules(pconfig.ingress); err != nil {
return fmt.Errorf("failed to set up iptables rules: %w", err)
}

Expand Down

0 comments on commit f362879

Please sign in to comment.