Add watcher for upstreams TLS certificates (#716)

* Add watcher to TLS certificates Signed-off-by: Ruben Vargas <[email protected]> * Fix CA loading logic Signed-off-by: Ruben Vargas <[email protected]> * Add some comments, change parameters order Signed-off-by: Ruben Vargas <[email protected]> * Fix some comments and clarify some names Signed-off-by: Ruben Vargas <[email protected]> --------- Signed-off-by: Ruben Vargas <[email protected]>
observatorium · Jul 17, 2024 · aa210f8 · aa210f8
1 parent f50a799
commit aa210f8
Show file tree

Hide file tree

Showing 13 changed files with 772 additions and 121 deletions.
diff --git a/README.md b/README.md
@@ -113,6 +113,8 @@ Usage of ./observatorium-api:
     	File containing the TLS client certificates to authenticate against upstream logs servers. Leave blank to disable mTLS.
   -logs.tls.key-file string
     	File containing the TLS client key to authenticate against upstream logs servers. Leave blank to disable mTLS.
+  -logs.tls.watch-certs
+    	Watch for certificate changes and reload
   -logs.write-timeout duration
     	The HTTP write timeout for proxied requests to the logs endpoint. (default 10m0s)
   -logs.write.endpoint string
@@ -133,6 +135,8 @@ Usage of ./observatorium-api:
     	File containing the TLS client certificates to authenticate against upstream logs servers. Leave blank to disable mTLS.
   -metrics.tls.key-file string
     	File containing the TLS client key to authenticate against upstream metrics servers. Leave blank to disable mTLS.
+  -metrics.tls.watch-certs
+    	Watch for certificate changes and reload
   -metrics.write-timeout duration
     	The HTTP write timeout for proxied requests to the metrics endpoint. (default 2m0s)
   -metrics.write.endpoint string
@@ -193,6 +197,8 @@ Usage of ./observatorium-api:
     	File containing the TLS client certificates to authenticate against upstream logs servers. Leave blank to disable mTLS.
   -traces.tls.key-file string
     	File containing the TLS client key to authenticate against upstream traces servers. Leave blank to disable mTLS.
+  -traces.tls.watch-certs
+    	Watch for certificate changes and reload
   -traces.write-timeout duration
     	The HTTP write timeout for proxied requests to the traces endpoint. (default 2m0s)
   -traces.write.otlpgrpc.endpoint string

diff --git a/api/logs/v1/http.go b/api/logs/v1/http.go
@@ -2,7 +2,6 @@
 package http
 
 import (
-	stdtls "crypto/tls"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -147,7 +146,7 @@ func (n nopInstrumentHandler) NewHandler(labels prometheus.Labels, handler http.
 	return handler.ServeHTTP
 }
 
-func NewHandler(read, tail, write, rules *url.URL, rulesReadOnly bool, upstreamCA []byte, upstreamCert *stdtls.Certificate, opts ...HandlerOption) http.Handler {
+func NewHandler(read, tail, write, rules *url.URL, rulesReadOnly bool, tlsOptions *tls.UpstreamOptions, opts ...HandlerOption) http.Handler {
 	c := &handlerConfiguration{
 		logger:     log.NewNopLogger(),
 		registry:   prometheus.NewRegistry(),
@@ -174,7 +173,7 @@ func NewHandler(read, tail, write, rules *url.URL, rulesReadOnly bool, upstreamC
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyRead = &httputil.ReverseProxy{
@@ -250,7 +249,7 @@ func NewHandler(read, tail, write, rules *url.URL, rulesReadOnly bool, upstreamC
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyReadRules = &httputil.ReverseProxy{
@@ -350,7 +349,7 @@ func NewHandler(read, tail, write, rules *url.URL, rulesReadOnly bool, upstreamC
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			tailRead = &httputil.ReverseProxy{
@@ -386,7 +385,7 @@ func NewHandler(read, tail, write, rules *url.URL, rulesReadOnly bool, upstreamC
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyWrite = &httputil.ReverseProxy{

diff --git a/api/metrics/legacy/http.go b/api/metrics/legacy/http.go
@@ -1,7 +1,6 @@
 package legacy
 
 import (
-	stdtls "crypto/tls"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -102,7 +101,7 @@ func (n nopInstrumentHandler) NewHandler(_ prometheus.Labels, handler http.Handl
 	return handler.ServeHTTP
 }
 
-func NewHandler(url *url.URL, upstreamCA []byte, upstreamCert *stdtls.Certificate, opts ...HandlerOption) http.Handler {
+func NewHandler(url *url.URL, tlsOptions *tls.UpstreamOptions, opts ...HandlerOption) http.Handler {
 	c := &handlerConfiguration{
 		logger:     log.NewNopLogger(),
 		registry:   prometheus.NewRegistry(),
@@ -130,7 +129,7 @@ func NewHandler(url *url.URL, upstreamCA []byte, upstreamCert *stdtls.Certificat
 			DialContext: (&net.Dialer{
 				Timeout: dialTimeout,
 			}).DialContext,
-			TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+			TLSClientConfig: tlsOptions.NewClientConfig(),
 		}
 
 		legacyProxy = &httputil.ReverseProxy{

diff --git a/api/metrics/v1/http.go b/api/metrics/v1/http.go
@@ -1,7 +1,6 @@
 package v1
 
 import (
-	stdtls "crypto/tls"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -174,7 +173,7 @@ type Endpoints struct {
 
 // NewHandler creates the new metrics v1 handler.
 // nolint:funlen
-func NewHandler(endpoints Endpoints, upstreamCA []byte, upstreamCert *stdtls.Certificate, opts ...HandlerOption) http.Handler {
+func NewHandler(endpoints Endpoints, tlsOptions *tls.UpstreamOptions, opts ...HandlerOption) http.Handler {
 	c := &handlerConfiguration{
 		logger:     log.NewNopLogger(),
 		registry:   prometheus.NewRegistry(),
@@ -258,7 +257,7 @@ func NewHandler(endpoints Endpoints, upstreamCA []byte, upstreamCert *stdtls.Cer
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyRead = &httputil.ReverseProxy{
@@ -345,7 +344,7 @@ func NewHandler(endpoints Endpoints, upstreamCA []byte, upstreamCert *stdtls.Cer
 			)
 
 			t := http.DefaultTransport.(*http.Transport)
-			t.TLSClientConfig = tls.NewClientConfig(upstreamCA, upstreamCert)
+			t.TLSClientConfig = tlsOptions.NewClientConfig()
 
 			uiProxy = &httputil.ReverseProxy{
 				Director:  middlewares,
@@ -384,7 +383,7 @@ func NewHandler(endpoints Endpoints, upstreamCA []byte, upstreamCert *stdtls.Cer
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyWrite = &httputil.ReverseProxy{
@@ -469,7 +468,7 @@ func NewHandler(endpoints Endpoints, upstreamCA []byte, upstreamCert *stdtls.Cer
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyAlertmanager = &httputil.ReverseProxy{

diff --git a/api/traces/v1/api.go b/api/traces/v1/api.go
@@ -2,7 +2,6 @@ package v1
 
 import (
 	"context"
-	stdtls "crypto/tls"
 	"time"
 
 	"github.com/go-kit/log"
@@ -18,9 +17,8 @@ import (
 const TraceRoute = "/opentelemetry.proto.collector.trace.v1.TraceService/Export"
 
 type connOptions struct {
-	logger             log.Logger
-	tracesUpstreamCert *stdtls.Certificate
-	tracesUpstreamCA   []byte
+	logger     log.Logger
+	tlsOptions *tls.UpstreamOptions
 }
 
 // ClientOption modifies the connection's configuration.
@@ -33,15 +31,14 @@ func WithLogger(logger log.Logger) ClientOption {
 	}
 }
 
-func WithUpstreamTLS(tracesUpstreamCA []byte, tracesUpstreamCert *stdtls.Certificate) ClientOption {
+func WithUpstreamTLSOptions(tlsOptions *tls.UpstreamOptions) ClientOption {
 	return func(h *connOptions) {
-		h.tracesUpstreamCA = tracesUpstreamCA
-		h.tracesUpstreamCert = tracesUpstreamCert
+		h.tlsOptions = tlsOptions
 	}
 }
 
-func newCredentials(upstreamCA []byte, upstreamCert *stdtls.Certificate) credentials.TransportCredentials {
-	tlsConfig := tls.NewClientConfig(upstreamCA, upstreamCert)
+func newCredentials(tlsOptions *tls.UpstreamOptions) credentials.TransportCredentials {
+	tlsConfig := tlsOptions.NewClientConfig()
 	if tlsConfig == nil {
 		return insecure.NewCredentials()
 	}
@@ -70,5 +67,5 @@ func NewOTelConnection(write string, opts ...ClientOption) (*grpc.ClientConn, er
 		// because the codec we need to register is also deprecated.  A better fix, is the newer
 		// version of mwitkow/grpc-proxy, but that version doesn't (currently) work with OTel protocol.
 		grpc.WithCodec(grpcproxy.Codec()), // nolint: staticcheck
-		grpc.WithTransportCredentials(newCredentials(c.tracesUpstreamCA, c.tracesUpstreamCert)))
+		grpc.WithTransportCredentials(newCredentials(c.tlsOptions)))
 }
diff --git a/api/traces/v1/http.go b/api/traces/v1/http.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
-	stdtls "crypto/tls"
 	"fmt"
 	"io"
 	"net"
@@ -109,7 +108,7 @@ func (n nopInstrumentHandler) NewHandler(labels prometheus.Labels, handler http.
 // The web UI handler is able to rewrite
 // HTML to change the <base> attribute so that it works with the Observatorium-style
 // "/api/v1/traces/{tenant}/" URLs.
-func NewV2Handler(read *url.URL, readTemplate string, tempo *url.URL, writeOTLPHttp *url.URL, upstreamCA []byte, upstreamCert *stdtls.Certificate, opts ...HandlerOption) http.Handler {
+func NewV2Handler(read *url.URL, readTemplate string, tempo *url.URL, writeOTLPHttp *url.URL, tlsOptions *tls.UpstreamOptions, opts ...HandlerOption) http.Handler {
 
 	if read == nil && readTemplate == "" && tempo == nil {
 		panic("missing Jaeger read url")
@@ -152,7 +151,7 @@ func NewV2Handler(read *url.URL, readTemplate string, tempo *url.URL, writeOTLPH
 				DialContext: (&net.Dialer{
 					Timeout: dialTimeout,
 				}).DialContext,
-				TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+				TLSClientConfig: tlsOptions.NewClientConfig(),
 			}
 
 			proxyRead = &httputil.ReverseProxy{
@@ -203,7 +202,7 @@ func NewV2Handler(read *url.URL, readTemplate string, tempo *url.URL, writeOTLPH
 			DialContext: (&net.Dialer{
 				Timeout: dialTimeout,
 			}).DialContext,
-			TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+			TLSClientConfig: tlsOptions.NewClientConfig(),
 		}
 
 		proxyOTLP := &httputil.ReverseProxy{
@@ -229,7 +228,7 @@ func NewV2Handler(read *url.URL, readTemplate string, tempo *url.URL, writeOTLPH
 			DialContext: (&net.Dialer{
 				Timeout: dialTimeout,
 			}).DialContext,
-			TLSClientConfig: tls.NewClientConfig(upstreamCA, upstreamCert),
+			TLSClientConfig: tlsOptions.NewClientConfig(),
 		}
 
 		middlewares := proxy.Middlewares(

diff --git a/go.mod b/go.mod
@@ -35,6 +35,7 @@ require (
 	github.com/prometheus/common v0.53.0
 	github.com/prometheus/prometheus v0.50.1
 	github.com/redis/rueidis v1.0.37
+	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0
 	go.opentelemetry.io/contrib/propagators/jaeger v1.28.0
 	go.opentelemetry.io/otel v1.28.0
@@ -159,7 +160,6 @@ require (
 	github.com/schollz/closestmatch v2.1.0+incompatible // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
 	github.com/tdewolff/minify/v2 v2.12.9 // indirect
 	github.com/tdewolff/parse/v2 v2.6.8 // indirect