Split metrics endpoint to a different port, default 8081 (Azure#562)

* Split metrics endpoint to a different port, default 8081 This will allow metrics to be scraped over HTTP while the application itself behind port 8443 is secured by Istio mTLS --------- Signed-off-by: Michael Shen <[email protected]>
ahitacat · Sep 4, 2024 · 8453d26 · 8453d26
1 parent 4cd03b2
commit 8453d26
Show file tree

Hide file tree

Showing 10 changed files with 51 additions and 17 deletions.
diff --git a/frontend/cmd/cmd.go b/frontend/cmd/cmd.go
@@ -27,8 +27,9 @@ type FrontendOpts struct {
 	clusterServiceNoopDeprovision bool
 	insecure                      bool
 
-	location string
-	port     int
+	location    string
+	metricsPort int
+	port        int
 
 	useCache   bool
 	cosmosName string
@@ -60,6 +61,7 @@ func NewRootCmd() *cobra.Command {
 	rootCmd.Flags().StringVar(&opts.cosmosURL, "cosmos-url", os.Getenv("DB_URL"), "Cosmos database url")
 	rootCmd.Flags().StringVar(&opts.location, "location", os.Getenv("LOCATION"), "Azure location")
 	rootCmd.Flags().IntVar(&opts.port, "port", 8443, "port to listen on")
+	rootCmd.Flags().IntVar(&opts.metricsPort, "metrics-port", 8081, "port to serve metrics on")
 
 	rootCmd.Flags().StringVar(&opts.clustersServiceURL, "clusters-service-url", "https://api.openshift.com", "URL of the OCM API gateway.")
 	rootCmd.Flags().BoolVar(&opts.insecure, "insecure", false, "Skip validating TLS for clusters-service.")
@@ -98,6 +100,11 @@ func (opts *FrontendOpts) Run() error {
 		return err
 	}
 
+	metricsListener, err := net.Listen("tcp4", fmt.Sprintf(":%d", opts.metricsPort))
+	if err != nil {
+		return err
+	}
+
 	// Initialize Clusters Service Client
 	conn, err := sdk.NewUnauthenticatedConnectionBuilder().
 		URL(opts.clustersServiceURL).
@@ -122,7 +129,7 @@ func (opts *FrontendOpts) Run() error {
 	}
 	logger.Info(fmt.Sprintf("Application running in %s", opts.location))
 
-	f := frontend.NewFrontend(logger, listener, prometheusEmitter, dbClient, opts.location, csCfg)
+	f := frontend.NewFrontend(logger, listener, metricsListener, prometheusEmitter, dbClient, opts.location, csCfg)
 
 	stop := make(chan struct{})
 	signalChannel := make(chan os.Signal, 1)

diff --git a/frontend/deploy/base/deployment.yml b/frontend/deploy/base/deployment.yml
@@ -47,6 +47,8 @@ spec:
           ports:
             - containerPort: 8443
               protocol: TCP
+            - containerPort: 8081
+              protocol: TCP
           resources:
             limits:
               memory: 1Gi

diff --git a/frontend/go.mod b/frontend/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/prometheus/client_golang v1.20.2
 	github.com/spf13/cobra v1.8.1
 	golang.org/x/exp v0.0.0-20240707233637-46b078467d37
+	golang.org/x/sync v0.7.0
 )
 
 require github.com/klauspost/compress v1.17.9 // indirect

diff --git a/frontend/go.sum b/frontend/go.sum
@@ -165,6 +165,7 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

diff --git a/frontend/pkg/frontend/frontend.go b/frontend/pkg/frontend/frontend.go
@@ -22,6 +22,7 @@ import (
 	"github.com/google/uuid"
 	cmv2alpha1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v2alpha1"
 	ocmerrors "github.com/openshift-online/ocm-sdk-go/errors"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/Azure/ARO-HCP/internal/api"
 	"github.com/Azure/ARO-HCP/internal/api/arm"
@@ -33,32 +34,42 @@ type Frontend struct {
 	clusterServiceConfig ocm.ClusterServiceConfig
 	logger               *slog.Logger
 	listener             net.Listener
+	metricsListener      net.Listener
 	server               http.Server
+	metricsServer        http.Server
 	dbClient             database.DBClient
 	ready                atomic.Value
 	done                 chan struct{}
 	metrics              Emitter
 	location             string
 }
 
-func NewFrontend(logger *slog.Logger, listener net.Listener, emitter Emitter, dbClient database.DBClient, location string, csCfg ocm.ClusterServiceConfig) *Frontend {
+func NewFrontend(logger *slog.Logger, listener net.Listener, metricsListener net.Listener, emitter Emitter, dbClient database.DBClient, location string, csCfg ocm.ClusterServiceConfig) *Frontend {
 	f := &Frontend{
 		clusterServiceConfig: csCfg,
 		logger:               logger,
 		listener:             listener,
+		metricsListener:      metricsListener,
 		metrics:              emitter,
 		server: http.Server{
 			ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
 			BaseContext: func(net.Listener) context.Context {
 				return ContextWithLogger(context.Background(), logger)
 			},
 		},
+		metricsServer: http.Server{
+			ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError),
+			BaseContext: func(net.Listener) context.Context {
+				return ContextWithLogger(context.Background(), logger)
+			},
+		},
 		dbClient: dbClient,
 		done:     make(chan struct{}),
 		location: strings.ToLower(location),
 	}
 
 	f.server.Handler = f.routes()
+	f.metricsServer.Handler = f.metricsRoutes()
 
 	return f
 }
@@ -69,13 +80,23 @@ func (f *Frontend) Run(ctx context.Context, stop <-chan struct{}) {
 			<-stop
 			f.ready.Store(false)
 			_ = f.server.Shutdown(ctx)
+			_ = f.metricsServer.Shutdown(ctx)
 		}()
 	}
 
 	f.logger.Info(fmt.Sprintf("listening on %s", f.listener.Addr().String()))
+	f.logger.Info(fmt.Sprintf("metrics listening on %s", f.metricsListener.Addr().String()))
 	f.ready.Store(true)
 
-	if err := f.server.Serve(f.listener); !errors.Is(err, http.ErrServerClosed) {
+	errs, ctx := errgroup.WithContext(ctx)
+	errs.Go(func() error {
+		return f.server.Serve(f.listener)
+	})
+	errs.Go(func() error {
+		return f.metricsServer.Serve(f.metricsListener)
+	})
+
+	if err := errs.Wait(); !errors.Is(err, http.ErrServerClosed) {
 		f.logger.Error(err.Error())
 		os.Exit(1)
 	}

diff --git a/frontend/pkg/frontend/routes.go b/frontend/pkg/frontend/routes.go
@@ -55,7 +55,6 @@ func (f *Frontend) routes() *MiddlewareMux {
 	// Unauthenticated routes
 	mux.HandleFunc("/", f.NotFound)
 	mux.HandleFunc(MuxPattern(http.MethodGet, "healthz"), f.Healthz)
-	mux.Handle(MuxPattern(http.MethodGet, "metrics"), promhttp.Handler())
 
 	// List endpoints
 	postMuxMiddleware := NewMiddleware(
@@ -130,3 +129,10 @@ func (f *Frontend) routes() *MiddlewareMux {
 
 	return mux
 }
+
+func (f *Frontend) metricsRoutes() *http.ServeMux {
+	mux := http.NewServeMux()
+	mux.Handle("GET /metrics", promhttp.Handler())
+
+	return mux
+}
diff --git a/internal/go.mod b/internal/go.mod
@@ -44,7 +44,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_golang v1.20.2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

diff --git a/internal/go.sum b/internal/go.sum
@@ -85,6 +85,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -117,8 +118,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=

diff --git a/tooling/image-sync/go.mod b/tooling/image-sync/go.mod
@@ -5,7 +5,7 @@ go 1.22.0
 toolchain go1.22.2
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/containers/azcontainerregistry v0.2.1
 	github.com/containers/image/v5 v5.32.0
@@ -22,7 +22,6 @@ require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
 	github.com/containers/ocicrypt v1.2.0 // indirect
 	github.com/containers/storage v1.55.0 // indirect
@@ -76,9 +75,8 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/proglottis/gpgme v0.1.3 // indirect
-	github.com/prometheus/client_golang v1.19.1 // indirect
+	github.com/prometheus/client_golang v1.20.2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.6.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

diff --git a/tooling/image-sync/go.sum b/tooling/image-sync/go.sum
@@ -1,7 +1,6 @@
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
 github.com/Azure/azure-sdk-for-go/sdk/containers/azcontainerregistry v0.2.1 h1:Rj6ScDn/5amy1qlQwodwbh+eqXjlopD0LpS6TuN8qcU=
@@ -162,8 +161,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0=
 github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0=
-github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
-github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=