telemetry: add direct to otel logging

logging to stdout is now disabled if it is determined that os.Stdout
is not a tty/terminal and will only log into a configured otel logger.
old logging is the same if stdout is connected.

website: add grafana routing to the admin panel

cmd/hanyuu/main.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io"
+	"log"
 	"os"
 	"os/signal"
 	"runtime/debug"
@@ -22,6 +24,7 @@ import (
 	"github.com/R-a-dio/valkyrie/website"
 	"github.com/google/subcommands"
 	"github.com/rs/zerolog"
+	"golang.org/x/term"
 )
 
 type executeFn func(context.Context, config.Loader) error
@@ -53,6 +56,7 @@ func (c cmd) Execute(ctx context.Context, f *flag.FlagSet, args ...interface{})
 	zerolog.Ctx(ctx).UpdateContext(func(zc zerolog.Context) zerolog.Context {
 		return zc.Str("service", c.name)
 	})
+
 	// setup telemetry if wanted
 
 	var telemetryShutdown func()
@@ -77,6 +81,7 @@ func (c cmd) Execute(ctx context.Context, f *flag.FlagSet, args ...interface{})
 		if err != nil {
 			zerolog.Ctx(ctx).Error().Err(err).Msg("failed to initialize telemetry")
 		}
+
 		return cfg, err
 	}
 
@@ -255,14 +260,20 @@ func main() {
 	// exit code passed to os.Exit
 	var code int
 	// setup logger
-	logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stdout}).With().Timestamp().Logger()
+
+	// discard logs unless we are connected to a terminal
+	lo := io.Discard
+	if term.IsTerminal(int(os.Stdout.Fd())) {
+		lo = zerolog.ConsoleWriter{Out: os.Stdout}
+	}
+	logger := zerolog.New(lo).With().Timestamp().Logger()
 	// change the level to what the flag told us
 	level, err := zerolog.ParseLevel(logLevel)
 	if err != nil {
 		logger.Error().Err(err).Msg("failed to parse loglevel flag")
 		os.Exit(1)
 	}
-	logger = logger.Level(level)
+	logger = logger.Level(level).Hook(telemetry.Hook)
 
 	// setup root context
 	ctx := context.Background()
@@ -289,6 +300,7 @@ func main() {
 		// normal non-nil error, we exit with the default failure exit code
 		code = 1
 		// print the error if it's a non-ExitError since it's probably important
+		log.Println("exit error:", err)
 		logger.Fatal().Err(err).Msg("exit")
 	}
 

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/DATA-DOG/go-sqlmock v1.5.1
 	github.com/XSAM/otelsql v0.27.0
+	github.com/agoda-com/opentelemetry-go/otelzerolog v0.0.1
+	github.com/agoda-com/opentelemetry-logs-go v0.4.3
 	github.com/alevinval/sse v1.0.2
 	github.com/alexedwards/scs/v2 v2.7.0
 	github.com/cenkalti/backoff v2.2.1+incompatible
@@ -17,6 +19,7 @@ require (
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/leanovate/gopter v0.2.9
 	github.com/lrstanley/girc v0.0.0-20240130161140-6cf311abc7b0
+	github.com/prometheus/client_golang v1.19.0
 	github.com/rs/xid v1.5.0
 	github.com/rs/zerolog v1.31.0
 	github.com/spf13/afero v1.11.0
@@ -35,6 +38,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0
 	golang.org/x/crypto v0.18.0
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
+	golang.org/x/term v0.18.0
 	golang.org/x/text v0.14.0
 	golang.org/x/tools v0.17.0
 	google.golang.org/grpc v1.61.0
@@ -78,7 +82,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/prometheus/client_golang v1.19.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
@@ -88,13 +91,12 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -19,6 +19,10 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/XSAM/otelsql v0.27.0 h1:i9xtxtdcqXV768a5C6SoT/RkG+ue3JTOgkYInzlTOqs=
 github.com/XSAM/otelsql v0.27.0/go.mod h1:0mFB3TvLa7NCuhm/2nU7/b2wEtsczkj8Rey8ygO7V+A=
+github.com/agoda-com/opentelemetry-go/otelzerolog v0.0.1 h1:R1FRCOPXI+TefJdE3p6S10vP7R5P45dYJiSfu/xO5oE=
+github.com/agoda-com/opentelemetry-go/otelzerolog v0.0.1/go.mod h1:PtATrdQ3evitYHGwOqirLvxwD1jEk1xFWkITtI1tIcI=
+github.com/agoda-com/opentelemetry-logs-go v0.4.3 h1:dYAx/q9di+/Pv6HuGq59DFIOjqKT0LTy3PYTIz8ccq8=
+github.com/agoda-com/opentelemetry-logs-go v0.4.3/go.mod h1:gPQ0fHqroxNP2DlQFZt29/pfqGiP2m6Q5CCxEgLo6yQ=
 github.com/alevinval/sse v1.0.2 h1:ooc08hn9B5X/u7vOMpnYDkXxIKA0y5DOw9qBVVK3YKY=
 github.com/alevinval/sse v1.0.2/go.mod h1:X4J1/nTNs4yKbvjXFWJB+NdF9gaYkoAC4sw9Z9h7ASk=
 github.com/alexedwards/scs/v2 v2.7.0 h1:DY4rqLCM7UIR9iwxFS0++z1NhTzQlKV30aMHkJCDWKw=
@@ -113,8 +117,6 @@ github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lrstanley/girc v0.0.0-20230911164840-f47717952bf9 h1:Kgp9FtxM8VZr2wDmXhCkd/f2EW5NeXJzZSWMYQB4M4s=
-github.com/lrstanley/girc v0.0.0-20230911164840-f47717952bf9/go.mod h1:lgrnhcF8bg/Bd5HA5DOb4Z+uGqUqGnp4skr+J2GwVgI=
 github.com/lrstanley/girc v0.0.0-20240130161140-6cf311abc7b0 h1:QOm14PCCLhhhEbTzL5dzQ9gNOvfCWpLlhpp/eUabS/k=
 github.com/lrstanley/girc v0.0.0-20240130161140-6cf311abc7b0/go.mod h1:lgrnhcF8bg/Bd5HA5DOb4Z+uGqUqGnp4skr+J2GwVgI=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@@ -201,10 +203,6 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw=
-go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 h1:dg9y+7ArpumB6zwImJv47RHfdgOGQ1EMkzP5vLkEnTU=
-go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0/go.mod h1:Ul4MtXqu/hJBM+v7a6dCF0nHwckPMLpIpLeCi4+zfdw=
-go.opentelemetry.io/otel v1.23.0 h1:Df0pqjqExIywbMCMTxkAwzjLZtRf+bBKLbUcpxO2C9E=
-go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0=
 go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
 go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.23.0 h1:97CpJflo7dJK4A4SLMNoP2loDEAiG0ifF6MnLhtSHUY=
@@ -213,16 +211,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYa
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 h1:H2JFgRcGiyHg7H7bwcwaQJYrNFqCqrbTQ8K4p1OvDu8=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0/go.mod h1:WfCWp1bGoYK8MeULtI15MmQVczfR+bFkk0DF3h06QmQ=
-go.opentelemetry.io/otel/metric v1.23.0 h1:pazkx7ss4LFVVYSxYew7L5I6qvLXHA0Ap2pwV+9Cnpo=
-go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo=
 go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
 go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
 go.opentelemetry.io/otel/sdk v1.23.0 h1:0KM9Zl2esnl+WSukEmlaAEjVY5HDZANOHferLq36BPc=
 go.opentelemetry.io/otel/sdk v1.23.0/go.mod h1:wUscup7byToqyKJSilEtMf34FgdCAsFpFOjXnAwFfO0=
 go.opentelemetry.io/otel/sdk/metric v1.23.0 h1:u81lMvmK6GMgN4Fty7K7S6cSKOZhMKJMK2TB+KaTs0I=
 go.opentelemetry.io/otel/sdk/metric v1.23.0/go.mod h1:2LUOToN/FdX6wtfpHybOnCZjoZ6ViYajJYMiJ1LKDtQ=
-go.opentelemetry.io/otel/trace v1.23.0 h1:37Ik5Ib7xfYVb4V1UtnT97T1jI+AoIYkJyPkuL4iJgI=
-go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk=
 go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
 go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI=
@@ -248,9 +242,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
-golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -269,8 +262,10 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

telemetry/otel.go

@@ -9,6 +9,10 @@ import (
 	"github.com/R-a-dio/valkyrie/storage/mariadb"
 	"github.com/R-a-dio/valkyrie/website"
 	"github.com/XSAM/otelsql"
+	"github.com/agoda-com/opentelemetry-go/otelzerolog"
+	"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs"
+	"github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogsgrpc"
+	logsSDK "github.com/agoda-com/opentelemetry-logs-go/sdk/logs"
 	"github.com/jmoiron/sqlx"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/push"
@@ -25,6 +29,18 @@ import (
 	"google.golang.org/grpc"
 )
 
+// temporary until otel gets log handling
+var LogProvider *logsSDK.LoggerProvider
+var logHook *otelzerolog.Hook
+
+var Hook = zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, message string) {
+	if logHook == nil {
+		return
+	}
+
+	logHook.Run(e, level, message)
+})
+
 func Init(ctx context.Context, cfg config.Config, service string) (func(), error) {
 	tp, err := InitTracer(ctx, cfg, service)
 	if err != nil {
@@ -38,6 +54,14 @@ func Init(ctx context.Context, cfg config.Config, service string) (func(), error
 	}
 	otel.SetMeterProvider(mp)
 
+	lp, err := InitLogs(ctx, cfg, service)
+	if err != nil {
+		return nil, err
+	}
+	// otel.SetLogProvider(lp)
+	LogProvider = lp
+	logHook = otelzerolog.NewHook(lp)
+
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
 	// done setting up, swap global functions to inject telemetry
 	mariadb.DatabaseConnectFunc = DatabaseConnect
@@ -75,6 +99,7 @@ func Init(ctx context.Context, cfg config.Config, service string) (func(), error
 	closeFn := func() {
 		tp.Shutdown(context.Background())
 		mp.Shutdown(context.Background())
+		lp.Shutdown(context.Background())
 	}
 
 	return closeFn, err
@@ -147,6 +172,37 @@ func InitMetric(ctx context.Context, cfg config.Config, service string) (*metric
 	return mp, nil
 }
 
+func InitLogs(ctx context.Context, cfg config.Config, service string) (*logsSDK.LoggerProvider, error) {
+	conf := cfg.Conf().Telemetry
+
+	logs_exporter, err := otlplogs.NewExporter(ctx,
+		otlplogs.WithClient(
+			otlplogsgrpc.NewClient(otlplogsgrpc.WithInsecure(),
+				otlplogsgrpc.WithEndpoint(conf.Endpoint),
+			),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := resource.Merge(resource.Default(), resource.Environment())
+	if err != nil {
+		return nil, err
+	}
+	res, err = resource.Merge(res, resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName("radio:"+service)))
+	if err != nil {
+		return nil, err
+	}
+
+	lp := logsSDK.NewLoggerProvider(
+		logsSDK.WithBatcher(logs_exporter),
+		logsSDK.WithResource(res),
+	)
+
+	return lp, nil
+}
+
 // DatabaseConnect applies telemetry to a database/sql driver
 func DatabaseConnect(ctx context.Context, driverName string, dataSourceName string) (*sqlx.DB, error) {
 	db, err := otelsql.Open(driverName, dataSourceName, otelsql.WithSpanOptions(otelsql.SpanOptions{

website/admin/router.go

@@ -85,7 +85,7 @@ func Route(ctx context.Context, s State) func(chi.Router) {
 		// proxy to the grafana host
 		grafana, _ := url.Parse("http://localhost:3000")
 		proxy := httputil.NewSingleHostReverseProxy(grafana)
-		r.Handle("/grafana", vmiddleware.RequirePermission(radio.PermDev, proxy.ServeHTTP))
+		r.Handle("/grafana/*", vmiddleware.RequirePermission(radio.PermDev, proxy.ServeHTTP))
 
 		// debug handlers, might not be needed later
 		r.HandleFunc("/streamer/start", vmiddleware.RequirePermission(radio.PermAdmin, s.StartStreamer))