From aaf4402fda68d3d1d6c416eb136932fa7f5331d9 Mon Sep 17 00:00:00 2001 From: Wessie Date: Fri, 20 Dec 2024 20:38:42 +0100 Subject: [PATCH] telemetry: add our own otelzerolog hook that uses the opentelemetry log api --- cmd/hanyuu/main.go | 15 ++-- go.mod | 3 + go.sum | 6 ++ telemetry/otel.go | 47 +++++-------- telemetry/otelzerolog/hook.go | 125 ++++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 36 deletions(-) create mode 100644 telemetry/otelzerolog/hook.go diff --git a/cmd/hanyuu/main.go b/cmd/hanyuu/main.go index fd1b955..9a1b897 100644 --- a/cmd/hanyuu/main.go +++ b/cmd/hanyuu/main.go @@ -22,13 +22,13 @@ import ( _ "github.com/R-a-dio/valkyrie/search/storage" // storage search interface _ "github.com/R-a-dio/valkyrie/storage/mariadb" // mariadb storage interface "github.com/R-a-dio/valkyrie/telemetry" + "github.com/R-a-dio/valkyrie/telemetry/otelzerolog" "github.com/R-a-dio/valkyrie/tracker" "github.com/R-a-dio/valkyrie/util" "github.com/R-a-dio/valkyrie/website" "github.com/Wessie/fdstore" "github.com/google/subcommands" "github.com/rs/zerolog" - "golang.org/x/term" ) type executeFn func(context.Context, config.Loader) error @@ -269,10 +269,12 @@ var bleveCmd = cmd{ } func main() { + var disableStdout bool // setup configuration file as top-level flag flag.StringVar(&configFile, "config", "hanyuu.toml", "filepath to configuration file") flag.StringVar(&logLevel, "loglevel", "info", "loglevel to use") flag.BoolVar(&useTelemetry, "telemetry", false, "to enable telemetry") + flag.BoolVar(&disableStdout, "disable-stdout", false, "set to true to stop logs being printed to stdout") // add all our top-level flags as important flags to subcommands flag.VisitAll(func(f *flag.Flag) { @@ -311,12 +313,10 @@ func main() { var code int // setup logger - // discard logs unless we are connected to a terminal - lo := io.Discard - if term.IsTerminal(int(os.Stdout.Fd())) { - lo = zerolog.ConsoleWriter{Out: os.Stdout} + var lo io.Writer = zerolog.ConsoleWriter{Out: os.Stdout} + if disableStdout { // discard logs if asked for + lo = io.Discard } - lo = zerolog.ConsoleWriter{Out: os.Stdout} logger := zerolog.New(lo).With().Timestamp().Logger() // change the level to what the flag told us @@ -325,7 +325,8 @@ func main() { logger.Error().Err(err).Msg("failed to parse loglevel flag") os.Exit(1) } - logger = logger.Level(level).Hook(telemetry.Hook) + // use the opentelemetry zerolog hook + logger = logger.Level(level).Hook(otelzerolog.Hook()) // setup root context ctx := context.Background() diff --git a/go.mod b/go.mod index d867191..dfba613 100644 --- a/go.mod +++ b/go.mod @@ -139,8 +139,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.9.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/log v0.9.0 // indirect go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.9.0 // indirect go.opentelemetry.io/proto/otlp v1.4.0 // indirect golang.org/x/net v0.32.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/go.sum b/go.sum index 2a01ad6..908c865 100644 --- a/go.sum +++ b/go.sum @@ -562,6 +562,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEj go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.9.0 h1:gA2gh+3B3NDvRFP30Ufh7CC3TtJRbUSf2TTD0LbCagw= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.9.0/go.mod h1:smRTR+02OtrVGjvWE1sQxhuazozKc/BXvvqqnmOxy+s= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0 h1:7F29RDmnlqk6B5d+sUqemt8TBfDqxryYW5gX6L74RFA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0/go.mod h1:ZiGDq7xwDMKmWDrN1XsXAj0iC7hns+2DhxBFSncNHSE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= @@ -570,10 +572,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojm go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/log v0.9.0 h1:0OiWRefqJ2QszpCiqwGO0u9ajMPe17q6IscQvvp3czY= +go.opentelemetry.io/otel/log v0.9.0/go.mod h1:WPP4OJ+RBkQ416jrFCQFuFKtXKD6mOoYCQm6ykK8VaU= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/sdk/log v0.9.0 h1:YPCi6W1Eg0vwT/XJWsv2/PaQ2nyAJYuF7UUjQSBe3bc= +go.opentelemetry.io/otel/sdk/log v0.9.0/go.mod h1:y0HdrOz7OkXQBuc2yjiqnEHc+CRKeVhRE3hx4RwTmV4= go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU= go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= diff --git a/telemetry/otel.go b/telemetry/otel.go index 9440467..8c18670 100644 --- a/telemetry/otel.go +++ b/telemetry/otel.go @@ -9,38 +9,25 @@ import ( "github.com/R-a-dio/valkyrie/storage/mariadb" "github.com/R-a-dio/valkyrie/website" "github.com/XSAM/otelsql" - "github.com/agoda-com/opentelemetry-go/otelzerolog" - "github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs" - "github.com/agoda-com/opentelemetry-logs-go/exporters/otlp/otlplogs/otlplogsgrpc" - logsSDK "github.com/agoda-com/opentelemetry-logs-go/sdk/logs" "github.com/jmoiron/sqlx" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/push" "github.com/rs/zerolog" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/log/global" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" "google.golang.org/grpc" ) -// temporary until otel gets log handling -var LogProvider *logsSDK.LoggerProvider -var logHook *otelzerolog.Hook - -var Hook = zerolog.HookFunc(func(e *zerolog.Event, level zerolog.Level, message string) { - if logHook == nil { - return - } - - logHook.Run(e, level, message) -}) - func Init(ctx context.Context, cfg config.Config, service string) (func(), error) { tp, err := InitTracer(ctx, cfg, service) if err != nil { @@ -58,9 +45,9 @@ func Init(ctx context.Context, cfg config.Config, service string) (func(), error if err != nil { return nil, err } + // swap the next two lines once otlplog goes stable + global.SetLoggerProvider(lp) // otel.SetLogProvider(lp) - LogProvider = lp - logHook = otelzerolog.NewHook(lp) otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) // done setting up, swap global functions to inject telemetry @@ -172,15 +159,17 @@ func InitMetric(ctx context.Context, cfg config.Config, service string) (*metric return mp, nil } -func InitLogs(ctx context.Context, cfg config.Config, service string) (*logsSDK.LoggerProvider, error) { +func InitLogs(ctx context.Context, cfg config.Config, service string) (*log.LoggerProvider, error) { conf := cfg.Conf().Telemetry - logs_exporter, err := otlplogs.NewExporter(ctx, - otlplogs.WithClient( - otlplogsgrpc.NewClient(otlplogsgrpc.WithInsecure(), - otlplogsgrpc.WithEndpoint(conf.Endpoint), - ), - ), + logs_exporter, err := otlploggrpc.New(ctx, + otlploggrpc.WithInsecure(), + otlploggrpc.WithEndpoint(conf.Endpoint), + otlploggrpc.WithHeaders(map[string]string{ + "Authorization": conf.Auth, + "organization": "default", + "stream-name": "default", + }), ) if err != nil { return nil, err @@ -195,9 +184,9 @@ func InitLogs(ctx context.Context, cfg config.Config, service string) (*logsSDK. return nil, err } - lp := logsSDK.NewLoggerProvider( - logsSDK.WithBatcher(logs_exporter), - logsSDK.WithResource(res), + lp := log.NewLoggerProvider( + log.WithProcessor(log.NewBatchProcessor(logs_exporter)), + log.WithResource(res), ) return lp, nil diff --git a/telemetry/otelzerolog/hook.go b/telemetry/otelzerolog/hook.go new file mode 100644 index 0000000..9bb7ee4 --- /dev/null +++ b/telemetry/otelzerolog/hook.go @@ -0,0 +1,125 @@ +package otelzerolog + +import ( + "encoding/json" + "fmt" + "math" + "reflect" + "time" + + "github.com/rs/zerolog" + "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/global" +) + +var ( + InstrumentationName = "github.com/R-a-dio/valkyrie/telemetry/otelzerolog" + InstrumentationVersion = "0.1.0" +) + +func Hook() zerolog.Hook { + logger := global.GetLoggerProvider().Logger( + // TODO: make this use proper names and version + InstrumentationName, + log.WithInstrumentationVersion(InstrumentationVersion), + ) + + return &hook{logger} +} + +type hook struct { + logger log.Logger +} + +func (h hook) Run(e *zerolog.Event, level zerolog.Level, msg string) { + if !e.Enabled() { + return + } + + r := log.Record{} + ctx := e.GetCtx() + now := time.Now() + + r.SetBody(log.StringValue(msg)) + + r.SetTimestamp(now) + r.SetObservedTimestamp(now) + + r.SetSeverity(convertLevel(level)) + r.SetSeverityText(level.String()) + + logData := make(map[string]interface{}) + // create a string that appends } to the end of the buf variable you access via reflection + ev := fmt.Sprintf("%s}", reflect.ValueOf(e).Elem().FieldByName("buf")) + _ = json.Unmarshal([]byte(ev), &logData) + + for k, v := range logData { + r.AddAttributes(convertToKeyValue(k, v)) + } + + h.logger.Emit(ctx, r) +} + +func convertLevel(level zerolog.Level) log.Severity { + switch level { + case zerolog.DebugLevel: + return log.SeverityDebug + case zerolog.InfoLevel: + return log.SeverityInfo + case zerolog.WarnLevel: + return log.SeverityWarn + case zerolog.ErrorLevel: + return log.SeverityError + case zerolog.PanicLevel: + return log.SeverityFatal1 + case zerolog.FatalLevel: + return log.SeverityFatal2 + default: + return log.SeverityUndefined + } +} + +func convertToKeyValue(key string, value any) log.KeyValue { + return log.KeyValue{ + Key: key, + Value: convertToValue(value), + } +} + +func convertArray(value []any) log.Value { + values := make([]log.Value, 0, len(value)) + for _, v := range value { + values = append(values, convertToValue(v)) + } + return log.SliceValue(values...) +} + +func convertMap(value map[string]any) log.Value { + kvs := make([]log.KeyValue, 0, len(value)) + for k, v := range value { + kvs = append(kvs, convertToKeyValue(k, v)) + } + return log.MapValue(kvs...) +} + +func convertToValue(value any) log.Value { + switch value := value.(type) { + case bool: + return log.BoolValue(value) + case float64: + if _, frac := math.Modf(value); frac == 0.0 { + return log.Int64Value(int64(value)) + } + return log.Float64Value(value) + case string: + return log.StringValue(value) + case []any: + return convertArray(value) + case map[string]any: + return convertMap(value) + } + + // should be unreachable if this only gets input from encoding/json, but handle it + // anyway by just turning whatever value we got into a string with fmt + return log.StringValue(fmt.Sprintf("%v", value)) +}