Otel dist metrics change (paypal#96)

* Occ config logging (paypal#394)

* occ configurations logging

* cal event success

* adding cal data

* added TODOs

* Remove log_occ_confogs.go

* Remove testing files

* source of configs - files

* whitelist format change

* code clean up

* code review changes-1

* CR fixes

* CR fixes

* Delete tests/unittest/config_logging/main_test.go

* clean up

* Merge branch 'occ-config-logging' of /Users/simmidisetty/Documents/GitHub/OpenSourceHera/src/github.com/paypal/hera with conflicts.

* test for config logging

* removing test changes

* tests for all cases

* test

* making minor changes for logging feature specific data

* changes for incorporate review comments

---------

Co-authored-by: simmidisetty <[email protected]>
Co-authored-by: Rajesh S <[email protected]>

* changes for otel integration in hera

---------

Co-authored-by: satyakamala03 <[email protected]>
Co-authored-by: simmidisetty <[email protected]>
Co-authored-by: Rajesh S <[email protected]>

lib/statelog.go

@@ -19,6 +19,7 @@ package lib
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	otel_logger "github.com/paypal/hera/utility/logger/otel"
@@ -571,7 +572,7 @@ func (sl *StateLog) init() error {
 
 	if otelconfig.OTelConfigData.Enabled {
 		// Initialize statelog_metrics to send metrics information currently we are ignoring registration object returned from this call
-		stateStartErr := otel_logger.StartMetricsCollection(totalWorkersCount,
+		stateStartErr := otel_logger.StartMetricsCollection(context.Background(), totalWorkersCount,
 			otel_logger.WithMetricProvider(otel.GetMeterProvider()),
 			otel_logger.WithAppName(otelconfig.OTelConfigData.PoolName))
 

tests/unittest/otel_basic/main_test.go

@@ -28,9 +28,11 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) {
 	appcfg["rac_sql_interval"] = "0"
 	appcfg["child.executable"] = "mysqlworker"
 	appcfg["enable_otel"] = "true"
-	appcfg["otel_resolution_time_in_sec"] = "1"
+	appcfg["otel_resolution_time_in_sec"] = "10"
 	opscfg := make(map[string]string)
 	opscfg["opscfg.default.server.max_connections"] = "3"
+	appcfg["cfg_from_tns"] = "false"
+	appcfg["num_standby_dbs"] = "0"
 	opscfg["opscfg.default.server.log_level"] = "5"
 	os.Setenv("AVAILABILITY_ZONE", "test-dev")
 	os.Setenv("ENVIRONMENT", "dev")
@@ -108,9 +110,9 @@ func TestOTELMetricsBasic(t *testing.T) {
 	if count < 1 {
 		t.Fatalf("OTEL event should contain application as hera-test")
 	}
-	initCount := testutil.RegexCountFile("\"name\":\"pp.occ.init_connection.cnt\"", logFilePath)
+	initCount := testutil.RegexCountFile("\"name\":\"pp.occ.init_connection\"", logFilePath)
 	if initCount < 1 {
-		t.Fatalf("OTEL event should contain metric name pp.occ.init_connection.cnt")
+		t.Fatalf("OTEL event should contain metric name pp.occ.init_connection")
 	}
 	tagsCount := testutil.RegexCountFile("{\"key\":\"InstanceId\",\"value\":{\"intValue\":\"0\"}},{\"key\":\"ShardId\",\"value\":{\"intValue\":\"0\"}},{\"key\":\"WorkerType\",\"value\":{\"intValue\":\"0\"}",
 		logFilePath)

tests/unittest/testutil/main.go

@@ -23,9 +23,9 @@ func setup(cfg cfgFunc) error {
 	if appcfg["enable_otel"] == "true" {
 		err = mx.StartOTelAgent()
 	}
-	if err != nil {
-		return err
-	}
+	//if err != nil {
+	//	return err
+	//}
 	err = mx.StartServer()
 	return err
 }

utility/logger/otel/defs.go

@@ -9,30 +9,19 @@ import (
 // Following Metric Names will get instrumented as part of StateLogMetrics
 const (
 	// Worker States
-	InitConnGuageMetric      = "init_connection.cnt"
-	AccptConnGuageMetric     = "accept_connection.cnt"
-	WaitConnGuageMetric      = "wait_connection.cnt"
-	BusyConnGuageMetric      = "busy_connection.cnt"
-	ScheduledConnGuageMetric = "scheduled_connection.cnt"
-	FinishedConnGuageMetric  = "finished_connection.cnt"
-	QuiescedConnGuageMetric  = "quiesced_connection.cnt"
+	InitConnMetric      = "init_connection"
+	AccptConnMetric     = "accept_connection"
+	WaitConnMetric      = "wait_connection"
+	BusyConnMetric      = "busy_connection"
+	ScheduledConnMetric = "scheduled_connection"
+	FinishedConnMetric  = "finished_connection"
+	QuiescedConnMetric  = "quiesced_connection"
 
 	// Connection States
-	AssignedConnGuageMetric = "assigned_connection.cnt"
-	IdleConnGuageMetric     = "idle_connection.cnt"
-	BacklogConnGuageMetric  = "backlog_connection.cnt"
-	StrdConnGuageMetric     = "stranded_connection.cnt"
-
-	InitMaxGuageMetric     = "init_connection.cnt.max"
-	AcceptMinGuageMetric   = "accept_connection.cnt.min"
-	WaitMaxGuageMetric     = "wait_connection.cnt.max"
-	BusyMaxGuageMetric     = "busy_connection.cnt.max"
-	SchdMaxGuageMetric     = "scheduled_connection.cnt.max"
-	QuiescedMaxGuageMetric = "quiesced_connection.cnt.max"
-
-	IdleMaxGuageMetric    = "idle_connection.cnt.max"
-	BacklogMaxGuageMetric = "backlog_connection.cnt.max"
-	StrdMaxGuageMetric    = "stranded_connection.cnt.max"
+	AssignedConnMetric = "assigned_connection"
+	IdleConnMetric     = "idle_connection"
+	BacklogConnMetric  = "backlog_connection"
+	StrdConnMetric     = "stranded_connection"
 )
 
 const (
@@ -51,6 +40,9 @@ const (
 	ContainerHostDimName = string("container_host")
 )
 
+var StatelogBucket = []float64{0, 5, 10, 15, 20, 25, 30, 40, 50, 60, 80, 100, 120, 160, 200}
+var ConnectionStateBucket = []float64{0, 25, 50, 75, 100, 150, 200, 300, 400, 500, 600, 700, 800, 1200, 2400, 4800, 9600, 19200, 39400, 65536}
+
 const OtelInstrumentationVersion string = "v1.0"
 
 // DEFAULT_OTEL_COLLECTOR_PROTOCOL default OTEL configurations point to QA collector
@@ -84,18 +76,6 @@ type (
 	ServerType int
 )
 
-// StateData Represents stats by a worker
-type StateData struct {
-	Name       string
-	Value      float64
-	Dimensions metric.MeasurementOption
-}
-
-type DataPoint struct {
-	attr metric.MeasurementOption
-	data int64
-}
-
 // StateLogMetrics state_log_metrics reports workers states
 type StateLogMetrics struct {
 
@@ -114,31 +94,17 @@ type StateLogMetrics struct {
 
 	stateLock sync.Mutex
 
-	registration metric.Registration
-
-	initState metric.Int64ObservableGauge
-	acptState metric.Int64ObservableGauge
-	waitState metric.Int64ObservableGauge
-	busyState metric.Int64ObservableGauge
-	schdState metric.Int64ObservableGauge
-	fnshState metric.Int64ObservableGauge
-	quceState metric.Int64ObservableGauge
-	asgnState metric.Int64ObservableGauge
-	idleState metric.Int64ObservableGauge
-	bklgState metric.Int64ObservableGauge
-	strdState metric.Int64ObservableGauge
-
-	initStateMax metric.Int64ObservableGauge
-	waitStateMax metric.Int64ObservableGauge
-	busyStateMax metric.Int64ObservableGauge
-	schdStateMax metric.Int64ObservableGauge
-	quceStateMax metric.Int64ObservableGauge
-
-	idleStateMax metric.Int64ObservableGauge
-	bklgStateMax metric.Int64ObservableGauge
-	strdStateMax metric.Int64ObservableGauge
-
-	acptStateMin metric.Int64ObservableGauge
+	initState metric.Int64Histogram
+	acptState metric.Int64Histogram
+	waitState metric.Int64Histogram
+	busyState metric.Int64Histogram
+	schdState metric.Int64Histogram
+	fnshState metric.Int64Histogram
+	quceState metric.Int64Histogram
+	asgnState metric.Int64Histogram
+	idleState metric.Int64Histogram
+	bklgState metric.Int64Histogram
+	strdState metric.Int64Histogram
 }
 
 // Object represents the workers states data for worker belongs to specific shardId and workperType with flat-map

utility/logger/otel/logger.go

@@ -13,6 +13,7 @@ import (
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/sdk/instrumentation"
 	"go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/metric/metricdata"
 	"go.opentelemetry.io/otel/sdk/resource"
@@ -116,15 +117,130 @@ func newMeterProvider(ctx context.Context) (*metric.MeterProvider, error) {
 		logger.GetLogger().Log(logger.Alert, "failed to initialize metric exporter, error %v", err)
 		return nil, err
 	}
-
+	metricViews := getStateLogMetricsViews()
 	meterProvider := metric.NewMeterProvider(
 		metric.WithResource(getResourceInfo(config.OTelConfigData.PoolName)),
 		metric.WithReader(metric.NewPeriodicReader(metricExporter,
 			metric.WithInterval(time.Duration(config.OTelConfigData.ResolutionTimeInSec)*time.Second))),
+		metric.WithView(metricViews...),
 	)
 	return meterProvider, nil
 }
 
+func getStateLogMetricsViews() []metric.View {
+	initView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(InitConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	acptStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(AccptConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	waitStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(WaitConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	busyStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(BusyConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	schdStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(ScheduledConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	fnshStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(FinishedConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	quceStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(QuiescedConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	asgnStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(AssignedConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	idleStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(IdleConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	bklgStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(BacklogConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+
+	strdStateView := metric.NewView(
+		metric.Instrument{
+			Name:  config.OTelConfigData.PopulateMetricNamePrefix(StrdConnMetric),
+			Scope: instrumentation.Scope{Name: StateLogMeterName},
+		},
+		metric.Stream{
+			Aggregation: metric.AggregationBase2ExponentialHistogram{MaxSize: 32, MaxScale: 20},
+		},
+	)
+	return []metric.View{initView, acptStateView, waitStateView, busyStateView, schdStateView,
+		fnshStateView, quceStateView, asgnStateView, idleStateView, bklgStateView, strdStateView}
+}
+
 // getMetricExporter Initialize metric exporter based protocol selected by user.
 func getMetricExporter(ctx context.Context) (metric.Exporter, error) {
 	if config.OTelConfigData.OtelMetricGRPC {