From cfdfe0b2f0fa3691936fdc541b9fb486d31310fc Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Sun, 11 Aug 2024 08:51:10 -0600 Subject: [PATCH 1/7] Initial update to have attributes use the ordered map so the order is maintained into clickhouse --- .../internal/metrics_model.go | 21 ++++--- .../internal/metrics_model_test.go | 47 ++++++++------ .../internal/ordered_map.go | 62 +++++++++++++++++++ .../internal/ordered_map_test.go | 51 +++++++++++++++ 4 files changed, 156 insertions(+), 25 deletions(-) create mode 100644 exporter/clickhouseexporter/internal/ordered_map.go create mode 100644 exporter/clickhouseexporter/internal/ordered_map_test.go diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index b4a1e6a3ab71..147b2b7ae071 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -13,6 +13,7 @@ import ( "sync" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/column" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" @@ -102,9 +103,9 @@ func InsertMetrics(ctx context.Context, db *sql.DB, metricsMap map[pmetric.Metri return errs } -func convertExemplars(exemplars pmetric.ExemplarSlice) (clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet) { +func convertExemplars(exemplars pmetric.ExemplarSlice) (OrderedMap, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet) { var ( - attrs clickhouse.ArraySet + attrs OrderedMap times clickhouse.ArraySet values clickhouse.ArraySet traceIDs clickhouse.ArraySet @@ -112,7 +113,11 @@ func convertExemplars(exemplars pmetric.ExemplarSlice) (clickhouse.ArraySet, cli ) for i := 0; i < exemplars.Len(); i++ { exemplar := exemplars.At(i) - attrs = append(attrs, attributesToMap(exemplar.FilteredAttributes())) + orderedMap := attributesToMap(exemplar.FilteredAttributes()) + mapIterator := orderedMap.Iterator() + for mapIterator.Next() { + attrs.Put(mapIterator.Key(), mapIterator.Value()) + } times = append(times, exemplar.Timestamp().AsTime()) values = append(values, getValue(exemplar.IntValue(), exemplar.DoubleValue(), exemplar.ValueType())) @@ -159,13 +164,15 @@ func getValue(intValue int64, floatValue float64, dataType any) float64 { } } -func attributesToMap(attributes pcommon.Map) map[string]string { - m := make(map[string]string, attributes.Len()) +func attributesToMap(attributes pcommon.Map) column.IterableOrderedMap { + + om := NewOrderedMap() + //m := make(map[string]string, attributes.Len()) attributes.Range(func(k string, v pcommon.Value) bool { - m[k] = v.AsString() + om.Put(k, v.AsString()) return true }) - return m + return om } func convertSliceToArraySet[T any](slice []T) clickhouse.ArraySet { diff --git a/exporter/clickhouseexporter/internal/metrics_model_test.go b/exporter/clickhouseexporter/internal/metrics_model_test.go index 83d983bb8987..01e249791cc0 100644 --- a/exporter/clickhouseexporter/internal/metrics_model_test.go +++ b/exporter/clickhouseexporter/internal/metrics_model_test.go @@ -4,6 +4,7 @@ package internal import ( + "fmt" "testing" "time" @@ -21,16 +22,26 @@ func Test_attributesToMap(t *testing.T) { attributes.PutInt("int", 0) attributes.PutDouble("double", 0.0) result := attributesToMap(attributes) - require.Equal( - t, - map[string]string{ - "key": "value", - "bool": "true", - "int": "0", - "double": "0", - }, - result, - ) + + expected := map[string]string{ + "key": "value", + "bool": "true", + "int": "0", + "double": "0", + } + + iter := result.Iterator() + for iter.Next() { + found := false + key := iter.Key().(string) + for k, value := range expected { + if key == k { + found = true + require.Equal(t, value, iter.Value()) + } + } + require.Equal(t, true, found, fmt.Sprintf( "key %v not found in expected", key)) + } } func Test_convertExemplars(t *testing.T) { @@ -38,7 +49,7 @@ func Test_convertExemplars(t *testing.T) { t.Run("empty exemplar", func(t *testing.T) { exemplars := pmetric.NewExemplarSlice() var ( - expectAttrs clickhouse.ArraySet + expectAttrs OrderedMap expectTimes clickhouse.ArraySet expectValues clickhouse.ArraySet expectTraceIDs clickhouse.ArraySet @@ -59,7 +70,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.FilteredAttributes().PutStr("key2", "value2") attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{"key1": "value1", "key2": "value2"}}, attrs) + require.Equal(t, map[string]string{"key1": "value1", "key2": "value2"}, ConvertOrderedMapToMap(&attrs)) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -71,7 +82,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTimestamp(pcommon.NewTimestampFromTime(time.Unix(1672218930, 0))) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, OrderedMap{}, attrs) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -83,7 +94,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetDoubleValue(15.0) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, OrderedMap{}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{15.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -95,7 +106,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetIntValue(20) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, OrderedMap{}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{20.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -107,7 +118,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetSpanID([8]byte{1, 2, 3, 4}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, OrderedMap{}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -119,7 +130,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTraceID([16]byte{1, 2, 3, 4}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, OrderedMap{}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000"}, traceIDs) @@ -146,7 +157,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTraceID([16]byte{1, 2, 3, 5}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{"key1": "value1", "key2": "value2"}, map[string]string{"key3": "value3", "key4": "value4"}}, attrs) + require.Equal(t, map[string]string{"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}, ConvertOrderedMapToMap(&attrs)) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC(), time.Unix(1672219930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{20.0, 16.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000", "01020305000000000000000000000000"}, traceIDs) diff --git a/exporter/clickhouseexporter/internal/ordered_map.go b/exporter/clickhouseexporter/internal/ordered_map.go new file mode 100644 index 000000000000..290182205814 --- /dev/null +++ b/exporter/clickhouseexporter/internal/ordered_map.go @@ -0,0 +1,62 @@ +package internal + +import ( + "github.com/ClickHouse/clickhouse-go/v2/lib/column" +) + +// OrderedMap is a simple (non thread safe) ordered map +type OrderedMap struct { + Keys []any + Values []any +} + +func NewOrderedMap() *OrderedMap { + return &OrderedMap{} +} + +func (om *OrderedMap) Put(key any, value any) { + om.Keys = append(om.Keys, key) + om.Values = append(om.Values, value) +} + +func (om *OrderedMap) Iterator() column.MapIterator { + return NewOrderedMapIterator(om) +} + +type OrderedMapIter struct { + om *OrderedMap + iterIndex int +} + +func NewOrderedMapIterator(om *OrderedMap) column.MapIterator { + return &OrderedMapIter{om: om, iterIndex: -1} +} + +func (i *OrderedMapIter) Next() bool { + i.iterIndex++ + return i.iterIndex < len(i.om.Keys) +} + +func (i *OrderedMapIter) Key() any { + return i.om.Keys[i.iterIndex] +} + +func (i *OrderedMapIter) Value() any { + return i.om.Values[i.iterIndex] +} + +func ConvertOrderedMapToMap(orderedMap *OrderedMap) map[string]string { + result := make(map[string]string) + for i, key := range orderedMap.Keys { + strKey, ok := key.(string) + if !ok { + continue // Skip if the key is not a string + } + strValue, ok := orderedMap.Values[i].(string) + if !ok { + continue // Skip if the value is not a string + } + result[strKey] = strValue + } + return result +} diff --git a/exporter/clickhouseexporter/internal/ordered_map_test.go b/exporter/clickhouseexporter/internal/ordered_map_test.go new file mode 100644 index 000000000000..b4c12f1febcf --- /dev/null +++ b/exporter/clickhouseexporter/internal/ordered_map_test.go @@ -0,0 +1,51 @@ +package internal + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConvertOrderedMapToMap(t *testing.T) { + t.Run("Normal case with valid string keys and values", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put("key1", "value1") + orderedMap.Put("key2", "value2") + + result := ConvertOrderedMapToMap(orderedMap) + + expected := map[string]string{"key1": "value1", "key2": "value2"} + assert.Equal(t, expected, result) + }) + + t.Run("Case with non-string keys", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put(1, "value1") + orderedMap.Put("key2", "value2") + + result := ConvertOrderedMapToMap(orderedMap) + + expected := map[string]string{"key2": "value2"} + assert.Equal(t, expected, result) + }) + + t.Run("Case with non-string values", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put("key1", 1) + orderedMap.Put("key2", "value2") + + result := ConvertOrderedMapToMap(orderedMap) + + expected := map[string]string{"key2": "value2"} + assert.Equal(t, expected, result) + }) + + t.Run("Empty OrderedMap", func(t *testing.T) { + orderedMap := NewOrderedMap() + + result := ConvertOrderedMapToMap(orderedMap) + + expected := map[string]string{} + assert.Equal(t, expected, result) + }) +} \ No newline at end of file From 6256fce87d9f777084b94389ae553eb21dff01b8 Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Mon, 12 Aug 2024 11:12:30 -0600 Subject: [PATCH 2/7] Updated exempler attribute creation after integration testing identified issue --- .../internal/metrics_model.go | 22 +++++-- .../internal/metrics_model_test.go | 23 +++---- .../internal/ordered_map.go | 25 +++++++ .../internal/ordered_map_test.go | 66 +++++++++---------- .../internal/sum_metrics.go | 1 + 5 files changed, 86 insertions(+), 51 deletions(-) diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index 147b2b7ae071..c5f9afbaaaf8 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -103,21 +103,29 @@ func InsertMetrics(ctx context.Context, db *sql.DB, metricsMap map[pmetric.Metri return errs } -func convertExemplars(exemplars pmetric.ExemplarSlice) (OrderedMap, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet) { +func convertExemplars(exemplars pmetric.ExemplarSlice) (clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet, clickhouse.ArraySet) { var ( - attrs OrderedMap - times clickhouse.ArraySet - values clickhouse.ArraySet - traceIDs clickhouse.ArraySet - spanIDs clickhouse.ArraySet + attrs column.IterableOrderedMap + exemplarAttrs clickhouse.ArraySet + times clickhouse.ArraySet + values clickhouse.ArraySet + traceIDs clickhouse.ArraySet + spanIDs clickhouse.ArraySet ) + + attrs = NewOrderedMap() + for i := 0; i < exemplars.Len(); i++ { exemplar := exemplars.At(i) orderedMap := attributesToMap(exemplar.FilteredAttributes()) mapIterator := orderedMap.Iterator() + for mapIterator.Next() { attrs.Put(mapIterator.Key(), mapIterator.Value()) } + + exemplarAttrs = append(exemplarAttrs, attrs) + times = append(times, exemplar.Timestamp().AsTime()) values = append(values, getValue(exemplar.IntValue(), exemplar.DoubleValue(), exemplar.ValueType())) @@ -125,7 +133,7 @@ func convertExemplars(exemplars pmetric.ExemplarSlice) (OrderedMap, clickhouse.A traceIDs = append(traceIDs, hex.EncodeToString(traceID[:])) spanIDs = append(spanIDs, hex.EncodeToString(spanID[:])) } - return attrs, times, values, traceIDs, spanIDs + return exemplarAttrs, times, values, traceIDs, spanIDs } // https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto#L358 diff --git a/exporter/clickhouseexporter/internal/metrics_model_test.go b/exporter/clickhouseexporter/internal/metrics_model_test.go index 01e249791cc0..71b42bf7a9db 100644 --- a/exporter/clickhouseexporter/internal/metrics_model_test.go +++ b/exporter/clickhouseexporter/internal/metrics_model_test.go @@ -35,12 +35,12 @@ func Test_attributesToMap(t *testing.T) { found := false key := iter.Key().(string) for k, value := range expected { - if key == k { + if key == k { found = true require.Equal(t, value, iter.Value()) - } + } } - require.Equal(t, true, found, fmt.Sprintf( "key %v not found in expected", key)) + require.Equal(t, true, found, fmt.Sprintf("key %v not found in expected", key)) } } @@ -49,7 +49,7 @@ func Test_convertExemplars(t *testing.T) { t.Run("empty exemplar", func(t *testing.T) { exemplars := pmetric.NewExemplarSlice() var ( - expectAttrs OrderedMap + expectAttrs clickhouse.ArraySet expectTimes clickhouse.ArraySet expectValues clickhouse.ArraySet expectTraceIDs clickhouse.ArraySet @@ -70,7 +70,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.FilteredAttributes().PutStr("key2", "value2") attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, map[string]string{"key1": "value1", "key2": "value2"}, ConvertOrderedMapToMap(&attrs)) + require.Equal(t, map[string]string{"key1": "value1", "key2": "value2"}, ConvertOArraySetToMap(attrs)) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -82,7 +82,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTimestamp(pcommon.NewTimestampFromTime(time.Unix(1672218930, 0))) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, OrderedMap{}, attrs) + require.Equal(t, clickhouse.ArraySet{NewOrderedMap()}, attrs) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -94,7 +94,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetDoubleValue(15.0) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, OrderedMap{}, attrs) + require.Equal(t, clickhouse.ArraySet{NewOrderedMap()}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{15.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -106,7 +106,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetIntValue(20) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, OrderedMap{}, attrs) + require.Equal(t, clickhouse.ArraySet{NewOrderedMap()}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{20.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -118,7 +118,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetSpanID([8]byte{1, 2, 3, 4}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, OrderedMap{}, attrs) + require.Equal(t, clickhouse.ArraySet{NewOrderedMap()}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -130,7 +130,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTraceID([16]byte{1, 2, 3, 4}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, OrderedMap{}, attrs) + require.Equal(t, clickhouse.ArraySet{NewOrderedMap()}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000"}, traceIDs) @@ -157,7 +157,8 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTraceID([16]byte{1, 2, 3, 5}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, map[string]string{"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}, ConvertOrderedMapToMap(&attrs)) + + require.Equal(t, map[string]string{"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}, ConvertOArraySetToMap(attrs)) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC(), time.Unix(1672219930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{20.0, 16.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000", "01020305000000000000000000000000"}, traceIDs) diff --git a/exporter/clickhouseexporter/internal/ordered_map.go b/exporter/clickhouseexporter/internal/ordered_map.go index 290182205814..22b985391213 100644 --- a/exporter/clickhouseexporter/internal/ordered_map.go +++ b/exporter/clickhouseexporter/internal/ordered_map.go @@ -1,6 +1,7 @@ package internal import ( + "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/column" ) @@ -60,3 +61,27 @@ func ConvertOrderedMapToMap(orderedMap *OrderedMap) map[string]string { } return result } + +func ConvertOArraySetToMap(list clickhouse.ArraySet) map[string]string { + result := make(map[string]string) + for _, orderedMap := range list { + mapValue, ok := orderedMap.(column.IterableOrderedMap) + if !ok { + continue // Skip if the key is not a string + } + + for iter := mapValue.Iterator(); iter.Next(); { + strKey, ok := iter.Key().(string) + if !ok { + continue // Skip if the key is not a string + } + + strValue, ok := iter.Value().(string) + if !ok { + continue // Skip if the value is not a string + } + result[strKey] = strValue + } + } + return result +} diff --git a/exporter/clickhouseexporter/internal/ordered_map_test.go b/exporter/clickhouseexporter/internal/ordered_map_test.go index b4c12f1febcf..2a8bf0780b98 100644 --- a/exporter/clickhouseexporter/internal/ordered_map_test.go +++ b/exporter/clickhouseexporter/internal/ordered_map_test.go @@ -1,51 +1,51 @@ package internal import ( - "testing" + "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/assert" ) func TestConvertOrderedMapToMap(t *testing.T) { - t.Run("Normal case with valid string keys and values", func(t *testing.T) { - orderedMap := NewOrderedMap() - orderedMap.Put("key1", "value1") - orderedMap.Put("key2", "value2") + t.Run("Normal case with valid string keys and values", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put("key1", "value1") + orderedMap.Put("key2", "value2") - result := ConvertOrderedMapToMap(orderedMap) + result := ConvertOrderedMapToMap(orderedMap) - expected := map[string]string{"key1": "value1", "key2": "value2"} - assert.Equal(t, expected, result) - }) + expected := map[string]string{"key1": "value1", "key2": "value2"} + assert.Equal(t, expected, result) + }) - t.Run("Case with non-string keys", func(t *testing.T) { - orderedMap := NewOrderedMap() - orderedMap.Put(1, "value1") - orderedMap.Put("key2", "value2") + t.Run("Case with non-string keys", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put(1, "value1") + orderedMap.Put("key2", "value2") - result := ConvertOrderedMapToMap(orderedMap) + result := ConvertOrderedMapToMap(orderedMap) - expected := map[string]string{"key2": "value2"} - assert.Equal(t, expected, result) - }) + expected := map[string]string{"key2": "value2"} + assert.Equal(t, expected, result) + }) - t.Run("Case with non-string values", func(t *testing.T) { - orderedMap := NewOrderedMap() - orderedMap.Put("key1", 1) - orderedMap.Put("key2", "value2") + t.Run("Case with non-string values", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put("key1", 1) + orderedMap.Put("key2", "value2") - result := ConvertOrderedMapToMap(orderedMap) + result := ConvertOrderedMapToMap(orderedMap) - expected := map[string]string{"key2": "value2"} - assert.Equal(t, expected, result) - }) + expected := map[string]string{"key2": "value2"} + assert.Equal(t, expected, result) + }) - t.Run("Empty OrderedMap", func(t *testing.T) { - orderedMap := NewOrderedMap() + t.Run("Empty OrderedMap", func(t *testing.T) { + orderedMap := NewOrderedMap() - result := ConvertOrderedMapToMap(orderedMap) + result := ConvertOrderedMapToMap(orderedMap) - expected := map[string]string{} - assert.Equal(t, expected, result) - }) -} \ No newline at end of file + expected := map[string]string{} + assert.Equal(t, expected, result) + }) +} diff --git a/exporter/clickhouseexporter/internal/sum_metrics.go b/exporter/clickhouseexporter/internal/sum_metrics.go index c784dfdf7325..556868a78ce0 100644 --- a/exporter/clickhouseexporter/internal/sum_metrics.go +++ b/exporter/clickhouseexporter/internal/sum_metrics.go @@ -121,6 +121,7 @@ func (s *sumMetrics) insert(ctx context.Context, db *sql.DB) error { for i := 0; i < model.sum.DataPoints().Len(); i++ { dp := model.sum.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) + _, err = statement.ExecContext(ctx, model.metadata.ResAttr, model.metadata.ResURL, From 60c729bbe8bb940afb86cb021251ee51695b0a47 Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Tue, 13 Aug 2024 13:32:23 -0600 Subject: [PATCH 3/7] Updating the Logs and tracing to use the OrderedMap, explicitly Sort when converting from --- exporter/clickhouseexporter/exporter_logs.go | 9 ++-- .../clickhouseexporter/exporter_logs_test.go | 9 ++-- .../clickhouseexporter/exporter_metrics.go | 2 +- .../clickhouseexporter/exporter_traces.go | 18 ++++--- .../internal/exponential_histogram_metrics.go | 4 +- .../internal/gauge_metrics.go | 6 +-- .../internal/histogram_metrics.go | 6 +-- .../internal/metrics_model.go | 8 +-- .../internal/metrics_model_test.go | 6 +-- .../internal/ordered_map.go | 51 ++++++++++++++++++- .../internal/ordered_map_test.go | 17 +++++++ .../internal/sum_metrics.go | 8 +-- .../internal/summary_metrics.go | 6 +-- 13 files changed, 111 insertions(+), 39 deletions(-) diff --git a/exporter/clickhouseexporter/exporter_logs.go b/exporter/clickhouseexporter/exporter_logs.go index ab48b27b92a8..2e539214cccb 100644 --- a/exporter/clickhouseexporter/exporter_logs.go +++ b/exporter/clickhouseexporter/exporter_logs.go @@ -16,6 +16,7 @@ import ( conventions "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil" ) @@ -77,7 +78,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { logs := ld.ResourceLogs().At(i) res := logs.Resource() resURL := logs.SchemaUrl() - resAttr := attributesToMap(res.Attributes()) + resAttr := internal.OtelAttributesToOrderedMap(res.Attributes()) if v, ok := res.Attributes().Get(conventions.AttributeServiceName); ok { serviceName = v.Str() } @@ -87,7 +88,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { scopeURL := logs.ScopeLogs().At(j).SchemaUrl() scopeName := logs.ScopeLogs().At(j).Scope().Name() scopeVersion := logs.ScopeLogs().At(j).Scope().Version() - scopeAttr := attributesToMap(logs.ScopeLogs().At(j).Scope().Attributes()) + scopeAttr := internal.OtelAttributesToOrderedMap(logs.ScopeLogs().At(j).Scope().Attributes()) for k := 0; k < rs.Len(); k++ { r := rs.At(k) @@ -97,7 +98,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { timestamp = r.ObservedTimestamp() } - logAttr := attributesToMap(r.Attributes()) + logAttr := internal.OtelAttributesToOrderedMap(r.Attributes()) _, err = statement.ExecContext(ctx, timestamp.AsTime(), traceutil.TraceIDToHexOrEmptyString(r.TraceID()), @@ -129,7 +130,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { return err } -func attributesToMap(attributes pcommon.Map) map[string]string { +func attributesToStringMap(attributes pcommon.Map) map[string]string { m := make(map[string]string, attributes.Len()) attributes.Range(func(k string, v pcommon.Value) bool { m[k] = v.AsString() diff --git a/exporter/clickhouseexporter/exporter_logs_test.go b/exporter/clickhouseexporter/exporter_logs_test.go index 4856317a7cdd..21aeefda9434 100644 --- a/exporter/clickhouseexporter/exporter_logs_test.go +++ b/exporter/clickhouseexporter/exporter_logs_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -95,9 +96,9 @@ func TestExporter_pushLogsData(t *testing.T) { initClickhouseTestServer(t, func(query string, values []driver.Value) error { if strings.HasPrefix(query, "INSERT") { require.Equal(t, "https://opentelemetry.io/schemas/1.4.0", values[8]) - require.Equal(t, map[string]string{ + require.Equal(t, internal.NewOrderedMapFromMap(map[string]string{ "service.name": "test-service", - }, values[9]) + }), values[9]) } return nil }) @@ -110,9 +111,9 @@ func TestExporter_pushLogsData(t *testing.T) { require.Equal(t, "https://opentelemetry.io/schemas/1.7.0", values[10]) require.Equal(t, "io.opentelemetry.contrib.clickhouse", values[11]) require.Equal(t, "1.0.0", values[12]) - require.Equal(t, map[string]string{ + require.Equal(t, internal.NewOrderedMapFromMap(map[string]string{ "lib": "clickhouse", - }, values[13]) + }), values[13]) } return nil }) diff --git a/exporter/clickhouseexporter/exporter_metrics.go b/exporter/clickhouseexporter/exporter_metrics.go index 6f11ba940d57..2a4fadbf8301 100644 --- a/exporter/clickhouseexporter/exporter_metrics.go +++ b/exporter/clickhouseexporter/exporter_metrics.go @@ -63,7 +63,7 @@ func (e *metricsExporter) pushMetricsData(ctx context.Context, md pmetric.Metric metricsMap := internal.NewMetricsModel(e.cfg.MetricsTableName) for i := 0; i < md.ResourceMetrics().Len(); i++ { metrics := md.ResourceMetrics().At(i) - resAttr := attributesToMap(metrics.Resource().Attributes()) + resAttr := attributesToStringMap(metrics.Resource().Attributes()) for j := 0; j < metrics.ScopeMetrics().Len(); j++ { rs := metrics.ScopeMetrics().At(j).Metrics() scopeInstr := metrics.ScopeMetrics().At(j).Scope() diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index ff2aafb82f14..3e78d5ff0f17 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -10,12 +10,14 @@ import ( "strings" "time" + "github.com/ClickHouse/clickhouse-go/v2" _ "github.com/ClickHouse/clickhouse-go/v2" // For register database driver. "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/pdata/ptrace" conventions "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil" ) @@ -74,7 +76,7 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er for i := 0; i < td.ResourceSpans().Len(); i++ { spans := td.ResourceSpans().At(i) res := spans.Resource() - resAttr := attributesToMap(res.Attributes()) + resAttr := internal.OtelAttributesToOrderedMap(res.Attributes()) var serviceName string if v, ok := res.Attributes().Get(conventions.AttributeServiceName); ok { serviceName = v.Str() @@ -85,7 +87,7 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er scopeVersion := spans.ScopeSpans().At(j).Scope().Version() for k := 0; k < rs.Len(); k++ { r := rs.At(k) - spanAttr := attributesToMap(r.Attributes()) + spanAttr := internal.OtelAttributesToOrderedMap(r.Attributes()) status := r.Status() eventTimes, eventNames, eventAttrs := convertEvents(r.Events()) linksTraceIDs, linksSpanIDs, linksTraceStates, linksAttrs := convertLinks(r.Links()) @@ -127,34 +129,34 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er return err } -func convertEvents(events ptrace.SpanEventSlice) ([]time.Time, []string, []map[string]string) { +func convertEvents(events ptrace.SpanEventSlice) ([]time.Time, []string, clickhouse.ArraySet) { var ( times []time.Time names []string - attrs []map[string]string + attrs clickhouse.ArraySet ) for i := 0; i < events.Len(); i++ { event := events.At(i) times = append(times, event.Timestamp().AsTime()) names = append(names, event.Name()) - attrs = append(attrs, attributesToMap(event.Attributes())) + attrs = append(attrs, internal.OtelAttributesToOrderedMap(event.Attributes())) } return times, names, attrs } -func convertLinks(links ptrace.SpanLinkSlice) ([]string, []string, []string, []map[string]string) { +func convertLinks(links ptrace.SpanLinkSlice) ([]string, []string, []string, clickhouse.ArraySet) { var ( traceIDs []string spanIDs []string states []string - attrs []map[string]string + attrs clickhouse.ArraySet ) for i := 0; i < links.Len(); i++ { link := links.At(i) traceIDs = append(traceIDs, traceutil.TraceIDToHexOrEmptyString(link.TraceID())) spanIDs = append(spanIDs, traceutil.SpanIDToHexOrEmptyString(link.SpanID())) states = append(states, link.TraceState().AsRaw()) - attrs = append(attrs, attributesToMap(link.Attributes())) + attrs = append(attrs, internal.OtelAttributesToOrderedMap(link.Attributes())) } return traceIDs, spanIDs, states, attrs } diff --git a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go index d23c37d2e2fd..72b0f6ba2704 100644 --- a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go @@ -143,14 +143,14 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error { model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + OtelAttributesToOrderedMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, serviceName, model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + OtelAttributesToOrderedMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), dp.Count(), diff --git a/exporter/clickhouseexporter/internal/gauge_metrics.go b/exporter/clickhouseexporter/internal/gauge_metrics.go index a45121ccaa95..8466fd164640 100644 --- a/exporter/clickhouseexporter/internal/gauge_metrics.go +++ b/exporter/clickhouseexporter/internal/gauge_metrics.go @@ -118,18 +118,18 @@ func (g *gaugeMetrics) insert(ctx context.Context, db *sql.DB) error { dp := model.gauge.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + NewOrderedMapFromMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + OtelAttributesToOrderedMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, serviceName, model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + OtelAttributesToOrderedMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), getValue(dp.IntValue(), dp.DoubleValue(), dp.ValueType()), diff --git a/exporter/clickhouseexporter/internal/histogram_metrics.go b/exporter/clickhouseexporter/internal/histogram_metrics.go index 0f75d83c93e8..3552ebd741aa 100644 --- a/exporter/clickhouseexporter/internal/histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/histogram_metrics.go @@ -130,18 +130,18 @@ func (h *histogramMetrics) insert(ctx context.Context, db *sql.DB) error { dp := model.histogram.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + NewOrderedMapFromMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + OtelAttributesToOrderedMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, serviceName, model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + OtelAttributesToOrderedMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), dp.Count(), diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index c5f9afbaaaf8..431fed524f76 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -117,7 +117,7 @@ func convertExemplars(exemplars pmetric.ExemplarSlice) (clickhouse.ArraySet, cli for i := 0; i < exemplars.Len(); i++ { exemplar := exemplars.At(i) - orderedMap := attributesToMap(exemplar.FilteredAttributes()) + orderedMap := OtelAttributesToOrderedMap(exemplar.FilteredAttributes()) mapIterator := orderedMap.Iterator() for mapIterator.Next() { @@ -172,14 +172,16 @@ func getValue(intValue int64, floatValue float64, dataType any) float64 { } } -func attributesToMap(attributes pcommon.Map) column.IterableOrderedMap { +// OtelAttributesToOrderedMap converts Otel attributes to OrderedMap +// This will Sort the attributes by key +func OtelAttributesToOrderedMap(attributes pcommon.Map) column.IterableOrderedMap { om := NewOrderedMap() - //m := make(map[string]string, attributes.Len()) attributes.Range(func(k string, v pcommon.Value) bool { om.Put(k, v.AsString()) return true }) + om.Sort() return om } diff --git a/exporter/clickhouseexporter/internal/metrics_model_test.go b/exporter/clickhouseexporter/internal/metrics_model_test.go index 71b42bf7a9db..fdfc1552e488 100644 --- a/exporter/clickhouseexporter/internal/metrics_model_test.go +++ b/exporter/clickhouseexporter/internal/metrics_model_test.go @@ -21,7 +21,7 @@ func Test_attributesToMap(t *testing.T) { attributes.PutBool("bool", true) attributes.PutInt("int", 0) attributes.PutDouble("double", 0.0) - result := attributesToMap(attributes) + result := OtelAttributesToOrderedMap(attributes) expected := map[string]string{ "key": "value", @@ -70,7 +70,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.FilteredAttributes().PutStr("key2", "value2") attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, map[string]string{"key1": "value1", "key2": "value2"}, ConvertOArraySetToMap(attrs)) + require.Equal(t, map[string]string{"key1": "value1", "key2": "value2"}, convertArraySetOrderedMapToMap(attrs)) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -158,7 +158,7 @@ func Test_convertExemplars(t *testing.T) { attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, map[string]string{"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}, ConvertOArraySetToMap(attrs)) + require.Equal(t, map[string]string{"key1": "value1", "key2": "value2", "key3": "value3", "key4": "value4"}, convertArraySetOrderedMapToMap(attrs)) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC(), time.Unix(1672219930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{20.0, 16.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000", "01020305000000000000000000000000"}, traceIDs) diff --git a/exporter/clickhouseexporter/internal/ordered_map.go b/exporter/clickhouseexporter/internal/ordered_map.go index 22b985391213..03f26458d034 100644 --- a/exporter/clickhouseexporter/internal/ordered_map.go +++ b/exporter/clickhouseexporter/internal/ordered_map.go @@ -1,6 +1,8 @@ package internal import ( + "sort" + "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/column" ) @@ -15,6 +17,25 @@ func NewOrderedMap() *OrderedMap { return &OrderedMap{} } +// NewOrderedMapFromMap converts a map to an ordered map +// Provided to limit the changes to the existing code so OrderedMap can introduced +// without large changes to the types that are used +// +// Parameters: +// - mapIn: The map that will be converted into a OrderedMap. +// +// Returns: +// +// A OrderedMap containing the same key-value pairs as the mapIn. +func NewOrderedMapFromMap(mapIn map[string]string) *OrderedMap { + om := NewOrderedMap() + for k, v := range mapIn { + om.Put(k, v) + } + om.Sort() + return om +} + func (om *OrderedMap) Put(key any, value any) { om.Keys = append(om.Keys, key) om.Values = append(om.Values, value) @@ -24,6 +45,32 @@ func (om *OrderedMap) Iterator() column.MapIterator { return NewOrderedMapIterator(om) } +// Sort sorts the OrderedMap by its keys. +func (om *OrderedMap) Sort() { + // Create a slice of indices and sort it based on the keys + indices := make([]int, len(om.Keys)) + for i := range indices { + indices[i] = i + } + + sort.Slice(indices, func(i, j int) bool { + return om.Keys[indices[i]].(string) < om.Keys[indices[j]].(string) + }) + + // Create new slices for sorted keys and values + sortedKeys := make([]any, len(om.Keys)) + sortedValues := make([]any, len(om.Values)) + + for i, idx := range indices { + sortedKeys[i] = om.Keys[idx] + sortedValues[i] = om.Values[idx] + } + + // Update the OrderedMap with sorted keys and values + om.Keys = sortedKeys + om.Values = sortedValues +} + type OrderedMapIter struct { om *OrderedMap iterIndex int @@ -62,7 +109,9 @@ func ConvertOrderedMapToMap(orderedMap *OrderedMap) map[string]string { return result } -func ConvertOArraySetToMap(list clickhouse.ArraySet) map[string]string { +// convertArraySetOrderedMapToMap converts a clickhouse.ArraySet of ordered maps to a map +// This is used for testing purposes to compare the expected and actual values +func convertArraySetOrderedMapToMap(list clickhouse.ArraySet) map[string]string { result := make(map[string]string) for _, orderedMap := range list { mapValue, ok := orderedMap.(column.IterableOrderedMap) diff --git a/exporter/clickhouseexporter/internal/ordered_map_test.go b/exporter/clickhouseexporter/internal/ordered_map_test.go index 2a8bf0780b98..2d0d7bedb7be 100644 --- a/exporter/clickhouseexporter/internal/ordered_map_test.go +++ b/exporter/clickhouseexporter/internal/ordered_map_test.go @@ -1,6 +1,7 @@ package internal import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -48,4 +49,20 @@ func TestConvertOrderedMapToMap(t *testing.T) { expected := map[string]string{} assert.Equal(t, expected, result) }) + + t.Run("Sort OrderedMap", func(t *testing.T) { + orderedMap := NewOrderedMap() + orderedMap.Put("key1", "value1") + orderedMap.Put("key3", "value3") + orderedMap.Put("key0", "value0") + orderedMap.Put("key2", "value2") + + orderedMap.Sort() + + for i := range orderedMap.Keys { + assert.Equal(t, fmt.Sprintf("key%v", i), orderedMap.Keys[i]) + assert.Equal(t, fmt.Sprintf("value%v", i), orderedMap.Values[i]) + } + + }) } diff --git a/exporter/clickhouseexporter/internal/sum_metrics.go b/exporter/clickhouseexporter/internal/sum_metrics.go index 556868a78ce0..e27ab7004d54 100644 --- a/exporter/clickhouseexporter/internal/sum_metrics.go +++ b/exporter/clickhouseexporter/internal/sum_metrics.go @@ -121,20 +121,20 @@ func (s *sumMetrics) insert(ctx context.Context, db *sql.DB) error { for i := 0; i < model.sum.DataPoints().Len(); i++ { dp := model.sum.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) - + _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + NewOrderedMapFromMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + OtelAttributesToOrderedMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, serviceName, model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + OtelAttributesToOrderedMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), getValue(dp.IntValue(), dp.DoubleValue(), dp.ValueType()), diff --git a/exporter/clickhouseexporter/internal/summary_metrics.go b/exporter/clickhouseexporter/internal/summary_metrics.go index 5f3ca7beab8e..b955feb6cb52 100644 --- a/exporter/clickhouseexporter/internal/summary_metrics.go +++ b/exporter/clickhouseexporter/internal/summary_metrics.go @@ -113,18 +113,18 @@ func (s *summaryMetrics) insert(ctx context.Context, db *sql.DB) error { quantiles, values := convertValueAtQuantile(dp.QuantileValues()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + NewOrderedMapFromMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + OtelAttributesToOrderedMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, serviceName, model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + OtelAttributesToOrderedMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), dp.Count(), From 896d225b074d58233ee03339b398a3973d9c58f5 Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Tue, 13 Aug 2024 14:25:56 -0600 Subject: [PATCH 4/7] fmt clean up --- exporter/clickhouseexporter/internal/sum_metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/clickhouseexporter/internal/sum_metrics.go b/exporter/clickhouseexporter/internal/sum_metrics.go index e27ab7004d54..0cbc9217ad0a 100644 --- a/exporter/clickhouseexporter/internal/sum_metrics.go +++ b/exporter/clickhouseexporter/internal/sum_metrics.go @@ -121,7 +121,7 @@ func (s *sumMetrics) insert(ctx context.Context, db *sql.DB) error { for i := 0; i < model.sum.DataPoints().Len(); i++ { dp := model.sum.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) - + _, err = statement.ExecContext(ctx, NewOrderedMapFromMap(model.metadata.ResAttr), model.metadata.ResURL, From 5c60007311e77557ba13a21f12a1fc7a0cf13eb9 Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Wed, 14 Aug 2024 08:50:51 -0600 Subject: [PATCH 5/7] Added additional attribute coverage for OrderedMap support --- .../exporter_traces_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/exporter/clickhouseexporter/exporter_traces_test.go b/exporter/clickhouseexporter/exporter_traces_test.go index 11391d86e430..0c48cd841993 100644 --- a/exporter/clickhouseexporter/exporter_traces_test.go +++ b/exporter/clickhouseexporter/exporter_traces_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" @@ -46,6 +48,28 @@ func TestExporter_pushTracesData(t *testing.T) { exporter := newTestTracesExporter(t, defaultEndpoint) mustPushTracesData(t, exporter, simpleTraces(1)) }) + + t.Run("check OrderMap attributes", func(t *testing.T) { + initClickhouseTestServer(t, func(query string, values []driver.Value) error { + if strings.HasPrefix(query, "INSERT INTO otel_trace") { + require.Equal(t, internal.NewOrderedMapFromMap(map[string]string{ + "service.name": "test-service", + }), values[8]) + require.Equal(t, internal.NewOrderedMapFromMap(map[string]string{ + "service.name": "v", + }), values[11]) + require.Equal(t, clickhouse.ArraySet{internal.NewOrderedMapFromMap(map[string]string{ + "level": "info", + })}, values[17]) + require.Equal(t, clickhouse.ArraySet{internal.NewOrderedMapFromMap(map[string]string{ + "k": "v", + })}, values[21]) + } + return nil + }) + exporter := newTestTracesExporter(t, defaultEndpoint) + mustPushTracesData(t, exporter, simpleTraces(1)) + }) } func newTestTracesExporter(t *testing.T, dsn string, fns ...func(*Config)) *tracesExporter { From 0f480bc0f1607e3e36f0aa214588b12588787f58 Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Thu, 15 Aug 2024 15:20:05 -0600 Subject: [PATCH 6/7] Adding change log --- ...e-exporter-ordered-map-for-attributes.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/clickhouse-exporter-ordered-map-for-attributes.yaml diff --git a/.chloggen/clickhouse-exporter-ordered-map-for-attributes.yaml b/.chloggen/clickhouse-exporter-ordered-map-for-attributes.yaml new file mode 100644 index 000000000000..428fc76467b1 --- /dev/null +++ b/.chloggen/clickhouse-exporter-ordered-map-for-attributes.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Updated exporter to sort attributes and maintain order of attributes into clickhouse" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33634] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user, api] From 45c5bfac685fcbeea547f899e917130ec7bda665 Mon Sep 17 00:00:00 2001 From: Justin Mason Date: Fri, 16 Aug 2024 08:55:58 -0600 Subject: [PATCH 7/7] fixed golint errors --- exporter/clickhouseexporter/exporter_logs_test.go | 3 ++- exporter/clickhouseexporter/exporter_traces_test.go | 3 ++- exporter/clickhouseexporter/internal/ordered_map.go | 5 ++++- exporter/clickhouseexporter/internal/ordered_map_test.go | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/exporter/clickhouseexporter/exporter_logs_test.go b/exporter/clickhouseexporter/exporter_logs_test.go index 21aeefda9434..f042b24be1a3 100644 --- a/exporter/clickhouseexporter/exporter_logs_test.go +++ b/exporter/clickhouseexporter/exporter_logs_test.go @@ -12,13 +12,14 @@ import ( "testing" "time" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" conventions "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap" "go.uber.org/zap/zaptest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" ) func TestLogsExporter_New(t *testing.T) { diff --git a/exporter/clickhouseexporter/exporter_traces_test.go b/exporter/clickhouseexporter/exporter_traces_test.go index 0c48cd841993..2cfe60505321 100644 --- a/exporter/clickhouseexporter/exporter_traces_test.go +++ b/exporter/clickhouseexporter/exporter_traces_test.go @@ -11,12 +11,13 @@ import ( "time" "github.com/ClickHouse/clickhouse-go/v2" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" conventions "go.opentelemetry.io/collector/semconv/v1.18.0" "go.uber.org/zap/zaptest" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" ) func TestExporter_pushTracesData(t *testing.T) { diff --git a/exporter/clickhouseexporter/internal/ordered_map.go b/exporter/clickhouseexporter/internal/ordered_map.go index 03f26458d034..688c1f83e29b 100644 --- a/exporter/clickhouseexporter/internal/ordered_map.go +++ b/exporter/clickhouseexporter/internal/ordered_map.go @@ -1,4 +1,7 @@ -package internal +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" import ( "sort" diff --git a/exporter/clickhouseexporter/internal/ordered_map_test.go b/exporter/clickhouseexporter/internal/ordered_map_test.go index 2d0d7bedb7be..c99572443a8a 100644 --- a/exporter/clickhouseexporter/internal/ordered_map_test.go +++ b/exporter/clickhouseexporter/internal/ordered_map_test.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package internal import (