From cd90913ab380daa4be3add63ebc00f79a2e8ec6a Mon Sep 17 00:00:00 2001 From: William Dumont Date: Thu, 13 Feb 2025 15:26:39 +0100 Subject: [PATCH 01/14] fix flaky foreach test (#2717) --- internal/runtime/foreach_stringer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/runtime/foreach_stringer_test.go b/internal/runtime/foreach_stringer_test.go index d900a7d955..e01fd09970 100644 --- a/internal/runtime/foreach_stringer_test.go +++ b/internal/runtime/foreach_stringer_test.go @@ -52,14 +52,14 @@ func testConfigForEachStringer(t *testing.T, config string, expectedDebugInfo *s if expectedDebugInfo != nil { require.EventuallyWithT(t, func(c *assert.CollectT) { debugInfo := getDebugInfo[string](t, ctrl, "", "testcomponents.string_receiver.log") - require.Equal(t, *expectedDebugInfo, debugInfo) + assert.Equal(c, *expectedDebugInfo, debugInfo) }, 3*time.Second, 10*time.Millisecond) } if expectedDebugInfo2 != nil { require.EventuallyWithT(t, func(c *assert.CollectT) { debugInfo := getDebugInfo[string](t, ctrl, "", "testcomponents.string_receiver.log2") - require.Equal(t, *expectedDebugInfo2, debugInfo) + assert.Equal(c, *expectedDebugInfo2, debugInfo) }, 3*time.Second, 10*time.Millisecond) } } From 90c1721172ec4f73ccb37361c802fc598274fb52 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Thu, 13 Feb 2025 15:51:05 +0100 Subject: [PATCH 02/14] database_observability: fix quoting of in SHOW CREATE statement (#2715) database_observability: fix quoting of in SHOW CREATE statement Quote schema name and table name when formatting the SHOW CREATE query. --- CHANGELOG.md | 33 ++++++++++--------- .../mysql/collector/schema_table.go | 4 +-- .../mysql/collector/schema_table_test.go | 10 +++--- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca05a37768..d8f014414a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,20 +29,21 @@ Main (unreleased) - Add json format support for log export via faro receiver (@ravishankar15) - (_Experimental_) Various changes to the experimental component `database_observability.mysql`: - - Always log `instance` label key (@cristiangreco) - - Improve parsing of truncated queries (@cristiangreco) - - Capture schema name for query samples (@cristiangreco) - - Fix handling of view table types when detecting schema (@matthewnolf) - - Fix error handling during result set iteration (@cristiangreco) - - Better support for table name parsing (@cristiangreco) - - Better error handling for components (@cristiangreco) - - Add namespace to `connection_info` metric (@cristiangreco) - - Added table columns parsing (@cristiagreco) - - Add enable/disable collector configurability to `database_observability.mysql`. This removes the `query_samples_enabled` argument, now configurable via enable/disable collector. (@fridgepoet) - - Refactor cache config in schema_table collector (@cristiangreco) - - Use labels for some indexed logs elements (@cristiangreco) - -- Reduce CPU usage of `loki.source.windowsevent` by up to 85% by updating the bookmark file every 10 seconds instead of after every event and by + - `connection_info`: add namespace to the metric (@cristiangreco) + - `query_sample`: better support for table name parsing (@cristiangreco) + - `query_sample`: capture schema name for query samples (@cristiangreco) + - `query_sample`: fix error handling during result set iteration (@cristiangreco) + - `query_sample`: improve parsing of truncated queries (@cristiangreco) + - `schema_table`: add table columns parsing (@cristiagreco) + - `schema_table`: correctly quote schema and table name in SHOW CREATE (@cristiangreco) + - `schema_table`: fix handling of view table types when detecting schema (@matthewnolf) + - `schema_table`: refactor cache config in schema_table collector (@cristiangreco) + - Component: add enable/disable collector configurability to `database_observability.mysql`. This removes the `query_samples_enabled` argument, now configurable via enable/disable collector. (@fridgepoet) + - Component: always log `instance` label key (@cristiangreco) + - Component: better error handling for collectors (@cristiangreco) + - Component: use labels for some indexed logs elements (@cristiangreco) + +- Reduce CPU usage of `loki.source.windowsevent` by up to 85% by updating the bookmark file every 10 seconds instead of after every event and by optimizing the retrieval of the process name. (@wildum) - Ensure consistent service_name label handling in `pyroscope.receive_http` to match Pyroscope's behavior. (@marcsanmi) @@ -64,7 +65,7 @@ Main (unreleased) ### Other changes - Upgrading to Prometheus v2.54.1. (@ptodev) - - `discovery.docker` has a new `match_first_network` attribute for matching the first network + - `discovery.docker` has a new `match_first_network` attribute for matching the first network if the container has multiple networks defined, thus avoiding collecting duplicate targets. - `discovery.ec2`, `discovery.kubernetes`, `discovery.openstack`, and `discovery.ovhcloud` add extra `__meta_` labels. @@ -72,7 +73,7 @@ Main (unreleased) - `discovery.linode` has a new `region` attribute, as well as extra `__meta_` labels. - A new `scrape_native_histograms` argument for `prometheus.scrape`. This is enabled by default and can be used to explicitly disable native histogram support. - In previous versions of Alloy, native histogram support has also been enabled by default + In previous versions of Alloy, native histogram support has also been enabled by default as long as `scrape_protocols` starts with `PrometheusProto`. v1.6.1 diff --git a/internal/component/database_observability/mysql/collector/schema_table.go b/internal/component/database_observability/mysql/collector/schema_table.go index 1d597311a3..1446231b15 100644 --- a/internal/component/database_observability/mysql/collector/schema_table.go +++ b/internal/component/database_observability/mysql/collector/schema_table.go @@ -48,7 +48,7 @@ const ( TABLE_SCHEMA = ?` // Note that the fully qualified table name is appendend to the query, - // for some reason it doesn't work with placeholders. + // we can't use placeholders with this statement. showCreateTable = `SHOW CREATE TABLE` selectColumnNames = ` @@ -275,7 +275,7 @@ func (c *SchemaTable) extractSchema(ctx context.Context) error { // TODO(cristian): consider moving this into the loop above for _, table := range tables { - fullyQualifiedTable := fmt.Sprintf("%s.%s", table.schema, table.tableName) + fullyQualifiedTable := fmt.Sprintf("`%s`.`%s`", table.schema, table.tableName) cacheKey := fmt.Sprintf("%s@%d", fullyQualifiedTable, table.updateTime.Unix()) cacheHit := false diff --git a/internal/component/database_observability/mysql/collector/schema_table_test.go b/internal/component/database_observability/mysql/collector/schema_table_test.go index abb7de8b90..cc952328bc 100644 --- a/internal/component/database_observability/mysql/collector/schema_table_test.go +++ b/internal/component/database_observability/mysql/collector/schema_table_test.go @@ -66,7 +66,7 @@ func TestSchemaTable(t *testing.T) { ), ) - mock.ExpectQuery("SHOW CREATE TABLE some_schema.some_table").WithoutArgs().RowsWillBeClosed(). + mock.ExpectQuery("SHOW CREATE TABLE `some_schema`.`some_table`").WithoutArgs().RowsWillBeClosed(). WillReturnRows( sqlmock.NewRows([]string{ "table_name", @@ -168,7 +168,7 @@ func TestSchemaTable(t *testing.T) { ), ) - mock.ExpectQuery("SHOW CREATE TABLE some_schema.some_table").WithoutArgs().RowsWillBeClosed(). + mock.ExpectQuery("SHOW CREATE TABLE `some_schema`.`some_table`").WithoutArgs().RowsWillBeClosed(). WillReturnRows( sqlmock.NewRows([]string{ "table_name", @@ -271,7 +271,7 @@ func TestSchemaTable(t *testing.T) { ), ) - mock.ExpectQuery("SHOW CREATE TABLE some_schema.some_table").WithoutArgs().RowsWillBeClosed(). + mock.ExpectQuery("SHOW CREATE TABLE `some_schema`.`some_table`").WithoutArgs().RowsWillBeClosed(). WillReturnRows( sqlmock.NewRows([]string{ "table_name", @@ -405,7 +405,7 @@ func TestSchemaTable(t *testing.T) { ), ) - mock.ExpectQuery("SHOW CREATE TABLE some_schema.some_table").WithoutArgs().RowsWillBeClosed(). + mock.ExpectQuery("SHOW CREATE TABLE `some_schema`.`some_table`").WithoutArgs().RowsWillBeClosed(). WillReturnRows( sqlmock.NewRows([]string{ "table_name", @@ -625,7 +625,7 @@ func TestSchemaTable(t *testing.T) { time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC), ), ) - mock.ExpectQuery("SHOW CREATE TABLE some_schema.some_table").WithoutArgs().WillReturnRows( + mock.ExpectQuery("SHOW CREATE TABLE `some_schema`.`some_table`").WithoutArgs().WillReturnRows( sqlmock.NewRows([]string{ "table_name", "create_statement", From e9195be37f263f45264c50fd87c4b06742575f3d Mon Sep 17 00:00:00 2001 From: Piotr <17101802+thampiotr@users.noreply.github.com> Date: Thu, 13 Feb 2025 16:46:53 +0000 Subject: [PATCH 03/14] dashboard prototyping fixes (#2721) * dash prototyping fixes * add env file support * restore --- example/.gitignore | 2 ++ example/README.md | 4 ++-- example/databases.yaml | 6 +++--- example/grafana.yaml | 10 +++++++--- example/images/grizzly/Dockerfile | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 example/.gitignore diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 0000000000..d15cc863a6 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1,2 @@ +# Add here any additional datasources for testing +config/grafana/datasources/datasource-dev.yml \ No newline at end of file diff --git a/example/README.md b/example/README.md index ac78113de7..18d40d9386 100644 --- a/example/README.md +++ b/example/README.md @@ -20,7 +20,7 @@ dashboards for monitoring Grafana Alloy. To start the environment, run: ```bash -docker compose up -d +docker compose up --build -d ``` To stop the environment, run: @@ -39,7 +39,7 @@ To run Alloy within Docker Compose, pass `--profile=alloy` to `docker compose` when starting and stopping the environment: ```bash -docker compose --profile=alloy up -d +docker compose --profile=alloy up --build -d ``` ```bash diff --git a/example/databases.yaml b/example/databases.yaml index 4ea8f972be..662ef28e4c 100644 --- a/example/databases.yaml +++ b/example/databases.yaml @@ -1,6 +1,6 @@ services: mimir: - image: grafana/mimir:2.12.0 + image: grafana/mimir:2.14.3 restart: on-failure command: - -config.file=/etc/mimir-config/mimir.yaml @@ -10,13 +10,13 @@ services: - "9009:9009" loki: - image: grafana/loki:3.0.0 + image: grafana/loki:3.4.1 restart: on-failure ports: - "3100:3100" tempo: - image: grafana/tempo:2.4.1 + image: grafana/tempo:2.7.0 restart: on-failure command: - "-storage.trace.backend=local" # tell tempo where to permanently put traces diff --git a/example/grafana.yaml b/example/grafana.yaml index 2bd25bb638..9d2f7411ee 100644 --- a/example/grafana.yaml +++ b/example/grafana.yaml @@ -1,6 +1,6 @@ services: grafana: - image: grafana/grafana:10.1.9 + image: grafana/grafana:11.5.1 restart: on-failure command: - --config=/etc/grafana-config/grafana.ini @@ -10,11 +10,15 @@ services: ports: - "3000:3000" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3000/healthz"] + test: [ "CMD", "curl", "-f", "http://localhost:3000/healthz" ] interval: 1s start_interval: 0s timeout: 10s retries: 5 + env_file: + # Use this optional env file to add any secrets required by data sources you can add to config/grafana/datasources + - path: ~/.grafana_dev_datasources + required: false install-dashboard-dependencies: build: images/jb @@ -53,6 +57,6 @@ services: volumes: - ../operations/alloy-mixin:/etc/alloy-mixin working_dir: /etc/alloy-mixin - command: grr watch dashboards/ grizzly/dashboards.jsonnet + command: grr watch . ./grizzly.jsonnet diff --git a/example/images/grizzly/Dockerfile b/example/images/grizzly/Dockerfile index 6c7c4cec5c..c0c4631066 100644 --- a/example/images/grizzly/Dockerfile +++ b/example/images/grizzly/Dockerfile @@ -1,3 +1,3 @@ FROM golang:1.23-alpine -RUN go install github.com/grafana/grizzly/cmd/grr@v0.4.3 +RUN go install github.com/grafana/grizzly/cmd/grr@v0.7.1 From 9f783e43f85ad2db697b3f92fbb0e54dd499c9d2 Mon Sep 17 00:00:00 2001 From: Piotr <17101802+thampiotr@users.noreply.github.com> Date: Thu, 13 Feb 2025 16:58:02 +0000 Subject: [PATCH 04/14] Optimise Prometheus targets handling (#2474) --- CHANGELOG.md | 2 + internal/component/beyla/ebpf/beyla_linux.go | 6 +- .../component/common/relabel/label_builder.go | 11 + internal/component/common/relabel/relabel.go | 118 ++- .../component/common/relabel/relabel_test.go | 942 +++++++++++++++++- .../database_observability/mysql/component.go | 6 +- internal/component/discovery/discovery.go | 53 +- .../component/discovery/discovery_test.go | 65 +- .../discovery/distributed_targets.go | 4 +- .../discovery/distributed_targets_test.go | 4 +- .../component/discovery/process/container.go | 6 +- .../component/discovery/process/discover.go | 7 +- internal/component/discovery/process/join.go | 15 +- .../component/discovery/process/join_test.go | 39 +- .../component/discovery/relabel/relabel.go | 38 +- .../discovery/relabel/relabel_test.go | 5 +- internal/component/discovery/target.go | 372 +++++++ .../component/discovery/target_builder.go | 189 ++++ .../discovery/target_builder_test.go | 381 +++++++ internal/component/discovery/target_test.go | 866 ++++++++++++++++ .../component/local/file_match/file_test.go | 21 +- internal/component/local/file_match/watch.go | 15 +- .../component/loki/process/process_test.go | 2 +- .../component/loki/relabel/relabel_test.go | 2 +- .../component/loki/source/docker/docker.go | 14 +- internal/component/loki/source/file/file.go | 19 +- .../component/loki/source/file/file_test.go | 33 +- .../loki/source/kubernetes/kubernetes.go | 5 +- .../loki/source/kubernetes/kubernetes_test.go | 14 +- internal/component/loki/write/write_test.go | 2 +- .../prometheus/exporter/blackbox/blackbox.go | 11 +- .../exporter/blackbox/blackbox_test.go | 42 +- .../component/prometheus/exporter/exporter.go | 12 +- .../prometheus/exporter/kafka/kafka.go | 11 +- .../prometheus/exporter/kafka/kafka_test.go | 7 +- .../prometheus/exporter/snmp/snmp.go | 17 +- .../prometheus/exporter/snmp/snmp_test.go | 21 +- .../component/prometheus/scrape/scrape.go | 47 +- .../scrape/scrape_clustering_test.go | 4 +- .../component/pyroscope/ebpf/ebpf_linux.go | 11 +- .../pyroscope/ebpf/ebpf_linux_test.go | 8 +- internal/component/pyroscope/java/java.go | 9 +- internal/component/pyroscope/java/loop.go | 11 +- internal/component/pyroscope/java/target.go | 17 +- internal/component/pyroscope/scrape/scrape.go | 20 +- .../pyroscope/scrape/scrape_loop_test.go | 9 +- .../component/pyroscope/scrape/scrape_test.go | 43 +- .../internal/common/convert_targets.go | 9 +- .../internal/common/convert_targets_test.go | 27 +- .../prometheusconvert/component/scrape.go | 2 +- .../internal/build/service_discovery.go | 2 +- .../converter/internal/test_common/testing.go | 7 +- .../runtime/componenttest/componenttest.go | 12 +- internal/runtime/equality/equality.go | 139 +++ internal/runtime/equality/equality_test.go | 282 ++++++ .../controller/node_builtin_component.go | 5 +- .../controller/node_custom_component.go | 4 +- .../internal/controller/node_service.go | 3 +- .../internal/controller/value_cache.go | 4 +- .../internal/importsource/import_file.go | 5 +- .../internal/importsource/import_git.go | 6 +- .../internal/importsource/import_http.go | 4 +- .../internal/importsource/import_string.go | 4 +- .../internal/testcomponents/module/git/git.go | 5 +- .../internal/testcomponents/module/module.go | 4 +- internal/service/ui/ui.go | 1 + .../promsdprocessor/consumer/consumer.go | 34 +- .../promsdprocessor/consumer/consumer_test.go | 9 +- .../promsdprocessor/prom_sd_processor.go | 6 +- .../promsdprocessor/prom_sd_processor_test.go | 15 +- internal/util/testtarget/test_target.go | 4 +- syntax/encoding/alloyjson/alloyjson.go | 52 +- syntax/encoding/alloyjson/alloyjson_test.go | 43 +- syntax/internal/value/value.go | 29 +- syntax/token/builder/builder_test.go | 60 +- syntax/token/builder/value_tokens.go | 66 +- syntax/types.go | 6 + 77 files changed, 3845 insertions(+), 560 deletions(-) create mode 100644 internal/component/common/relabel/label_builder.go create mode 100644 internal/component/discovery/target.go create mode 100644 internal/component/discovery/target_builder.go create mode 100644 internal/component/discovery/target_builder_test.go create mode 100644 internal/component/discovery/target_test.go create mode 100644 internal/runtime/equality/equality.go create mode 100644 internal/runtime/equality/equality_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f014414a..2713a5e720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ Main (unreleased) - Ensure consistent service_name label handling in `pyroscope.receive_http` to match Pyroscope's behavior. (@marcsanmi) +- Improved memory and CPU performance of Prometheus pipelines by changing the underlying implementation of targets (@thampiotr) + ### Bugfixes - Fix log rotation for Windows in `loki.source.file` by refactoring the component to use the runner pkg. This should also reduce CPU consumption when tailing a lot of files in a dynamic environment. (@wildum) diff --git a/internal/component/beyla/ebpf/beyla_linux.go b/internal/component/beyla/ebpf/beyla_linux.go index d47845d2c1..a79ab82077 100644 --- a/internal/component/beyla/ebpf/beyla_linux.go +++ b/internal/component/beyla/ebpf/beyla_linux.go @@ -273,17 +273,17 @@ func (c *Component) Update(args component.Arguments) error { func (c *Component) baseTarget() (discovery.Target, error) { data, err := c.opts.GetServiceData(http_service.ServiceName) if err != nil { - return nil, fmt.Errorf("failed to get HTTP information: %w", err) + return discovery.EmptyTarget, fmt.Errorf("failed to get HTTP information: %w", err) } httpData := data.(http_service.Data) - return discovery.Target{ + return discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: httpData.MemoryListenAddr, model.SchemeLabel: "http", model.MetricsPathLabel: path.Join(httpData.HTTPPathForComponent(c.opts.ID), "metrics"), "instance": defaultInstance(), "job": "beyla", - }, nil + }), nil } func (c *Component) reportUnhealthy(err error) { diff --git a/internal/component/common/relabel/label_builder.go b/internal/component/common/relabel/label_builder.go new file mode 100644 index 0000000000..e3028df50c --- /dev/null +++ b/internal/component/common/relabel/label_builder.go @@ -0,0 +1,11 @@ +package relabel + +// LabelBuilder is an interface that can be used to change labels with relabel logic. +type LabelBuilder interface { + // Get returns given label value. If label is not present, an empty string is returned. + Get(label string) string + Range(f func(label string, value string)) + // Set will set given label to given value. Setting to empty value is equivalent to deleting this label. + Set(label string, val string) + Del(ns ...string) +} diff --git a/internal/component/common/relabel/relabel.go b/internal/component/common/relabel/relabel.go index 4bd3fb2970..f467ff3854 100644 --- a/internal/component/common/relabel/relabel.go +++ b/internal/component/common/relabel/relabel.go @@ -1,8 +1,27 @@ +// NOTE: this file is copied from Prometheus codebase and adapted to work correctly with Alloy types. +// For backwards compatibility purposes, the behaviour implemented here should not be changed. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package relabel import ( + "crypto/md5" + "encoding/binary" "fmt" "reflect" + "strings" "github.com/grafana/regexp" "github.com/prometheus/common/model" @@ -149,7 +168,13 @@ func (rc *Config) Validate() error { if (rc.Action == Replace || rc.Action == HashMod || rc.Action == Lowercase || rc.Action == Uppercase || rc.Action == KeepEqual || rc.Action == DropEqual) && rc.TargetLabel == "" { return fmt.Errorf("relabel configuration for %s action requires 'target_label' value", rc.Action) } - if (rc.Action == Replace || rc.Action == Lowercase || rc.Action == Uppercase || rc.Action == KeepEqual || rc.Action == DropEqual) && !relabelTarget.MatchString(rc.TargetLabel) { + if rc.Action == Replace && !strings.Contains(rc.TargetLabel, "$") && !model.LabelName(rc.TargetLabel).IsValid() { + return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action) + } + if rc.Action == Replace && strings.Contains(rc.TargetLabel, "$") && !relabelTarget.MatchString(rc.TargetLabel) { + return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action) + } + if (rc.Action == Lowercase || rc.Action == Uppercase || rc.Action == KeepEqual || rc.Action == DropEqual) && !model.LabelName(rc.TargetLabel).IsValid() { return fmt.Errorf("%q is invalid 'target_label' for %s action", rc.TargetLabel, rc.Action) } if (rc.Action == Lowercase || rc.Action == Uppercase || rc.Action == KeepEqual || rc.Action == DropEqual) && rc.Replacement != DefaultRelabelConfig.Replacement { @@ -186,6 +211,97 @@ func (rc *Config) Validate() error { return nil } +// ProcessBuilder should be called with lb LabelBuilder containing the initial set of labels, +// which are then modified following the configured rules using builder's methods such as Set and Del. +func ProcessBuilder(lb LabelBuilder, cfgs ...*Config) (keep bool) { + for _, cfg := range cfgs { + keep = doRelabel(cfg, lb) + if !keep { + return false + } + } + return true +} + +func doRelabel(cfg *Config, lb LabelBuilder) (keep bool) { + var va [16]string + values := va[:0] + if len(cfg.SourceLabels) > cap(values) { + values = make([]string, 0, len(cfg.SourceLabels)) + } + for _, ln := range cfg.SourceLabels { + values = append(values, lb.Get(string(ln))) + } + val := strings.Join(values, cfg.Separator) + + switch cfg.Action { + case Drop: + if cfg.Regex.MatchString(val) { + return false + } + case Keep: + if !cfg.Regex.MatchString(val) { + return false + } + case DropEqual: + if lb.Get(cfg.TargetLabel) == val { + return false + } + case KeepEqual: + if lb.Get(cfg.TargetLabel) != val { + return false + } + case Replace: + indexes := cfg.Regex.FindStringSubmatchIndex(val) + // If there is no match no replacement must take place. + if indexes == nil { + break + } + target := model.LabelName(cfg.Regex.ExpandString([]byte{}, cfg.TargetLabel, val, indexes)) + if !target.IsValid() { + break + } + res := cfg.Regex.ExpandString([]byte{}, cfg.Replacement, val, indexes) + if len(res) == 0 { + lb.Del(string(target)) + break + } + lb.Set(string(target), string(res)) + case Lowercase: + lb.Set(cfg.TargetLabel, strings.ToLower(val)) + case Uppercase: + lb.Set(cfg.TargetLabel, strings.ToUpper(val)) + case HashMod: + hash := md5.Sum([]byte(val)) + // Use only the last 8 bytes of the hash to give the same result as earlier versions of this code. + mod := binary.BigEndian.Uint64(hash[8:]) % cfg.Modulus + lb.Set(cfg.TargetLabel, fmt.Sprintf("%d", mod)) + case LabelMap: + lb.Range(func(name, value string) { + if cfg.Regex.MatchString(name) { + res := cfg.Regex.ReplaceAllString(name, cfg.Replacement) + lb.Set(res, value) + } + }) + case LabelDrop: + lb.Range(func(name, value string) { + if cfg.Regex.MatchString(name) { + lb.Del(name) + } + }) + case LabelKeep: + lb.Range(func(name, value string) { + if !cfg.Regex.MatchString(name) { + lb.Del(name) + } + }) + default: + panic(fmt.Errorf("relabel: unknown relabel action type %q", cfg.Action)) + } + + return true +} + // ComponentToPromRelabelConfigs bridges the Component-based configuration of // relabeling steps to the Prometheus implementation. func ComponentToPromRelabelConfigs(rcs []*Config) []*relabel.Config { diff --git a/internal/component/common/relabel/relabel_test.go b/internal/component/common/relabel/relabel_test.go index 3138870f1b..af83e0226a 100644 --- a/internal/component/common/relabel/relabel_test.go +++ b/internal/component/common/relabel/relabel_test.go @@ -1,69 +1,905 @@ +// NOTE: this file is copied from Prometheus codebase and adapted to work correctly with Alloy types. + +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package relabel import ( + "fmt" "testing" + "github.com/grafana/regexp" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" - "github.com/grafana/alloy/syntax" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/util/testutil" ) -func TestParseConfig(t *testing.T) { - for _, tt := range []struct { - name string - cfg string - expectErr bool +func TestRelabel(t *testing.T) { + tests := []struct { + input labels.Labels + relabel []*Config + output labels.Labels + drop bool }{ { - name: "valid keepequal config", - cfg: ` - action = "keepequal" - target_label = "foo" - source_labels = ["bar"] - `, - }, - { - name: "valid dropequal config", - cfg: ` - action = "dropequal" - target_label = "foo" - source_labels = ["bar"] - `, - }, - { - name: "missing dropequal target", - cfg: ` - action = "dropequal" - source_labels = ["bar"] - `, - expectErr: true, - }, - { - name: "missing keepequal target", - cfg: ` - action = "keepequal" - source_labels = ["bar"] - `, - expectErr: true, - }, - { - name: "unknown action", - cfg: ` - action = "foo" - `, - expectErr: true, - }, - } { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - var cfg Config - err := syntax.Unmarshal([]byte(tt.cfg), &cfg) - if tt.expectErr { - require.Error(t, err) - } else { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b": "bar", + "c": "baz", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("f(.*)"), + TargetLabel: "d", + Separator: ";", + Replacement: "ch${1}-ch${1}", + Action: Replace, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + "b": "bar", + "c": "baz", + "d": "choo-choo", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b": "bar", + "c": "baz", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a", "b"}, + Regex: MustNewRegexp("f(.*);(.*)r"), + TargetLabel: "a", + Separator: ";", + Replacement: "b${1}${2}m", // boobam + Action: Replace, + }, + { + SourceLabels: []string{"c", "a"}, + Regex: MustNewRegexp("(b).*b(.*)ba(.*)"), + TargetLabel: "d", + Separator: ";", + Replacement: "$1$2$2$3", + Action: Replace, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "boobam", + "b": "bar", + "c": "baz", + "d": "boooom", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp(".*o.*"), + Action: Drop, + }, { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("f(.*)"), + TargetLabel: "d", + Separator: ";", + Replacement: "ch$1-ch$1", + Action: Replace, + }, + }, + drop: true, + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b": "bar", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp(".*o.*"), + Action: Drop, + }, + }, + drop: true, + }, + { + input: labels.FromMap(map[string]string{ + "a": "abc", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp(".*(b).*"), + TargetLabel: "d", + Separator: ";", + Replacement: "$1", + Action: Replace, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "abc", + "d": "b", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("no-match"), + Action: Drop, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("f|o"), + Action: Drop, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("no-match"), + Action: Keep, + }, + }, + drop: true, + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("f.*"), + Action: Keep, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + }), + }, + { + // No replacement must be applied if there is no match. + input: labels.FromMap(map[string]string{ + "a": "boo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("f"), + TargetLabel: "b", + Replacement: "bar", + Action: Replace, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "boo", + }), + }, + { + // Blank replacement should delete the label. + input: labels.FromMap(map[string]string{ + "a": "foo", + "f": "baz", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("(f).*"), + TargetLabel: "$1", + Replacement: "$2", + Action: Replace, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b": "bar", + "c": "baz", + }), + relabel: []*Config{ + { + SourceLabels: []string{"c"}, + TargetLabel: "d", + Separator: ";", + Action: HashMod, + Modulus: 1000, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + "b": "bar", + "c": "baz", + "d": "976", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo\nbar", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + TargetLabel: "b", + Separator: ";", + Action: HashMod, + Modulus: 1000, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo\nbar", + "b": "734", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b1": "bar", + "b2": "baz", + }), + relabel: []*Config{ + { + Regex: MustNewRegexp("(b.*)"), + Replacement: "bar_${1}", + Action: LabelMap, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + "b1": "bar", + "b2": "baz", + "bar_b1": "bar", + "bar_b2": "baz", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "__meta_my_bar": "aaa", + "__meta_my_baz": "bbb", + "__meta_other": "ccc", + }), + relabel: []*Config{ + { + Regex: MustNewRegexp("__meta_(my.*)"), + Replacement: "${1}", + Action: LabelMap, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + "__meta_my_bar": "aaa", + "__meta_my_baz": "bbb", + "__meta_other": "ccc", + "my_bar": "aaa", + "my_baz": "bbb", + }), + }, + { // valid case + input: labels.FromMap(map[string]string{ + "a": "some-name-value", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${2}", + TargetLabel: "${1}", + }, + }, + output: labels.FromMap(map[string]string{ + "a": "some-name-value", + "name": "value", + }), + }, + { // invalid replacement "" + input: labels.FromMap(map[string]string{ + "a": "some-name-value", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${3}", + TargetLabel: "${1}", + }, + }, + output: labels.FromMap(map[string]string{ + "a": "some-name-value", + }), + }, + { // invalid target_labels + input: labels.FromMap(map[string]string{ + "a": "some-name-0", + }), + relabel: []*Config{ + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "${3}", + }, + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "${3}", + }, + { + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)(-[^,]+)"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "${3}", + }, + }, + output: labels.FromMap(map[string]string{ + "a": "some-name-0", + }), + }, + { // more complex real-life like usecase + input: labels.FromMap(map[string]string{ + "__meta_sd_tags": "path:/secret,job:some-job,label:foo=bar", + }), + relabel: []*Config{ + { + SourceLabels: []string{"__meta_sd_tags"}, + Regex: MustNewRegexp("(?:.+,|^)path:(/[^,]+).*"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "__metrics_path__", + }, + { + SourceLabels: []string{"__meta_sd_tags"}, + Regex: MustNewRegexp("(?:.+,|^)job:([^,]+).*"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "job", + }, + { + SourceLabels: []string{"__meta_sd_tags"}, + Regex: MustNewRegexp("(?:.+,|^)label:([^=]+)=([^,]+).*"), + Action: Replace, + Replacement: "${2}", + TargetLabel: "${1}", + }, + }, + output: labels.FromMap(map[string]string{ + "__meta_sd_tags": "path:/secret,job:some-job,label:foo=bar", + "__metrics_path__": "/secret", + "job": "some-job", + "foo": "bar", + }), + }, + { // From https://github.com/prometheus/prometheus/issues/12283 + input: labels.FromMap(map[string]string{ + "__meta_kubernetes_pod_container_port_name": "foo", + "__meta_kubernetes_pod_annotation_XXX_metrics_port": "9091", + }), + relabel: []*Config{ + { + Regex: MustNewRegexp("^__meta_kubernetes_pod_container_port_name$"), + Action: LabelDrop, + }, + { + SourceLabels: []string{"__meta_kubernetes_pod_annotation_XXX_metrics_port"}, + Regex: MustNewRegexp("(.+)"), + Action: Replace, + Replacement: "metrics", + TargetLabel: "__meta_kubernetes_pod_container_port_name", + }, + { + SourceLabels: []string{"__meta_kubernetes_pod_container_port_name"}, + Regex: MustNewRegexp("^metrics$"), + Action: Keep, + }, + }, + output: labels.FromMap(map[string]string{ + "__meta_kubernetes_pod_annotation_XXX_metrics_port": "9091", + "__meta_kubernetes_pod_container_port_name": "metrics", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b1": "bar", + "b2": "baz", + }), + relabel: []*Config{ + { + Regex: MustNewRegexp("(b.*)"), + Action: LabelKeep, + }, + }, + output: labels.FromMap(map[string]string{ + "b1": "bar", + "b2": "baz", + }), + }, + { + input: labels.FromMap(map[string]string{ + "a": "foo", + "b1": "bar", + "b2": "baz", + }), + relabel: []*Config{ + { + Regex: MustNewRegexp("(b.*)"), + Action: LabelDrop, + }, + }, + output: labels.FromMap(map[string]string{ + "a": "foo", + }), + }, + { + input: labels.FromMap(map[string]string{ + "foo": "bAr123Foo", + }), + relabel: []*Config{ + { + SourceLabels: []string{"foo"}, + Action: Uppercase, + TargetLabel: "foo_uppercase", + }, + { + SourceLabels: []string{"foo"}, + Action: Lowercase, + TargetLabel: "foo_lowercase", + }, + }, + output: labels.FromMap(map[string]string{ + "foo": "bAr123Foo", + "foo_lowercase": "bar123foo", + "foo_uppercase": "BAR123FOO", + }), + }, + { + input: labels.FromMap(map[string]string{ + "__tmp_port": "1234", + "__port1": "1234", + "__port2": "5678", + }), + relabel: []*Config{ + { + SourceLabels: []string{"__tmp_port"}, + Action: KeepEqual, + TargetLabel: "__port1", + }, + }, + output: labels.FromMap(map[string]string{ + "__tmp_port": "1234", + "__port1": "1234", + "__port2": "5678", + }), + }, + { + input: labels.FromMap(map[string]string{ + "__tmp_port": "1234", + "__port1": "1234", + "__port2": "5678", + }), + relabel: []*Config{ + { + SourceLabels: []string{"__tmp_port"}, + Action: DropEqual, + TargetLabel: "__port1", + }, + }, + drop: true, + }, + { + input: labels.FromMap(map[string]string{ + "__tmp_port": "1234", + "__port1": "1234", + "__port2": "5678", + }), + relabel: []*Config{ + { + SourceLabels: []string{"__tmp_port"}, + Action: DropEqual, + TargetLabel: "__port2", + }, + }, + output: labels.FromMap(map[string]string{ + "__tmp_port": "1234", + "__port1": "1234", + "__port2": "5678", + }), + }, + { + input: labels.FromMap(map[string]string{ + "__tmp_port": "1234", + "__port1": "1234", + "__port2": "5678", + }), + relabel: []*Config{ + { + SourceLabels: []string{"__tmp_port"}, + Action: KeepEqual, + TargetLabel: "__port2", + }, + }, + drop: true, + }, + } + + for _, test := range tests { + // Setting default fields, mimicking the behaviour in Prometheus. + for _, cfg := range test.relabel { + if cfg.Action == "" { + cfg.Action = DefaultRelabelConfig.Action + } + if cfg.Separator == "" { + cfg.Separator = DefaultRelabelConfig.Separator + } + if cfg.Regex.Regexp == nil || cfg.Regex.String() == "" { + cfg.Regex = DefaultRelabelConfig.Regex + } + if cfg.Replacement == "" { + cfg.Replacement = DefaultRelabelConfig.Replacement + } + require.NoError(t, cfg.Validate()) + } + + builder := newBuilder(test.input) + keep := ProcessBuilder(builder, test.relabel...) + require.Equal(t, !test.drop, keep) + if keep { + testutil.RequireEqual(t, test.output, builder.Labels()) + } + } +} + +func TestRelabelValidate(t *testing.T) { + tests := []struct { + config Config + expected string + }{ + { + config: Config{}, + expected: `relabel action cannot be empty`, + }, + { + config: Config{ + Action: Replace, + }, + expected: `requires 'target_label' value`, + }, + { + config: Config{ + Action: Lowercase, + }, + expected: `requires 'target_label' value`, + }, + { + config: Config{ + Action: Lowercase, + Replacement: DefaultRelabelConfig.Replacement, + TargetLabel: "${3}", + }, + expected: `"${3}" is invalid 'target_label'`, + }, + { + config: Config{ + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "${3}", + }, + }, + { + config: Config{ + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "0${3}", + }, + expected: `"0${3}" is invalid 'target_label'`, + }, + { + config: Config{ + SourceLabels: []string{"a"}, + Regex: MustNewRegexp("some-([^-]+)-([^,]+)"), + Action: Replace, + Replacement: "${1}", + TargetLabel: "-${3}", + }, + expected: `"-${3}" is invalid 'target_label' for replace action`, + }, + } + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + err := test.config.Validate() + if test.expected == "" { require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.expected) } }) } } + +func TestTargetLabelValidity(t *testing.T) { + tests := []struct { + str string + valid bool + }{ + {"-label", false}, + {"label", true}, + {"label${1}", true}, + {"${1}label", true}, + {"${1}", true}, + {"${1}label", true}, + {"${", false}, + {"$", false}, + {"${}", false}, + {"foo${", false}, + {"$1", true}, + {"asd$2asd", true}, + {"-foo${1}bar-", false}, + {"_${1}_", true}, + {"foo${bar}foo", true}, + } + for _, test := range tests { + require.Equal(t, test.valid, relabelTarget.Match([]byte(test.str)), + "Expected %q to be %v", test.str, test.valid) + } +} + +func BenchmarkRelabel(b *testing.B) { + tests := []struct { + name string + lbls labels.Labels + config string + cfgs []*Config + }{ + { + name: "example", // From prometheus/config/testdata/conf.good.yml. + config: ` + - source_labels: [job, __meta_dns_name] + regex: "(.*)some-[regex]" + target_label: job + replacement: foo-${1} + # action defaults to 'replace' + - source_labels: [abc] + target_label: cde + - replacement: static + target_label: abc + - regex: + replacement: static + target_label: abc`, + lbls: labels.FromStrings("__meta_dns_name", "example-some-x.com", "abc", "def", "job", "foo"), + }, + { + name: "kubernetes", + config: ` + - source_labels: + - __meta_kubernetes_pod_container_port_name + regex: .*-metrics + action: keep + - source_labels: + - __meta_kubernetes_pod_label_name + action: drop + regex: "" + - source_labels: + - __meta_kubernetes_pod_phase + regex: Succeeded|Failed + action: drop + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + regex: "false" + action: drop + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + regex: (https?) + replacement: $1 + action: replace + - source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + regex: (.+) + replacement: $1 + action: replace + - source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + regex: (.+?)(\:\d+)?;(\d+) + replacement: $1:$3 + action: replace + - regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + action: labelmap + - regex: __meta_kubernetes_pod_label_prometheus_io_label_(.+) + action: labelmap + - regex: __meta_kubernetes_pod_annotation_prometheus_io_label_(.+) + action: labelmap + - source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_label_name + separator: / + target_label: job + replacement: $1 + action: replace + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + action: replace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + action: replace + - source_labels: + - __meta_kubernetes_pod_container_name + target_label: container + action: replace + - source_labels: + - __meta_kubernetes_pod_name + - __meta_kubernetes_pod_container_name + - __meta_kubernetes_pod_container_port_name + separator: ':' + target_label: instance + action: replace + - target_label: cluster + replacement: dev-us-central-0 + - source_labels: + - __meta_kubernetes_namespace + regex: hosted-grafana + action: drop + - source_labels: + - __address__ + target_label: __tmp_hash + modulus: 3 + action: hashmod + - source_labels: + - __tmp_hash + regex: ^0$ + action: keep + - regex: __tmp_hash + action: labeldrop`, + lbls: labels.FromStrings( + "__address__", "10.132.183.40:80", + "__meta_kubernetes_namespace", "loki-boltdb-shipper", + "__meta_kubernetes_pod_annotation_promtail_loki_boltdb_shipper_hash", "50523b9759094a144adcec2eae0aa4ad", + "__meta_kubernetes_pod_annotationpresent_promtail_loki_boltdb_shipper_hash", "true", + "__meta_kubernetes_pod_container_init", "false", + "__meta_kubernetes_pod_container_name", "promtail", + "__meta_kubernetes_pod_container_port_name", "http-metrics", + "__meta_kubernetes_pod_container_port_number", "80", + "__meta_kubernetes_pod_container_port_protocol", "TCP", + "__meta_kubernetes_pod_controller_kind", "DaemonSet", + "__meta_kubernetes_pod_controller_name", "promtail-loki-boltdb-shipper", + "__meta_kubernetes_pod_host_ip", "10.128.0.178", + "__meta_kubernetes_pod_ip", "10.132.183.40", + "__meta_kubernetes_pod_label_controller_revision_hash", "555b77cd7d", + "__meta_kubernetes_pod_label_name", "promtail-loki-boltdb-shipper", + "__meta_kubernetes_pod_label_pod_template_generation", "45", + "__meta_kubernetes_pod_labelpresent_controller_revision_hash", "true", + "__meta_kubernetes_pod_labelpresent_name", "true", + "__meta_kubernetes_pod_labelpresent_pod_template_generation", "true", + "__meta_kubernetes_pod_name", "promtail-loki-boltdb-shipper-jgtr7", + "__meta_kubernetes_pod_node_name", "gke-dev-us-central-0-main-n2s8-2-14d53341-9hkr", + "__meta_kubernetes_pod_phase", "Running", + "__meta_kubernetes_pod_ready", "true", + "__meta_kubernetes_pod_uid", "4c586419-7f6c-448d-aeec-ca4fa5b05e60", + "__metrics_path__", "/metrics", + "__scheme__", "http", + "__scrape_interval__", "15s", + "__scrape_timeout__", "10s", + "job", "kubernetes-pods"), + }, + } + for i := range tests { + err := yaml.UnmarshalStrict([]byte(tests[i].config), &tests[i].cfgs) + require.NoError(b, err) + } + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + tb := newBuilder(tt.lbls) + _ = ProcessBuilder(tb, tt.cfgs...) + } + }) + } +} + +// MustNewRegexp works like NewRegexp, but panics if the regular expression does not compile. +func MustNewRegexp(s string) Regexp { + re, err := NewRegexp(s) + if err != nil { + panic(err) + } + return re +} + +// NewRegexp creates a new anchored Regexp and returns an error if the +// passed-in regular expression does not compile. +func NewRegexp(s string) (Regexp, error) { + regex, err := regexp.Compile("^(?:" + s + ")$") + return Regexp{Regexp: regex}, err +} + +type builderAdapter struct { + b *labels.Builder +} + +func newBuilder(ls labels.Labels) *builderAdapter { + return &builderAdapter{ + b: labels.NewBuilder(ls), + } +} + +func (b *builderAdapter) Get(label string) string { + return b.b.Get(label) +} + +func (b *builderAdapter) Range(f func(label string, value string)) { + b.b.Range(func(l labels.Label) { + f(l.Name, l.Value) + }) +} + +func (b *builderAdapter) Set(label string, val string) { + b.b.Set(label, val) +} + +func (b *builderAdapter) Del(ns ...string) { + b.b = b.b.Del(ns...) +} + +func (b *builderAdapter) Labels() labels.Labels { + return b.b.Labels() +} diff --git a/internal/component/database_observability/mysql/component.go b/internal/component/database_observability/mysql/component.go index 515c107fd9..f86427d8e8 100644 --- a/internal/component/database_observability/mysql/component.go +++ b/internal/component/database_observability/mysql/component.go @@ -163,17 +163,17 @@ func (c *Component) Run(ctx context.Context) error { func (c *Component) getBaseTarget() (discovery.Target, error) { data, err := c.opts.GetServiceData(http_service.ServiceName) if err != nil { - return nil, fmt.Errorf("failed to get HTTP information: %w", err) + return discovery.EmptyTarget, fmt.Errorf("failed to get HTTP information: %w", err) } httpData := data.(http_service.Data) - return discovery.Target{ + return discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: httpData.MemoryListenAddr, model.SchemeLabel: "http", model.MetricsPathLabel: path.Join(httpData.HTTPPathForComponent(c.opts.ID), "metrics"), "instance": c.instanceKey, "job": database_observability.JobName, - }, nil + }), nil } func (c *Component) Update(args component.Arguments) error { diff --git a/internal/component/discovery/discovery.go b/internal/component/discovery/discovery.go index 03670814e1..9be7520312 100644 --- a/internal/component/discovery/discovery.go +++ b/internal/component/discovery/discovery.go @@ -3,58 +3,17 @@ package discovery import ( "context" "fmt" - "slices" - "sort" - "strings" "sync" "time" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/discovery/targetgroup" - "github.com/prometheus/prometheus/model/labels" "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/service/livedebugging" ) -// Target refers to a singular discovered endpoint found by a discovery -// component. -type Target map[string]string - -// Labels converts Target into a set of sorted labels. -func (t Target) Labels() labels.Labels { - var lset labels.Labels - for k, v := range t { - lset = append(lset, labels.Label{Name: k, Value: v}) - } - sort.Sort(lset) - return lset -} - -func (t Target) NonMetaLabels() labels.Labels { - var lset labels.Labels - for k, v := range t { - if !strings.HasPrefix(k, model.MetaLabelPrefix) { - lset = append(lset, labels.Label{Name: k, Value: v}) - } - } - sort.Sort(lset) - return lset -} - -func (t Target) SpecificLabels(lbls []string) labels.Labels { - var lset labels.Labels - for k, v := range t { - if slices.Contains(lbls, k) { - lset = append(lset, labels.Label{Name: k, Value: v}) - } - } - sort.Sort(lset) - return lset -} - // Exports holds values which are exported by all discovery components. type Exports struct { Targets []Target `alloy:"targets,attr"` @@ -269,17 +228,7 @@ func toAlloyTargets(cache map[string]*targetgroup.Group) []Target { for _, group := range cache { for _, target := range group.Targets { - tLabels := make(map[string]string, len(group.Labels)+len(target)) - - // first add the group labels, and then the - // target labels, so that target labels take precedence. - for k, v := range group.Labels { - tLabels[string(k)] = string(v) - } - for k, v := range target { - tLabels[string(k)] = string(v) - } - allTargets = append(allTargets, tLabels) + allTargets = append(allTargets, NewTargetFromSpecificAndBaseLabelSet(target, group.Labels)) } } return allTargets diff --git a/internal/component/discovery/discovery_test.go b/internal/component/discovery/discovery_test.go index 71927dba95..407ff5d7ef 100644 --- a/internal/component/discovery/discovery_test.go +++ b/internal/component/discovery/discovery_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/service/livedebugging" ) @@ -40,21 +41,21 @@ var updateTestCases = []discovererUpdateTestCase{ {Source: "test", Labels: model.LabelSet{"test_key": "value"}, Targets: []model.LabelSet{{"foo": "bar"}}}, }, expectedInitialExports: []component.Exports{ - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial export }, updatedTargets: []*targetgroup.Group{ {Source: "test", Labels: model.LabelSet{"test_key_2": "value"}, Targets: []model.LabelSet{{"baz": "bux"}}}, }, expectedUpdatedExports: []component.Exports{ - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial export - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial re-published on shutdown - Exports{Targets: []Target{{"test_key_2": "value", "baz": "bux"}}}, // Updated export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial re-published on shutdown + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"test_key_2": "value", "baz": "bux"})}}, // Updated export }, expectedFinalExports: []component.Exports{ - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial export - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial re-published on shutdown - Exports{Targets: []Target{{"test_key_2": "value", "baz": "bux"}}}, // Updated export - Exports{Targets: []Target{{"test_key_2": "value", "baz": "bux"}}}, // Updated re-published on shutdown + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial re-published on shutdown + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"test_key_2": "value", "baz": "bux"})}}, // Updated export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"test_key_2": "value", "baz": "bux"})}}, // Updated re-published on shutdown }, }, { @@ -86,15 +87,15 @@ var updateTestCases = []discovererUpdateTestCase{ {Source: "test", Labels: model.LabelSet{"test_key_2": "value"}, Targets: []model.LabelSet{{"baz": "bux"}}}, }, expectedUpdatedExports: []component.Exports{ - Exports{Targets: []Target{}}, // Initial publish - Exports{Targets: []Target{}}, // Initial re-published on shutdown - Exports{Targets: []Target{{"test_key_2": "value", "baz": "bux"}}}, // Updated export. + Exports{Targets: []Target{}}, // Initial publish + Exports{Targets: []Target{}}, // Initial re-published on shutdown + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"test_key_2": "value", "baz": "bux"})}}, // Updated export. }, expectedFinalExports: []component.Exports{ - Exports{Targets: []Target{}}, // Initial publish - Exports{Targets: []Target{}}, // Initial re-published on shutdown - Exports{Targets: []Target{{"test_key_2": "value", "baz": "bux"}}}, // Updated export. - Exports{Targets: []Target{{"test_key_2": "value", "baz": "bux"}}}, // Updated export re-published on shutdown. + Exports{Targets: []Target{}}, // Initial publish + Exports{Targets: []Target{}}, // Initial re-published on shutdown + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"test_key_2": "value", "baz": "bux"})}}, // Updated export. + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"test_key_2": "value", "baz": "bux"})}}, // Updated export re-published on shutdown. }, }, { @@ -103,19 +104,19 @@ var updateTestCases = []discovererUpdateTestCase{ {Source: "test", Labels: model.LabelSet{"test_key": "value"}, Targets: []model.LabelSet{{"foo": "bar"}}}, }, expectedInitialExports: []component.Exports{ - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial export }, updatedTargets: nil, expectedUpdatedExports: []component.Exports{ - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial export - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial re-published on shutdown - Exports{Targets: []Target{}}, // Updated export should publish empty! + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial re-published on shutdown + Exports{Targets: []Target{}}, // Updated export should publish empty! }, expectedFinalExports: []component.Exports{ - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial export - Exports{Targets: []Target{{"foo": "bar", "test_key": "value"}}}, // Initial re-published on shutdown - Exports{Targets: []Target{}}, // Updated export should publish empty! - Exports{Targets: []Target{}}, // Updated re-published on shutdown + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial export + Exports{Targets: []Target{NewTargetFromMap(map[string]string{"foo": "bar", "test_key": "value"})}}, // Initial re-published on shutdown + Exports{Targets: []Target{}}, // Updated export should publish empty! + Exports{Targets: []Target{}}, // Updated re-published on shutdown }, }, } @@ -175,7 +176,7 @@ func TestDiscoveryUpdates(t *testing.T) { require.EventuallyWithT(t, func(t *assert.CollectT) { publishedExportsMut.Lock() defer publishedExportsMut.Unlock() - assert.Equal(t, tc.expectedInitialExports, publishedExports) + assertExportsEqual(t, tc.expectedInitialExports, publishedExports) }, 3*time.Second, time.Millisecond) discoverer = newFakeDiscoverer() @@ -188,7 +189,7 @@ func TestDiscoveryUpdates(t *testing.T) { require.EventuallyWithT(t, func(t *assert.CollectT) { publishedExportsMut.Lock() defer publishedExportsMut.Unlock() - assert.Equal(t, tc.expectedUpdatedExports, publishedExports) + assertExportsEqual(t, tc.expectedUpdatedExports, publishedExports) }, 3*time.Second, time.Millisecond) ctxCancel() @@ -197,12 +198,24 @@ func TestDiscoveryUpdates(t *testing.T) { require.EventuallyWithT(t, func(t *assert.CollectT) { publishedExportsMut.Lock() defer publishedExportsMut.Unlock() - assert.Equal(t, tc.expectedFinalExports, publishedExports) + assertExportsEqual(t, tc.expectedFinalExports, publishedExports) }, 3*time.Second, time.Millisecond) }) } } +func assertExportsEqual(t *assert.CollectT, expected []component.Exports, actual []component.Exports) { + if actual == nil { + assert.NotNil(t, actual) + return + } + equal := equality.DeepEqual(expected, actual) + assert.True(t, equal, "expected and actual exports are different") + if !equal { // also do assert.Equal to get a nice diff view if there is an issue. + assert.Equal(t, expected, actual) + } +} + /* on darwin/arm64/Apple M2: Benchmark_ToAlloyTargets-8 150 7549967 ns/op 12768249 B/op 40433 allocs/op diff --git a/internal/component/discovery/distributed_targets.go b/internal/component/discovery/distributed_targets.go index 5e776f6b40..d4dc858011 100644 --- a/internal/component/discovery/distributed_targets.go +++ b/internal/component/discovery/distributed_targets.go @@ -99,11 +99,11 @@ func (dt *DistributedTargets) MovedToRemoteInstance(prev *DistributedTargets) [] } func keyFor(tgt Target) shard.Key { - return shard.Key(tgt.NonMetaLabels().Hash()) + return shard.Key(tgt.NonMetaLabelsHash()) } func keyForLabels(tgt Target, lbls []string) shard.Key { - return shard.Key(tgt.SpecificLabels(lbls).Hash()) + return shard.Key(tgt.SpecificLabelsHash(lbls)) } type disabledCluster struct{} diff --git a/internal/component/discovery/distributed_targets_test.go b/internal/component/discovery/distributed_targets_test.go index fb810859aa..bac2fae574 100644 --- a/internal/component/discovery/distributed_targets_test.go +++ b/internal/component/discovery/distributed_targets_test.go @@ -285,11 +285,11 @@ func BenchmarkDistributedTargets(b *testing.B) { } func mkTarget(kv ...string) Target { - target := make(Target) + target := make(map[string]string) for i := 0; i < len(kv); i += 2 { target[kv[i]] = kv[i+1] } - return target + return NewTargetFromMap(target) } func testDistTargets(lookupMap map[shard.Key][]peer.Peer) *DistributedTargets { diff --git a/internal/component/discovery/process/container.go b/internal/component/discovery/process/container.go index 1ac7842818..62e56af7c8 100644 --- a/internal/component/discovery/process/container.go +++ b/internal/component/discovery/process/container.go @@ -42,15 +42,15 @@ func getContainerIDFromK8S(k8sContainerID string) string { } func getContainerIDFromTarget(target discovery.Target) string { - cid, ok := target[labelProcessContainerID] + cid, ok := target.Get(labelProcessContainerID) if ok && cid != "" { return cid } - cid, ok = target["__meta_kubernetes_pod_container_id"] + cid, ok = target.Get("__meta_kubernetes_pod_container_id") if ok && cid != "" { return getContainerIDFromK8S(cid) } - cid, ok = target["__meta_docker_container_id"] + cid, ok = target.Get("__meta_docker_container_id") if ok && cid != "" { return cid } diff --git a/internal/component/discovery/process/discover.go b/internal/component/discovery/process/discover.go index 388c4f5823..c740af28b5 100644 --- a/internal/component/discovery/process/discover.go +++ b/internal/component/discovery/process/discover.go @@ -11,9 +11,10 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/grafana/alloy/internal/component/discovery" gopsutil "github.com/shirou/gopsutil/v3/process" "golang.org/x/sys/unix" + + "github.com/grafana/alloy/internal/component/discovery" ) const ( @@ -52,7 +53,7 @@ func convertProcesses(ps []process) []discovery.Target { } func convertProcess(p process) discovery.Target { - t := make(discovery.Target, 8) + t := make(map[string]string, 8) t[labelProcessID] = p.pid if p.exe != "" { t[labelProcessExe] = p.exe @@ -75,7 +76,7 @@ func convertProcess(p process) discovery.Target { if p.cgroupPath != "" { t[labelProcessCgroupPath] = p.cgroupPath } - return t + return discovery.NewTargetFromMap(t) } func discover(l log.Logger, cfg *DiscoverConfig) ([]process, error) { diff --git a/internal/component/discovery/process/join.go b/internal/component/discovery/process/join.go index 5c2728cc6e..5643d4be9d 100644 --- a/internal/component/discovery/process/join.go +++ b/internal/component/discovery/process/join.go @@ -2,7 +2,9 @@ package process -import "github.com/grafana/alloy/internal/component/discovery" +import ( + "github.com/grafana/alloy/internal/component/discovery" +) func join(processes, containers []discovery.Target) []discovery.Target { res := make([]discovery.Target, 0, len(processes)+len(containers)) @@ -27,14 +29,9 @@ func join(processes, containers []discovery.Target) []discovery.Target { res = append(res, p) continue } - mergedTarget := make(discovery.Target, len(p)+len(container)) - for k, v := range p { - mergedTarget[k] = v - } - for k, v := range container { - mergedTarget[k] = v - } - res = append(res, mergedTarget) + mergedBuilder := discovery.NewTargetBuilderFrom(p) + mergedBuilder.MergeWith(container) + res = append(res, mergedBuilder.Target()) } for _, target := range cid2container { res = append(res, target) diff --git a/internal/component/discovery/process/join_test.go b/internal/component/discovery/process/join_test.go index 3f2e9dfa5a..2aedd1c445 100644 --- a/internal/component/discovery/process/join_test.go +++ b/internal/component/discovery/process/join_test.go @@ -6,8 +6,9 @@ import ( "fmt" "testing" - "github.com/grafana/alloy/internal/component/discovery" "github.com/stretchr/testify/assert" + + "github.com/grafana/alloy/internal/component/discovery" ) func TestJoin(t *testing.T) { @@ -37,50 +38,50 @@ func TestJoin(t *testing.T) { containerID: "", }), }, []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ "__meta_docker_container_id": "7edda1de1e0d1d366351e478359cf5fa16bb8ab53063a99bb119e56971bfb7e2", "foo": "bar", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__meta_kubernetes_pod_container_id": "docker://47e320f795efcec1ecf2001c3a09c95e3701ed87de8256837b70b10e23818251", "qwe": "asd", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "lol": "lol", - }, + }), }, []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ "__process_pid__": "239", "__meta_process_exe": "/bin/foo", "__meta_process_cwd": "/", "__container_id__": "7edda1de1e0d1d366351e478359cf5fa16bb8ab53063a99bb119e56971bfb7e2", "__meta_docker_container_id": "7edda1de1e0d1d366351e478359cf5fa16bb8ab53063a99bb119e56971bfb7e2", "foo": "bar", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__process_pid__": "240", "__meta_process_exe": "/bin/bar", "__meta_process_cwd": "/tmp", "__container_id__": "7edda1de1e0d1d366351e478359cf5fa16bb8ab53063a99bb119e56971bfb7e2", "__meta_docker_container_id": "7edda1de1e0d1d366351e478359cf5fa16bb8ab53063a99bb119e56971bfb7e2", "foo": "bar", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__meta_docker_container_id": "7edda1de1e0d1d366351e478359cf5fa16bb8ab53063a99bb119e56971bfb7e2", "foo": "bar", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__process_pid__": "241", "__meta_process_exe": "/bin/bash", "__meta_process_cwd": "/opt", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__meta_kubernetes_pod_container_id": "docker://47e320f795efcec1ecf2001c3a09c95e3701ed87de8256837b70b10e23818251", "qwe": "asd", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "lol": "lol", - }, + }), }, }, { diff --git a/internal/component/discovery/relabel/relabel.go b/internal/component/discovery/relabel/relabel.go index 6613c6c801..4d2d545231 100644 --- a/internal/component/discovery/relabel/relabel.go +++ b/internal/component/discovery/relabel/relabel.go @@ -10,8 +10,6 @@ import ( "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/service/livedebugging" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/relabel" ) func init() { @@ -47,7 +45,6 @@ type Component struct { opts component.Options mut sync.RWMutex - rcs []*relabel.Config debugDataPublisher livedebugging.DebugDataPublisher } @@ -82,24 +79,23 @@ func (c *Component) Run(ctx context.Context) error { // Update implements component.Component. func (c *Component) Update(args component.Arguments) error { - c.mut.Lock() - defer c.mut.Unlock() - newArgs := args.(Arguments) targets := make([]discovery.Target, 0, len(newArgs.Targets)) - relabelConfigs := alloy_relabel.ComponentToPromRelabelConfigs(newArgs.RelabelConfigs) - c.rcs = relabelConfigs for _, t := range newArgs.Targets { - lset := componentMapToPromLabels(t) - relabelled, keep := relabel.Process(lset, relabelConfigs...) + var ( + relabelled discovery.Target + builder = discovery.NewTargetBuilderFrom(t) + keep = alloy_relabel.ProcessBuilder(builder, newArgs.RelabelConfigs...) + ) if keep { - targets = append(targets, promLabelsToComponent(relabelled)) + relabelled = builder.Target() + targets = append(targets, relabelled) } componentID := livedebugging.ComponentID(c.opts.ID) if c.debugDataPublisher.IsActive(componentID) { - c.debugDataPublisher.Publish(componentID, fmt.Sprintf("%s => %s", lset.String(), relabelled.String())) + c.debugDataPublisher.Publish(componentID, fmt.Sprintf("%s => %s", t, relabelled)) } } @@ -112,21 +108,3 @@ func (c *Component) Update(args component.Arguments) error { } func (c *Component) LiveDebugging(_ int) {} - -func componentMapToPromLabels(ls discovery.Target) labels.Labels { - res := make([]labels.Label, 0, len(ls)) - for k, v := range ls { - res = append(res, labels.Label{Name: k, Value: v}) - } - - return res -} - -func promLabelsToComponent(ls labels.Labels) discovery.Target { - res := make(map[string]string, len(ls)) - for _, l := range ls { - res[l.Name] = l.Value - } - - return res -} diff --git a/internal/component/discovery/relabel/relabel_test.go b/internal/component/discovery/relabel/relabel_test.go index 6055fe05e0..920f5af4d4 100644 --- a/internal/component/discovery/relabel/relabel_test.go +++ b/internal/component/discovery/relabel/relabel_test.go @@ -4,12 +4,13 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + alloy_relabel "github.com/grafana/alloy/internal/component/common/relabel" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/discovery/relabel" "github.com/grafana/alloy/internal/runtime/componenttest" "github.com/grafana/alloy/syntax" - "github.com/stretchr/testify/require" ) func TestRelabelConfigApplication(t *testing.T) { @@ -56,7 +57,7 @@ rule { } ` expectedOutput := []discovery.Target{ - map[string]string{"__address__": "localhost", "app": "backend", "destination": "localhost/one", "meta_bar": "bar", "meta_foo": "foo", "name": "one"}, + discovery.NewTargetFromMap(map[string]string{"__address__": "localhost", "app": "backend", "destination": "localhost/one", "meta_bar": "bar", "meta_foo": "foo", "name": "one"}), } var args relabel.Arguments diff --git a/internal/component/discovery/target.go b/internal/component/discovery/target.go new file mode 100644 index 0000000000..e72de2492c --- /dev/null +++ b/internal/component/discovery/target.go @@ -0,0 +1,372 @@ +package discovery + +import ( + "fmt" + "slices" + "strings" + "sync" + + "github.com/cespare/xxhash/v2" + commonlabels "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + modellabels "github.com/prometheus/prometheus/model/labels" + "golang.org/x/exp/maps" + + "github.com/grafana/alloy/internal/runtime/equality" + "github.com/grafana/alloy/syntax" +) + +type Target struct { + group commonlabels.LabelSet + own commonlabels.LabelSet + size int +} + +var ( + seps = []byte{'\xff'} + // used in tests to simulate hash conflicts + labelSetEqualsFn = func(l1, l2 commonlabels.LabelSet) bool { return &l1 == &l2 || l1.Equal(l2) } + stringSlicesPool = sync.Pool{New: func() interface{} { return make([]string, 0, 20) }} + + _ syntax.Capsule = Target{} + _ syntax.ConvertibleIntoCapsule = Target{} + _ syntax.ConvertibleFromCapsule = &Target{} + _ equality.CustomEquality = Target{} +) + +var EmptyTarget = Target{ + group: commonlabels.LabelSet{}, + own: commonlabels.LabelSet{}, + size: 0, +} + +func NewTargetFromLabelSet(ls commonlabels.LabelSet) Target { + return NewTargetFromSpecificAndBaseLabelSet(ls, nil) +} + +func NewTargetFromSpecificAndBaseLabelSet(own, group commonlabels.LabelSet) Target { + if group == nil { + group = commonlabels.LabelSet{} + } + if own == nil { + own = commonlabels.LabelSet{} + } + ret := Target{ + group: group, + own: own, + } + size := 0 + ret.ForEachLabel(func(key string, value string) bool { + size++ + return true + }) + ret.size = size + return ret +} + +// NewTargetFromModelLabels creates a target from model Labels. +// NOTE: this is not optimised and should be avoided on a hot path. +func NewTargetFromModelLabels(labels modellabels.Labels) Target { + l := make(commonlabels.LabelSet, len(labels)) + for _, label := range labels { + l[commonlabels.LabelName(label.Name)] = commonlabels.LabelValue(label.Value) + } + return NewTargetFromLabelSet(l) +} + +func NewTargetFromMap(m map[string]string) Target { + l := make(commonlabels.LabelSet, len(m)) + for k, v := range m { + l[commonlabels.LabelName(k)] = commonlabels.LabelValue(v) + } + return NewTargetFromLabelSet(l) +} + +// PromLabels converts this target into prometheus/prometheus/model/labels.Labels. It is not efficient and should be +// avoided on a hot path. +func (t Target) PromLabels() modellabels.Labels { + // This method allocates less than Builder or ScratchBuilder, as proven by benchmarks. + lb := make([]modellabels.Label, 0, t.Len()) + t.ForEachLabel(func(key string, value string) bool { + lb = append(lb, modellabels.Label{ + Name: key, + Value: value, + }) + return true + }) + slices.SortFunc(lb, func(a, b modellabels.Label) int { return strings.Compare(a.Name, b.Name) }) + return lb +} + +func (t Target) NonReservedLabelSet() commonlabels.LabelSet { + // This may not be the most optimal way, but this method is NOT a known hot spot at the time of this comment. + result := make(commonlabels.LabelSet, t.Len()) + t.ForEachLabel(func(key string, value string) bool { + if !strings.HasPrefix(key, commonlabels.ReservedLabelPrefix) { + result[commonlabels.LabelName(key)] = commonlabels.LabelValue(value) + } + return true + }) + return result +} + +// ForEachLabel runs f over each key value pair in the Target. f must not modify Target while iterating. If f returns +// false, the iteration is interrupted. If f returns true, the iteration continues until the last element. ForEachLabel +// returns true if all the labels were iterated over or false if any call to f has interrupted the iteration. +// ForEachLabel does not guarantee iteration order or sort labels in any way. +func (t Target) ForEachLabel(f func(key string, value string) bool) bool { + for k, v := range t.own { + if !f(string(k), string(v)) { + // f has returned false, interrupt the iteration and return false. + return false + } + } + // Now go over the group ones only if they were not part of own labels + for k, v := range t.group { + if _, ok := t.own[k]; ok { + continue + } + if !f(string(k), string(v)) { + // f has returned false, interrupt the iteration and return false. + return false + } + } + + // We finished the iteration, return true. + return true +} + +// AsMap returns target's labels as a map of strings. +// Deprecated: this should not be used on any hot path as it leads to more allocation. +func (t Target) AsMap() map[string]string { + ret := make(map[string]string, t.Len()) + t.ForEachLabel(func(key string, value string) bool { + ret[key] = value + return true + }) + return ret +} + +func (t Target) Get(key string) (string, bool) { + lv, ok := t.own[commonlabels.LabelName(key)] + if ok { + return string(lv), ok + } + lv, ok = t.group[commonlabels.LabelName(key)] + return string(lv), ok +} + +// LabelSet converts this target in to a LabelSet +// Deprecated: this is not optimised and should be avoided if possible. +func (t Target) LabelSet() commonlabels.LabelSet { + merged := make(commonlabels.LabelSet, t.Len()) + for k, v := range t.group { + merged[k] = v + } + for k, v := range t.own { + merged[k] = v + } + return merged +} + +func (t Target) Len() int { + return t.size +} + +// AlloyCapsule marks FastTarget as a capsule so Alloy syntax can marshal to or from it. +func (t Target) AlloyCapsule() {} + +// ConvertInto is called by Alloy syntax to try converting Target to another type. +func (t Target) ConvertInto(dst interface{}) error { + switch dst := dst.(type) { + case *map[string]syntax.Value: + result := make(map[string]syntax.Value, t.Len()) + // NOTE: no need to sort as value_tokens.go in syntax/token/builder package sorts the map's keys. + t.ForEachLabel(func(key string, value string) bool { + result[key] = syntax.ValueFromString(value) + return true + }) + *dst = result + return nil + } + return fmt.Errorf("target::ConvertInto: conversion to '%T' is not supported", dst) +} + +// ConvertFrom is called by Alloy syntax to try converting from another type to Target. +func (t *Target) ConvertFrom(src interface{}) error { + switch src := src.(type) { + case map[string]syntax.Value: + labelSet := make(commonlabels.LabelSet, len(src)) + for k, v := range src { + var strValue string + switch { + case v.IsString(): + strValue = v.Text() + case v.Reflect().CanInterface(): + strValue = fmt.Sprintf("%v", v.Reflect().Interface()) + default: + return fmt.Errorf("target::ConvertFrom: cannot convert value that can't be interfaced to (e.g. unexported struct field)") + } + labelSet[commonlabels.LabelName(k)] = commonlabels.LabelValue(strValue) + } + *t = NewTargetFromLabelSet(labelSet) + return nil + } + return fmt.Errorf("target: conversion from '%T' is not supported", src) +} + +func (t Target) String() string { + s := make([]string, 0, t.Len()) + t.ForEachLabel(func(key string, value string) bool { + s = append(s, fmt.Sprintf("%q=%q", key, value)) + return true + }) + slices.Sort(s) + return fmt.Sprintf("{%s}", strings.Join(s, ", ")) +} + +// Equals implements equality.CustomEquality. Works only with pointers. +func (t Target) Equals(other any) bool { + if ot, ok := other.(*Target); ok { + return t.EqualsTarget(ot) + } + return false +} + +func (t Target) EqualsTarget(other *Target) bool { + if t.Len() != other.Len() { + return false + } + finished := t.ForEachLabel(func(key string, value string) bool { + otherValue, ok := other.Get(key) + if !ok || otherValue != value { + return false + } + return true + }) + return finished +} + +func (t Target) NonMetaLabelsHash() uint64 { + return t.HashLabelsWithPredicate(func(key string) bool { + return !strings.HasPrefix(key, commonlabels.MetaLabelPrefix) + }) +} + +func (t Target) SpecificLabelsHash(labelNames []string) uint64 { + return t.HashLabelsWithPredicate(func(key string) bool { + return slices.Contains(labelNames, key) + }) +} + +func (t Target) HashLabelsWithPredicate(pred func(key string) bool) uint64 { + // For hash to be deterministic, we need labels order to be deterministic too. Figure this out first. + labelsInOrder := stringSlicesPool.Get().([]string) + defer stringSlicesPool.Put(labelsInOrder[:]) + t.ForEachLabel(func(key string, value string) bool { + if pred(key) { + labelsInOrder = append(labelsInOrder, key) + } + return true + }) + slices.Sort(labelsInOrder) + return t.hashLabelsInOrder(labelsInOrder) +} + +func (t Target) groupLabelsHash() uint64 { + // For hash to be deterministic, we need labels order to be deterministic too. Figure this out first. + labelsInOrder := stringSlicesPool.Get().([]string) + defer stringSlicesPool.Put(labelsInOrder[:]) + + for name := range t.group { + labelsInOrder = append(labelsInOrder, string(name)) + } + slices.Sort(labelsInOrder) + return t.hashLabelsInOrder(labelsInOrder) +} + +// NOTE 1: This function is copied from Prometheus codebase and adapted to work correctly with Alloy types. +// NOTE 2: It is important to keep the hashing function consistent between Alloy versions in order to have +// +// smooth rollouts without duplicated or missing data. There are tests to verify this behaviour. Do not change it. +func (t Target) hashLabelsInOrder(order []string) uint64 { + // This optimisation is adapted from prometheus/model/labels. + // Use xxhash.Sum64(b) for fast path as it's faster. + b := make([]byte, 0, 1024) + mustGet := func(label string) string { + val, _ := t.Get(label) + // if val is not found it would mean there is a bug and Target is no longer immutable. But we can still provide + // a consistent hashing behaviour by returning empty string we got from Get. + return val + } + + for i, key := range order { + value := mustGet(key) + if len(b)+len(key)+len(value)+2 >= cap(b) { + // If labels entry is 1KB+ do not allocate whole entry. + h := xxhash.New() + _, _ = h.Write(b) + for _, key := range order[i:] { + _, _ = h.WriteString(key) + _, _ = h.Write(seps) + _, _ = h.WriteString(mustGet(key)) + _, _ = h.Write(seps) + } + return h.Sum64() + } + + b = append(b, key...) + b = append(b, seps[0]) + b = append(b, value...) + b = append(b, seps[0]) + } + return xxhash.Sum64(b) +} + +func ComponentTargetsToPromTargetGroups(jobName string, tgs []Target) map[string][]*targetgroup.Group { + targetIndWithCommonGroupLabels := map[uint64][]int{} // target group hash --> index of target in tgs array + for ind, t := range tgs { + fp := t.groupLabelsHash() + targetIndWithCommonGroupLabels[fp] = append(targetIndWithCommonGroupLabels[fp], ind) + } + + // Sort by hash to get deterministic order + sortedKeys := maps.Keys(targetIndWithCommonGroupLabels) + slices.Sort(sortedKeys) + + allGroups := make([]*targetgroup.Group, 0, len(targetIndWithCommonGroupLabels)) + var hashConflicts []commonlabels.LabelSet + for _, hash := range sortedKeys { + // targetIndices = indices of all the targets that have the same group labels hash + targetIndices := targetIndWithCommonGroupLabels[hash] + // since we grouped them by their group labels hash, their group labels should all be the same (except for hash collision handled below) + sharedLabels := tgs[targetIndices[0]].group + individualLabels := make([]commonlabels.LabelSet, 0, len(targetIndices)) + for _, ind := range targetIndices { + target := tgs[ind] + // detect hash collisions - we'll append them separately - it's still correct, just may be less efficient + if !labelSetEqualsFn(sharedLabels, target.group) { + hashConflicts = append(hashConflicts, target.LabelSet()) + continue + } + individualLabels = append(individualLabels, target.own) + } + + if len(individualLabels) != 0 { + allGroups = append(allGroups, &targetgroup.Group{ + Source: fmt.Sprintf("%s_part_%v", jobName, hash), + Labels: sharedLabels, + Targets: individualLabels, + }) + } + } + + if len(hashConflicts) > 0 { // these are consolidated already, no common group labels here. + allGroups = append(allGroups, &targetgroup.Group{ + Source: fmt.Sprintf("%s_rest", jobName), + Targets: hashConflicts, + }) + } + + return map[string][]*targetgroup.Group{jobName: allGroups} +} diff --git a/internal/component/discovery/target_builder.go b/internal/component/discovery/target_builder.go new file mode 100644 index 0000000000..7ea393dbca --- /dev/null +++ b/internal/component/discovery/target_builder.go @@ -0,0 +1,189 @@ +package discovery + +import ( + commonlabels "github.com/prometheus/common/model" + + "github.com/grafana/alloy/internal/component/common/relabel" +) + +type TargetBuilder interface { + relabel.LabelBuilder + Target() Target + MergeWith(Target) TargetBuilder +} + +type targetBuilder struct { + group commonlabels.LabelSet + own commonlabels.LabelSet + + toAdd map[string]string + toDel map[string]struct{} +} + +// NewTargetBuilder creates an empty labels builder. +func NewTargetBuilder() TargetBuilder { + return targetBuilder{ + group: nil, + own: make(commonlabels.LabelSet), + toAdd: make(map[string]string), + toDel: make(map[string]struct{}), + } +} + +func NewTargetBuilderFrom(t Target) TargetBuilder { + return NewTargetBuilderFromLabelSets(t.group, t.own) +} + +func NewTargetBuilderFromLabelSets(group, own commonlabels.LabelSet) TargetBuilder { + toAdd := make(map[string]string) + toDel := make(map[string]struct{}) + + // if we are given labels that are set to empty value, it should be treated as deleting them + for name, value := range group { + if len(value) == 0 { // if group has empty value + // and own doesn't override it OR overrides it with an empty value + if ownValue, ok := own[name]; !ok || len(ownValue) == 0 { + toDel[string(name)] = struct{}{} // mark label as deleted + } + } + } + for name, value := range own { + if len(value) == 0 { + toDel[string(name)] = struct{}{} + } + } + + return targetBuilder{ + group: group, + own: own, + toAdd: toAdd, + toDel: toDel, + } +} + +func (t targetBuilder) Get(label string) string { + if v, ok := t.toAdd[label]; ok { + return v + } + if _, ok := t.toDel[label]; ok { + return "" + } + lv, ok := t.own[commonlabels.LabelName(label)] + if ok { + return string(lv) + } + lv, ok = t.group[commonlabels.LabelName(label)] + return string(lv) +} + +func (t targetBuilder) Range(f func(label string, value string)) { + for k, v := range t.toAdd { + f(k, v) + } + for k, v := range t.own { + if _, deleted := t.toDel[string(k)]; deleted { + continue // skip if it's deleted + } + if _, added := t.toAdd[string(k)]; added { + continue // skip if it was in toAdd + } + f(string(k), string(v)) + } + for k, v := range t.group { + if _, deleted := t.toDel[string(k)]; deleted { + continue // skip if it's deleted + } + if _, added := t.toAdd[string(k)]; added { + continue // skip if it was in toAdd + } + if _, inOwn := t.own[k]; inOwn { + continue // skip if it was in own + } + f(string(k), string(v)) + } +} + +func (t targetBuilder) Set(label string, val string) { + if val == "" { // Setting to empty is treated as deleting. + t.Del(label) + return + } + t.toAdd[label] = val +} + +func (t targetBuilder) Del(labels ...string) { + for _, label := range labels { + t.toDel[label] = struct{}{} + // If we were adding one, need to clean it up too. + if _, ok := t.toAdd[label]; ok { + delete(t.toAdd, label) + } + } +} + +func (t targetBuilder) MergeWith(target Target) TargetBuilder { + // Not on a hot path, so doesn't really need to be optimised. + target.ForEachLabel(func(key string, value string) bool { + t.Set(key, value) + return true + }) + return t +} + +func (t targetBuilder) Target() Target { + if len(t.toAdd) == 0 && len(t.toDel) == 0 { + return NewTargetFromSpecificAndBaseLabelSet(t.own, t.group) + } + // Figure out if we need to modify own set + modifyOwn := false + if len(t.toAdd) > 0 { // if there is anything to add + modifyOwn = true + } else { + for label := range t.toDel { // if there is anything to delete + if _, ok := t.own[commonlabels.LabelName(label)]; ok { + modifyOwn = true + break + } + } + } + + modifyGroup := false + for label := range t.toDel { // if there is anything to delete from group + if _, ok := t.group[commonlabels.LabelName(label)]; ok { + modifyGroup = true + break + } + } + + var ( + newOwn = t.own + newGroup = t.group + ) + + if modifyOwn { + newOwn = make(commonlabels.LabelSet, len(t.own)+len(t.toAdd)) + for k, v := range t.own { + if _, ok := t.toDel[string(k)]; ok { + continue + } + newOwn[k] = v + } + for k, v := range t.toAdd { + newOwn[commonlabels.LabelName(k)] = commonlabels.LabelValue(v) + } + } + if modifyGroup { + // TODO(thampiotr): When relabeling a lot of targets that require changes to t.group, we might produce a lot of + // t.groups that will be essentially the same. If this becomes a hot spot, it could be + // remediated with an extra step to consolidate them using perhaps a hash as an ID. + newGroup = make(commonlabels.LabelSet, len(t.group)) + for k, v := range t.group { + if _, ok := t.toDel[string(k)]; ok { + continue + } + newGroup[k] = v + } + } + + return NewTargetFromSpecificAndBaseLabelSet(newOwn, newGroup) +} diff --git a/internal/component/discovery/target_builder_test.go b/internal/component/discovery/target_builder_test.go new file mode 100644 index 0000000000..c68d11a259 --- /dev/null +++ b/internal/component/discovery/target_builder_test.go @@ -0,0 +1,381 @@ +package discovery + +import ( + "fmt" + "testing" + + commonlabels "github.com/prometheus/common/model" + modellabels "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/assert" + + "github.com/grafana/alloy/internal/runtime/equality" +) + +func TestTargetBuilder(t *testing.T) { + testCases := []struct { + name string + init map[string]string + op func(tb TargetBuilder) + asserts func(t *testing.T, tb TargetBuilder) + expected map[string]string + }{ + { + name: "no changes", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + }, + { + name: "no changes with empty value deleted", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": ""}, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "no changes with all values deleted", + init: map[string]string{"hip": "", "boom": "", "tiki": ""}, + expected: map[string]string{}, + }, + { + name: "no changes from nil", + init: nil, + expected: nil, + }, + { + name: "get", + init: map[string]string{"hip": "hop", "boom": "bap"}, + asserts: func(t *testing.T, tb TargetBuilder) { + assert.Equal(t, "bap", tb.Get("boom")) + }, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "set and get", + init: map[string]string{}, + op: func(tb TargetBuilder) { tb.Set("ka", "boom") }, + asserts: func(t *testing.T, tb TargetBuilder) { assert.Equal(t, "boom", tb.Get("ka")) }, + expected: map[string]string{"ka": "boom"}, + }, + { + name: "set and get from nil", + init: nil, + op: func(tb TargetBuilder) { tb.Set("ka", "boom") }, + asserts: func(t *testing.T, tb TargetBuilder) { assert.Equal(t, "boom", tb.Get("ka")) }, + expected: map[string]string{"ka": "boom"}, + }, + { + name: "add one", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { tb.Set("ka", "boom") }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta", "ka": "boom"}, + }, + { + name: "add two", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.Set("ka", "boom") + tb.Set("foo", "bar") + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta", "ka": "boom", "foo": "bar"}, + }, + { + name: "overwrite one", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { tb.Set("tiki", "tiki") }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "tiki"}, + }, + { + name: "merge with target", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.MergeWith(NewTargetFromMap(map[string]string{"ka": "boom", "tiki": "tiki", "kung": "fu"})) + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "tiki", "ka": "boom", "kung": "fu"}, + }, + { + name: "delete one", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { tb.Del("tiki") }, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "delete one by setting to empty", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { tb.Set("tiki", "") }, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "delete multiple", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { tb.Del("tiki", "hip") }, + expected: map[string]string{"boom": "bap"}, + }, + { + name: "add and delete one", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.Set("tiki", "tiki") + tb.Del("tiki") + }, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "add and delete one by setting to empty", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.Set("tiki", "tiki") + tb.Set("tiki", "") + }, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "delete and add one", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.Del("tiki") + tb.Set("tiki", "tiki") + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "tiki"}, + }, + { + name: "delete by setting to empty and add one", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.Set("tiki", "") + tb.Set("tiki", "tiki") + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "tiki"}, + }, + { + name: "get after adding", + init: map[string]string{"hip": "hop", "boom": "bap"}, + op: func(tb TargetBuilder) { + tb.Set("tiki", "taki") + }, + asserts: func(t *testing.T, tb TargetBuilder) { + assert.Equal(t, "taki", tb.Get("tiki")) + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "taki"}, + }, + { + name: "get after deleting", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + op: func(tb TargetBuilder) { + tb.Del("tiki") + }, + asserts: func(t *testing.T, tb TargetBuilder) { + assert.Equal(t, "", tb.Get("tiki")) + }, + expected: map[string]string{"hip": "hop", "boom": "bap"}, + }, + { + name: "simple range", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + asserts: func(t *testing.T, tb TargetBuilder) { + seen := map[string]struct{}{} + tb.Range(func(label string, value string) { + seen[fmt.Sprintf("%q: %q", label, value)] = struct{}{} + }) + assert.Equal( + t, + map[string]struct{}{`"hip": "hop"`: {}, `"boom": "bap"`: {}, `"tiki": "ta"`: {}}, + seen, + ) + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + }, + { + name: "range with pending additions and deletes", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + asserts: func(t *testing.T, tb TargetBuilder) { + tb.Set("tiki", "taki") + tb.Del("boom") + tb.Set("kung", "fu") + + seen := map[string]struct{}{} + tb.Range(func(label string, value string) { + seen[fmt.Sprintf("%q: %q", label, value)] = struct{}{} + }) + assert.Equal( + t, + map[string]struct{}{`"hip": "hop"`: {}, `"kung": "fu"`: {}, `"tiki": "taki"`: {}}, + seen, + ) + }, + expected: map[string]string{"hip": "hop", "kung": "fu", "tiki": "taki"}, + }, + { + name: "simple range on nil", + init: nil, + asserts: func(t *testing.T, tb TargetBuilder) { + seen := map[string]struct{}{} + tb.Range(func(label string, value string) { + seen[fmt.Sprintf("%q: %q", label, value)] = struct{}{} + }) + assert.Equal(t, map[string]struct{}{}, seen) + }, + expected: nil, + }, + { + name: "range with adding", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + asserts: func(t *testing.T, tb TargetBuilder) { + tb.Range(func(label string, value string) { + tb.Set(value, label) + }) + }, + expected: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta", "hop": "hip", "bap": "boom", "ta": "tiki"}, + }, + { + name: "range with overwriting", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + asserts: func(t *testing.T, tb TargetBuilder) { + tb.Range(func(label string, value string) { + tb.Set(label, label) + }) + }, + expected: map[string]string{"hip": "hip", "boom": "boom", "tiki": "tiki"}, + }, + { + name: "range with deleting", + init: map[string]string{"hip": "hop", "boom": "bap", "tiki": "ta"}, + asserts: func(t *testing.T, tb TargetBuilder) { + tb.Range(func(label string, value string) { + if len(label) > 3 { + tb.Del(label) + } + }) + }, + expected: map[string]string{"hip": "hop"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + expected := NewTargetFromMap(tc.expected) + + runTest := func(t *testing.T, tb TargetBuilder) { + if tc.op != nil { + tc.op(tb) + } + if tc.asserts != nil { + tc.asserts(t, tb) + } + actual := tb.Target() + + equal := equality.DeepEqual(actual, expected) + assert.True(t, equal) + if !equal { // if not equal, run this to get a nice diff view + assert.Equal(t, expected, actual) + } + + assert.Equal(t, actual.PromLabels().Hash(), actual.HashLabelsWithPredicate(func(key string) bool { + return true + }), "prometheus and alloy target hash codes should match") + } + + t.Run("prometheus compliant", func(t *testing.T) { + tb := newPromBuilderAdapter(modellabels.FromMap(tc.init)) + runTest(t, tb) + }) + + t.Run("all labels own", func(t *testing.T) { + tb := NewTargetBuilderFromLabelSets(nil, labelSetFromMap(tc.init)) + runTest(t, tb) + }) + + t.Run("all labels from group", func(t *testing.T) { + tb := NewTargetBuilderFromLabelSets(labelSetFromMap(tc.init), nil) + runTest(t, tb) + }) + + t.Run("group and own split by half", func(t *testing.T) { + first, second := splitMap(len(tc.init)/2, tc.init) + tb := NewTargetBuilderFromLabelSets(labelSetFromMap(first), labelSetFromMap(second)) + runTest(t, tb) + }) + + t.Run("group labels overwritten", func(t *testing.T) { + first, second := splitMap(len(tc.init)/2, tc.init) + for k, v := range second { + first[k] = fmt.Sprintf("overwritten_%s", v) + } + tb := NewTargetBuilderFromLabelSets(labelSetFromMap(first), labelSetFromMap(second)) + runTest(t, tb) + }) + + }) + } + +} + +func splitMap(firstSize int, m map[string]string) (map[string]string, map[string]string) { + if len(m) <= firstSize { + return m, nil + } + first, second := make(map[string]string), make(map[string]string) + ind := 0 + for k, v := range m { + if ind < firstSize { + first[k] = v + } else { + second[k] = v + } + ind++ + } + return first, second +} + +func labelSetFromMap(m map[string]string) commonlabels.LabelSet { + r := make(commonlabels.LabelSet, len(m)) + for k, v := range m { + r[commonlabels.LabelName(k)] = commonlabels.LabelValue(v) + } + return r +} + +// builderAdapter is used to verify TargetBuilder implementation in this package matches the prometheus model.Builder. +type builderAdapter struct { + b *modellabels.Builder +} + +func (b *builderAdapter) SetKV(kv ...string) TargetBuilder { + for i := 0; i < len(kv); i += 2 { + b.b.Set(kv[i], kv[i+1]) + } + return b +} + +func (b *builderAdapter) MergeWith(target Target) TargetBuilder { + target.ForEachLabel(func(key string, value string) bool { + b.b.Set(key, value) + return true + }) + return b +} + +func newPromBuilderAdapter(ls modellabels.Labels) TargetBuilder { + return &builderAdapter{ + b: modellabels.NewBuilder(ls), + } +} + +func (b *builderAdapter) Get(label string) string { + return b.b.Get(label) +} + +func (b *builderAdapter) Range(f func(label string, value string)) { + b.b.Range(func(l modellabels.Label) { + f(l.Name, l.Value) + }) +} + +func (b *builderAdapter) Set(label string, val string) { + b.b.Set(label, val) +} + +func (b *builderAdapter) Del(ns ...string) { + b.b = b.b.Del(ns...) +} + +func (b *builderAdapter) Target() Target { + return NewTargetFromModelLabels(b.b.Labels()) +} diff --git a/internal/component/discovery/target_test.go b/internal/component/discovery/target_test.go new file mode 100644 index 0000000000..e0dc111045 --- /dev/null +++ b/internal/component/discovery/target_test.go @@ -0,0 +1,866 @@ +package discovery + +import ( + "fmt" + "slices" + "strings" + "testing" + + "github.com/Masterminds/goutils" + "github.com/grafana/ckit/peer" + "github.com/grafana/ckit/shard" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/runtime/equality" + "github.com/grafana/alloy/syntax/parser" + "github.com/grafana/alloy/syntax/token/builder" + "github.com/grafana/alloy/syntax/vm" +) + +func TestDecodeMap(t *testing.T) { + type testCase struct { + name string + input string + expected map[string]string + } + + tests := []testCase{ + { + name: "empty", + input: `{}`, + expected: map[string]string{}, + }, + { + name: "simple decode", + input: `{ "a" = "5", "b" = "10" }`, + expected: map[string]string{"a": "5", "b": "10"}, + }, + { + name: "decode no quotes on keys", + input: `{ a = "5", b = "10" }`, + expected: map[string]string{"a": "5", "b": "10"}, + }, + { + name: "decode no quotes", + input: `{ a = 5, b = 10 }`, + expected: map[string]string{"a": "5", "b": "10"}, + }, + { + name: "decode mixed quoting", + input: `{ a = "5", "b" = "10", "c" = 15, d = 20 }`, + expected: map[string]string{"a": "5", "b": "10", "c": "15", "d": "20"}, + }, + { + name: "decode different order", + input: `{ "b" = "10", "a" = "5" }`, + expected: map[string]string{"a": "5", "b": "10"}, + }, + { + name: "decode with string concat", + input: `{ "b" = "1"+"0", "a" = "5" }`, + expected: map[string]string{"a": "5", "b": "10"}, + }, + { + name: "decode with std function", + input: `{ "b" = string.format("%x", 31337), "a" = string.join(["2", "0"], ".") }`, + expected: map[string]string{"a": "2.0", "b": "7a69"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + scope := vm.NewScope(map[string]interface{}{}) + expr, err := parser.ParseExpression(tc.input) + require.NoError(t, err) + eval := vm.New(expr) + actual := Target{} + require.NoError(t, eval.Evaluate(scope, &actual)) + require.Equal(t, NewTargetFromMap(tc.expected), actual) + }) + } +} + +func TestEncode_Decode_Targets(t *testing.T) { + type testCase struct { + name string + input map[string]string + expected string + } + + tests := []testCase{ + { + name: "empty", + input: map[string]string{}, + expected: `target = {}`, + }, + { + name: "simple", + input: map[string]string{"banh": "mi", "char": "siu"}, + expected: `target = { + banh = "mi", + char = "siu", +}`, + }, + { + name: "simple order change", + input: map[string]string{"char": "siu", "banh": "mi", "bun": "cha"}, + expected: `target = { + banh = "mi", + bun = "cha", + char = "siu", +}`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Test encoding + f := builder.NewFile() + f.Body().SetAttributeValue("target", NewTargetFromMap(tc.input)) + encoded := string(f.Bytes()) + require.Equal(t, tc.expected, encoded) + + // Try decoding now + toDecode := strings.TrimPrefix(encoded, "target = ") + scope := vm.NewScope(map[string]interface{}{}) + expr, err := parser.ParseExpression(toDecode) + require.NoError(t, err) + eval := vm.New(expr) + actual := Target{} + require.NoError(t, eval.Evaluate(scope, &actual)) + require.Equal(t, NewTargetFromMap(tc.input), actual) + }) + } +} + +func TestEncode_Decode_TargetArrays(t *testing.T) { + type testCase struct { + name string + input []map[string]string + expected string + } + + tests := []testCase{ + { + name: "nil", + input: nil, + expected: `target = []`, + }, + { + name: "empty", + input: []map[string]string{}, + expected: `target = []`, + }, + { + name: "simple two targets", + input: []map[string]string{ + {"a": "5", "b": "10"}, + {"c": "5", "d": "10"}, + }, + expected: `target = [{ + a = "5", + b = "10", +}, { + c = "5", + d = "10", +}]`, + }, + { + name: "nil target", + input: []map[string]string{ + {"a": "5", "b": "10"}, + nil, + }, + expected: `target = [{ + a = "5", + b = "10", +}, {}]`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Set as map first to verify it acts the same as targets + f := builder.NewFile() + f.Body().SetAttributeValue("target", tc.input) + require.Equal(t, tc.expected, string(f.Bytes()), "compliance check using a map") + + // Set as targets and check it's the same + targets := make([]Target, 0) + for _, m := range tc.input { + targets = append(targets, NewTargetFromMap(m)) + } + f = builder.NewFile() + f.Body().SetAttributeValue("target", targets) + encoded := string(f.Bytes()) + require.Equal(t, tc.expected, encoded, "using a target") + + // Try decoding now + toDecode := strings.TrimPrefix(encoded, "target = ") + scope := vm.NewScope(map[string]interface{}{}) + expr, err := parser.ParseExpression(toDecode) + require.NoError(t, err) + eval := vm.New(expr) + var actual []Target + require.NoError(t, eval.Evaluate(scope, &actual)) + require.Equal(t, targets, actual) + }) + } +} + +func TestDecode_TargetArrays(t *testing.T) { + type testCase struct { + name string + input string + expected []map[string]string + } + + tests := []testCase{ + { + name: "empty array", + input: `[]`, + expected: []map[string]string{}, + }, + { + name: "simple two targets", + input: `[{a = "5", b = "10"}, {c = "5", d = "10"}]`, + expected: []map[string]string{ + {"a": "5", "b": "10"}, + {"c": "5", "d": "10"}, + }, + }, + { + name: "concat targets", + input: `array.concat([{a = "5", b = "10"}], [{c = "5", d = "10"}])`, + expected: []map[string]string{ + {"a": "5", "b": "10"}, + {"c": "5", "d": "10"}, + }, + }, + { + name: "concat nested targets", + input: `array.concat(array.concat([{a = "5", b = "10"}], [{c = "5", d = "10"}]), [{e = "5", f = "10"}])`, + expected: []map[string]string{ + {"a": "5", "b": "10"}, + {"c": "5", "d": "10"}, + {"e": "5", "f": "10"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + expectedTargets := make([]Target, 0) + for _, m := range tc.expected { + expectedTargets = append(expectedTargets, NewTargetFromMap(m)) + } + + scope := vm.NewScope(map[string]interface{}{}) + expr, err := parser.ParseExpression(tc.input) + require.NoError(t, err) + eval := vm.New(expr) + var actual []Target + require.NoError(t, eval.Evaluate(scope, &actual)) + require.Equal(t, expectedTargets, actual) + }) + } +} + +func TestTargetMisc(t *testing.T) { + target := NewTargetFromMap(map[string]string{"a": "5", "b": "10"}) + // Test can iterate over it + var seen []string + target.ForEachLabel(func(k string, v string) bool { + seen = append(seen, fmt.Sprintf("%s=%s", k, v)) + return true + }) + slices.Sort(seen) + require.Equal(t, []string{"a=5", "b=10"}, seen) + + // Some loggers print targets out, check it's all good. + require.Equal(t, `{"a"="5", "b"="10"}`, fmt.Sprintf("%s", target)) +} + +func TestConvertFromNative(t *testing.T) { + var nativeTargets = []model.LabelSet{ + {model.LabelName("hip"): model.LabelValue("hop")}, + {model.LabelName("nae"): model.LabelValue("nae")}, + } + + nativeGroup := &targetgroup.Group{ + Targets: nativeTargets, + Labels: model.LabelSet{ + model.LabelName("boom"): model.LabelValue("bap"), + }, + Source: "test", + } + + expected := []Target{ + NewTargetFromMap(map[string]string{"hip": "hop", "boom": "bap"}), + NewTargetFromMap(map[string]string{"nae": "nae", "boom": "bap"}), + } + + require.True(t, equality.DeepEqual(expected, toAlloyTargets(map[string]*targetgroup.Group{"test": nativeGroup}))) +} + +func TestEquals_Custom(t *testing.T) { + eq1 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"foo": "bar"}, + model.LabelSet{"hip": "hop"}, + ) + eq2 := NewTargetFromSpecificAndBaseLabelSet( + nil, + model.LabelSet{"hip": "hop", "foo": "bar"}, + ) + eq3 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"hip": "hop", "foo": "bar"}, + nil, + ) + eq4 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"hip": "hop", "foo": "bar"}, + model.LabelSet{"foo": "baz"}, // overwritten by own set + ) + + equalTargets := []Target{eq1, eq2, eq3, eq4} + + ne1 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"foo": "bar"}, + nil, + ) + ne2 := NewTargetFromSpecificAndBaseLabelSet( + nil, + model.LabelSet{"foo": "bar"}, + ) + ne3 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"boom": "bap"}, + model.LabelSet{"hip": "hop", "foo": "bar"}, + ) + ne4 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"hip": "hop", "foo": "bar"}, + model.LabelSet{"boom": "bap"}, + ) + ne5 := NewTargetFromSpecificAndBaseLabelSet( + model.LabelSet{"foo": "baz"}, // takes precedence over the group + model.LabelSet{"hip": "hop", "foo": "bar"}, + ) + notEqualTargets := []Target{ne1, ne2, ne3, ne4, ne5} + + for _, t1 := range equalTargets { + for _, t2 := range equalTargets { + require.True(t, t1.Equals(&t2), "should be equal: %v = %v", t1, t2) + require.True(t, t1.EqualsTarget(&t2), "should be equal: %v = %v", t1, t2) + require.True(t, t2.Equals(&t1), "should be equal: %v = %v", t1, t2) + require.True(t, t2.EqualsTarget(&t1), "should be equal: %v = %v", t1, t2) + } + } + + for _, t1 := range notEqualTargets { + for _, t2 := range equalTargets { + require.False(t, t1.Equals(&t2), "should not be equal: %v <> %v", t1, t2) + require.False(t, t1.EqualsTarget(&t2), "should not be equal: %v <> %v", t1, t2) + require.False(t, t2.Equals(&t1), "should not be equal: %v <> %v", t1, t2) + require.False(t, t2.EqualsTarget(&t1), "should not be equal: %v <> %v", t1, t2) + } + } +} + +func TestHashing(t *testing.T) { + labelsPerGenerator := 10 + targetsPerTestCase := 10 + type testCase struct { + name string + labelGenerators []func(targetInd, labelInd int) (string, string) + hashOp func(target Target) uint64 + expectedHash uint64 + expectAllDifferent bool + } + + testCases := []testCase{ + { + name: "labels names different", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d_%d", labelInd, targetInd), fmt.Sprintf("value_%d", labelInd) + }, + }, + hashOp: func(target Target) uint64 { + return target.HashLabelsWithPredicate(func(key string) bool { + return true + }) + }, + expectAllDifferent: true, + }, + { + name: "label values different", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("value_%d_%d", labelInd, targetInd) + }, + }, + hashOp: func(target Target) uint64 { + return target.HashLabelsWithPredicate(func(key string) bool { + return true + }) + }, + expectAllDifferent: true, + }, + { + name: "all labels same for all targets", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("value_%d", labelInd) + }, + }, + hashOp: func(target Target) uint64 { + return target.HashLabelsWithPredicate(func(key string) bool { + return true + }) + }, + expectedHash: 0xa28155048ff30d6f, + }, + { + name: "all labels same for all targets - non meta labels hash", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("value_%d", labelInd) + }, + }, + hashOp: func(target Target) uint64 { return target.NonMetaLabelsHash() }, + expectedHash: 0xa28155048ff30d6f, + }, + + { + name: "specific labels hash equal", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("t%d_l%d", targetInd, labelInd), fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + // some const labels same for all to use in SpecificLabelsHash + func(targetInd, labelInd int) (string, string) { return "foo", "bar" }, + func(targetInd, labelInd int) (string, string) { return "bin", "baz" }, + }, + hashOp: func(target Target) uint64 { return target.SpecificLabelsHash([]string{"foo", "bin"}) }, + expectedHash: 0xbbbe498586b668f3, + }, + { + name: "specific labels hash different", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("t%d_l%d", targetInd, labelInd), fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + // some const labels same for all to use in SpecificLabelsHash + func(targetInd, labelInd int) (string, string) { + return "foo", fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + func(targetInd, labelInd int) (string, string) { + return "bin", fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + }, + hashOp: func(target Target) uint64 { return target.SpecificLabelsHash([]string{"foo", "bin"}) }, + expectAllDifferent: true, + }, + { + name: "labels with predicate equal", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("t%d_l%d", targetInd, labelInd), fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("val_%d", labelInd) + }, + }, + hashOp: func(target Target) uint64 { + return target.HashLabelsWithPredicate(func(key string) bool { + return strings.HasPrefix(key, "label_") + }) + }, + expectedHash: 0x77c5d28715ca6a11, + }, + { + name: "labels with predicate different values", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("t%d_l%d", targetInd, labelInd), fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("val_%d_%d", labelInd, targetInd) + }, + }, + hashOp: func(target Target) uint64 { + return target.HashLabelsWithPredicate(func(key string) bool { + return strings.HasPrefix(key, "label_") + }) + }, + expectAllDifferent: true, + }, + { + name: "meta labels equal", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("__meta_t%d_l%d", targetInd, labelInd), fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("val_%d", labelInd) + }, + }, + hashOp: func(target Target) uint64 { return target.NonMetaLabelsHash() }, + expectedHash: 0x77c5d28715ca6a11, + }, + { + name: "meta labels different", + labelGenerators: []func(targetInd, labelInd int) (string, string){ + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("__meta_t%d_l%d", targetInd, labelInd), fmt.Sprintf("%d_%d", targetInd, labelInd) + }, + func(targetInd, labelInd int) (string, string) { + return fmt.Sprintf("label_%d", labelInd), fmt.Sprintf("val_%d_%d", labelInd, targetInd) + }, + }, + hashOp: func(target Target) uint64 { return target.NonMetaLabelsHash() }, + expectAllDifferent: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + + // verifies that all hashes are equal to each other and the expected hash (if it's specified) + verifyCollectionEqual := func(hashes []uint64) { + require.Greater(t, len(hashes), 0) + firstActual := hashes[0] + for _, hash := range hashes { + require.Equal(t, firstActual, hash, "returned hashes are different between each other: %v", hashes) + } + if tc.expectedHash != 0 { // if specified, check the expected hash + require.Equal(t, tc.expectedHash, firstActual, "returned hashes don't match the expected hash: %v vs %v", tc.expectedHash, hashes) + } + } + + verifyAllDifferent := func(hashes []uint64) { + require.Greater(t, len(hashes), 0) + unique := map[uint64]struct{}{} + for _, hash := range hashes { + unique[hash] = struct{}{} + } + require.Equal(t, len(unique), len(hashes), "hashes are not all unique: %v vs. unique: %v", hashes, unique) + } + + var allHashes []uint64 + // create targetsPerTestCase targets + for targetInd := 0; targetInd < targetsPerTestCase; targetInd++ { + tb := NewTargetBuilder() + // for each create bunch of labels using generators + for _, generator := range tc.labelGenerators { + for labelInd := 0; labelInd < labelsPerGenerator; labelInd++ { + l, v := generator(targetInd, labelInd) + tb.Set(l, v) + } + } + + // get the hashes + actual := tc.hashOp(tb.Target()) + allHashes = append(allHashes, actual) + } + // verify all targets hashes together + if tc.expectAllDifferent { + verifyAllDifferent(allHashes) + } else { + verifyCollectionEqual(allHashes) + } + }) + } + +} + +func TestHashLargeLabelSets(t *testing.T) { + sharedLabels := 50 + ownLabels := 100 + labelsLength := 100 // large labels to verify the "slow" code path + metaLabelsCount := 5 + + chars := "abcdefghijklmnopqrstuvwxyz" + + genLabel := func(id, length int) string { + sb := strings.Builder{} + for i := 0; i < length; i++ { + sb.WriteByte(chars[(i+id)%len(chars)]) + } + return sb.String() + } + + genLabelSet := func(size int) model.LabelSet { + ls := model.LabelSet{} + for i := 0; i < size; i++ { + name := genLabel(i, labelsLength) + value := genLabel(i, labelsLength) + ls[model.LabelName(name)] = model.LabelValue(value) + } + for i := 0; i < metaLabelsCount; i++ { + name := "__meta_" + genLabel(i, labelsLength) + value := genLabel(i, labelsLength) + ls[model.LabelName(name)] = model.LabelValue(value) + } + return ls + } + + target := NewTargetFromSpecificAndBaseLabelSet(genLabelSet(ownLabels), genLabelSet(sharedLabels)) + expectedNonMetaLabelsHash := 0x374005f6a622f4d8 + expectedAllLabelsHash := 0x174c789bf3b783a7 + + require.Equal(t, uint64(expectedNonMetaLabelsHash), target.NonMetaLabelsHash()) + require.Equal(t, uint64(expectedNonMetaLabelsHash), target.HashLabelsWithPredicate(func(key string) bool { + return !strings.HasPrefix(key, "__meta_") + })) + require.Equal(t, uint64(expectedAllLabelsHash), target.HashLabelsWithPredicate(func(key string) bool { + return true + })) + require.Equal(t, uint64(expectedAllLabelsHash), target.PromLabels().Hash()) // check it matches Prometheus algo + + var allNonMetaLabels []string + target.ForEachLabel(func(k string, v string) bool { + if !strings.HasPrefix(k, "__meta_") { + allNonMetaLabels = append(allNonMetaLabels, k) + } + return true + }) + + require.Equal(t, uint64(expectedNonMetaLabelsHash), target.SpecificLabelsHash(allNonMetaLabels)) +} + +func TestComponentTargetsToPromTargetGroups(t *testing.T) { + type testTarget struct { + own map[string]string + group map[string]string + } + type args struct { + jobName string + tgs []testTarget + } + tests := []struct { + name string + args args + mockLabelSetEqualFn func(l1, l2 model.LabelSet) bool + expected map[string][]*targetgroup.Group + }{ + { + name: "empty targets", + args: args{jobName: "job"}, + expected: map[string][]*targetgroup.Group{"job": {}}, + }, + { + name: "targets all in same group", + args: args{ + jobName: "job", + tgs: []testTarget{ + {group: map[string]string{"hip": "hop"}, own: map[string]string{"boom": "bap"}}, + {group: map[string]string{"hip": "hop"}, own: map[string]string{"tiki": "ta"}}, + }, + }, + expected: map[string][]*targetgroup.Group{"job": { + { + Source: "job_part_9994420383135092995", + Labels: mapToLabelSet(map[string]string{"hip": "hop"}), + Targets: []model.LabelSet{ + mapToLabelSet(map[string]string{"boom": "bap"}), + mapToLabelSet(map[string]string{"tiki": "ta"}), + }, + }, + }}, + }, + { + name: "two groups", + args: args{ + jobName: "job", + tgs: []testTarget{ + {group: map[string]string{"hip": "hop"}, own: map[string]string{"boom": "bap"}}, + {group: map[string]string{"kung": "foo"}, own: map[string]string{"tiki": "ta"}}, + {group: map[string]string{"hip": "hop"}, own: map[string]string{"hoo": "rey"}}, + {group: map[string]string{"kung": "foo"}, own: map[string]string{"bibim": "bap"}}, + }, + }, + expected: map[string][]*targetgroup.Group{"job": { + { + Source: "job_part_9994420383135092995", + Labels: mapToLabelSet(map[string]string{"hip": "hop"}), + Targets: []model.LabelSet{ + mapToLabelSet(map[string]string{"boom": "bap"}), + mapToLabelSet(map[string]string{"hoo": "rey"}), + }, + }, + { + Source: "job_part_13313558424202542889", + Labels: mapToLabelSet(map[string]string{"kung": "foo"}), + Targets: []model.LabelSet{ + mapToLabelSet(map[string]string{"tiki": "ta"}), + mapToLabelSet(map[string]string{"bibim": "bap"}), + }, + }, + }}, + }, + { + name: "two groups with hash conflict", + mockLabelSetEqualFn: func(l1, l2 model.LabelSet) bool { + if _, ok := l1[model.LabelName("hip")]; ok { + return false + } + return l1.Equal(l2) + }, + args: args{ + jobName: "job", + tgs: []testTarget{ + {group: map[string]string{"hip": "hop"}, own: map[string]string{"boom": "bap"}}, + {group: map[string]string{"kung": "foo"}, own: map[string]string{"tiki": "ta"}}, + {group: map[string]string{"hip": "hop"}, own: map[string]string{"hoo": "rey"}}, + {group: map[string]string{"kung": "foo"}, own: map[string]string{"bibim": "bap"}}, + }, + }, + expected: map[string][]*targetgroup.Group{"job": { + { + Source: "job_part_13313558424202542889", + Labels: mapToLabelSet(map[string]string{"kung": "foo"}), + Targets: []model.LabelSet{ + mapToLabelSet(map[string]string{"tiki": "ta"}), + mapToLabelSet(map[string]string{"bibim": "bap"}), + }, + }, + { + Source: "job_rest", + Targets: []model.LabelSet{ + mapToLabelSet(map[string]string{"boom": "bap", "hip": "hop"}), + mapToLabelSet(map[string]string{"hoo": "rey", "hip": "hop"}), + }, + }, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.mockLabelSetEqualFn != nil { + prev := labelSetEqualsFn + labelSetEqualsFn = tt.mockLabelSetEqualFn + defer func() { + labelSetEqualsFn = prev + }() + } + + targets := make([]Target, 0, len(tt.args.tgs)) + for _, tg := range tt.args.tgs { + targets = append(targets, NewTargetFromSpecificAndBaseLabelSet(mapToLabelSet(tg.own), mapToLabelSet(tg.group))) + } + actual := ComponentTargetsToPromTargetGroups(tt.args.jobName, targets) + assert.Contains(t, actual, tt.args.jobName) + assert.Equal(t, tt.expected, actual, "actual:\n%+v\nexpected:\n%+v\n", actual, tt.expected) + }) + } +} + +/* + Recent run: + +goos: darwin goarch: arm64 cpu: Apple M2 +Benchmark_Targets_TypicalPipeline-8 36 32868159 ns/op 6022494 B/op 100544 allocs/op +Benchmark_Targets_TypicalPipeline-8 34 34562724 ns/op 6109322 B/op 100543 allocs/op +Benchmark_Targets_TypicalPipeline-8 34 35662420 ns/op 6022429 B/op 100545 allocs/op +Benchmark_Targets_TypicalPipeline-8 36 33446308 ns/op 6021909 B/op 100541 allocs/op +Benchmark_Targets_TypicalPipeline-8 34 33537419 ns/op 6022333 B/op 100543 allocs/op +Benchmark_Targets_TypicalPipeline-8 34 33687083 ns/op 6109172 B/op 100543 allocs/op +*/ +func Benchmark_Targets_TypicalPipeline(b *testing.B) { + sharedLabels := 5 + labelsPerTarget := 5 + labelsLength := 10 + targetsCount := 20_000 + numPeers := 10 + + genLabelSet := func(size int) model.LabelSet { + ls := model.LabelSet{} + for i := 0; i < size; i++ { + name, _ := goutils.RandomAlphaNumeric(labelsLength) + value, _ := goutils.RandomAlphaNumeric(labelsLength) + ls[model.LabelName(name)] = model.LabelValue(value) + } + return ls + } + + var labelSets []model.LabelSet + for i := 0; i < targetsCount; i++ { + labelSets = append(labelSets, genLabelSet(labelsPerTarget)) + } + + cache := map[string]*targetgroup.Group{} + cache["test"] = &targetgroup.Group{ + Targets: labelSets, + Labels: genLabelSet(sharedLabels), + Source: "test", + } + + peers := make([]peer.Peer, 0, numPeers) + for i := 0; i < numPeers; i++ { + peerName := fmt.Sprintf("peer_%d", i) + peers = append(peers, peer.Peer{Name: peerName, Addr: peerName, Self: i == 0, State: peer.StateParticipant}) + } + + cluster := &randomCluster{ + peers: peers, + peersByIndex: make(map[int][]peer.Peer, len(peers)), + } + + b.ResetTimer() + + var prevDistTargets *DistributedTargets + for i := 0; i < b.N; i++ { + // Creating the targets in discovery + targets := toAlloyTargets(cache) + + // Relabel of targets in discovery.relabel + for ind := range targets { + tb := NewTargetBuilderFrom(targets[ind]) + // would do alloy_relabel.ProcessBuilder here to relabel + targets[ind] = tb.Target() + } + + // prometheus.scrape: distributing targets for clustering + dt := NewDistributedTargets(true, cluster, targets) + _ = dt.LocalTargets() + _ = dt.MovedToRemoteInstance(prevDistTargets) + // Sending LabelSet to Prometheus library for scraping + _ = ComponentTargetsToPromTargetGroups("test", targets) + + // Remote write happens on a sample level and largely outside Alloy's codebase, so skipping here. + + prevDistTargets = dt + } +} + +type randomCluster struct { + peers []peer.Peer + // stores results in a map to reduce the allocation noise in the benchmark + peersByIndex map[int][]peer.Peer +} + +func (f *randomCluster) Lookup(key shard.Key, _ int, _ shard.Op) ([]peer.Peer, error) { + ind := int(key) + if ind < 0 { + ind = -ind + } + peerIndex := ind % len(f.peers) + if _, ok := f.peersByIndex[peerIndex]; !ok { + f.peersByIndex[peerIndex] = []peer.Peer{f.peers[peerIndex]} + } + return f.peersByIndex[peerIndex], nil +} + +func (f *randomCluster) Peers() []peer.Peer { + return f.peers +} + +func mapToLabelSet(m map[string]string) model.LabelSet { + r := make(model.LabelSet, len(m)) + for k, v := range m { + r[model.LabelName(k)] = model.LabelValue(v) + } + return r +} diff --git a/internal/component/local/file_match/file_test.go b/internal/component/local/file_match/file_test.go index 63645315bd..b63cdb9e41 100644 --- a/internal/component/local/file_match/file_test.go +++ b/internal/component/local/file_match/file_test.go @@ -4,6 +4,7 @@ package file_match import ( + "context" "os" "path" "strings" @@ -12,12 +13,11 @@ import ( "github.com/grafana/alloy/internal/component/discovery" - "context" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/util" - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" ) func TestFile(t *testing.T) { @@ -254,7 +254,9 @@ func TestMultiLabels(t *testing.T) { "foo": "bar", "fruit": "apple", }) - c.args.PathTargets[0]["newlabel"] = "test" + tb := discovery.NewTargetBuilderFrom(c.args.PathTargets[0]) + tb.Set("newlabel", "test") + c.args.PathTargets[0] = tb.Target() ct := context.Background() ct, ccl := context.WithTimeout(ct, 40*time.Second) defer ccl() @@ -278,14 +280,15 @@ func createComponent(t *testing.T, dir string, paths []string, excluded []string func createComponentWithLabels(t *testing.T, dir string, paths []string, excluded []string, labels map[string]string) *Component { tPaths := make([]discovery.Target, 0) for i, p := range paths { - tar := discovery.Target{"__path__": p} + tb := discovery.NewTargetBuilder() + tb.Set("__path__", p) for k, v := range labels { - tar[k] = v + tb.Set(k, v) } if i < len(excluded) { - tar["__path_exclude__"] = excluded[i] + tb.Set("__path_exclude__", excluded[i]) } - tPaths = append(tPaths, tar) + tPaths = append(tPaths, tb.Target()) } c, err := New(component.Options{ ID: "test", @@ -308,7 +311,7 @@ func createComponentWithLabels(t *testing.T, dir string, paths []string, exclude func contains(sources []discovery.Target, match string) bool { for _, s := range sources { - p := s["__path__"] + p, _ := s.Get("__path__") if strings.Contains(p, match) { return true } diff --git a/internal/component/local/file_match/watch.go b/internal/component/local/file_match/watch.go index 04d8e456c3..25682ca878 100644 --- a/internal/component/local/file_match/watch.go +++ b/internal/component/local/file_match/watch.go @@ -59,21 +59,20 @@ func (w *watch) getPaths() ([]discovery.Target, error) { continue } - dt := discovery.Target{} - for dk, v := range w.target { - dt[dk] = v - } - dt["__path__"] = abs - allMatchingPaths = append(allMatchingPaths, dt) + tb := discovery.NewTargetBuilderFrom(w.target) + tb.Set("__path__", abs) + allMatchingPaths = append(allMatchingPaths, tb.Target()) } return allMatchingPaths, nil } func (w *watch) getPath() string { - return w.target["__path__"] + path, _ := w.target.Get("__path__") + return path } func (w *watch) getExcludePath() string { - return w.target["__path_exclude__"] + excludePath, _ := w.target.Get("__path_exclude__") + return excludePath } diff --git a/internal/component/loki/process/process_test.go b/internal/component/loki/process/process_test.go index 7558b0fc6c..a8962d73f6 100644 --- a/internal/component/loki/process/process_test.go +++ b/internal/component/loki/process/process_test.go @@ -400,7 +400,7 @@ stage.static_labels { go func() { err := ctrl.Run(ctx, lsf.Arguments{ - Targets: []discovery.Target{{"__path__": f.Name(), "somelbl": "somevalue"}}, + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__path__": f.Name(), "somelbl": "somevalue"})}, ForwardTo: []loki.LogsReceiver{ tc1.Exports().(Exports).Receiver, tc2.Exports().(Exports).Receiver, diff --git a/internal/component/loki/relabel/relabel_test.go b/internal/component/loki/relabel/relabel_test.go index b0abe69967..ed810bc47b 100644 --- a/internal/component/loki/relabel/relabel_test.go +++ b/internal/component/loki/relabel/relabel_test.go @@ -346,7 +346,7 @@ rule { go func() { err := ctrl.Run(context.Background(), lsf.Arguments{ - Targets: []discovery.Target{{"__path__": f.Name(), "somelbl": "somevalue"}}, + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__path__": f.Name(), "somelbl": "somevalue"})}, ForwardTo: []loki.LogsReceiver{ tc1.Exports().(Exports).Receiver, tc2.Exports().(Exports).Receiver, diff --git a/internal/component/loki/source/docker/docker.go b/internal/component/loki/source/docker/docker.go index ffeb9d26e6..12e36cd42a 100644 --- a/internal/component/loki/source/docker/docker.go +++ b/internal/component/loki/source/docker/docker.go @@ -16,6 +16,10 @@ import ( "github.com/docker/docker/client" "github.com/go-kit/log" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/relabel" + "github.com/grafana/alloy/internal/component" types "github.com/grafana/alloy/internal/component/common/config" "github.com/grafana/alloy/internal/component/common/loki" @@ -26,9 +30,6 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/useragent" - "github.com/prometheus/common/config" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/relabel" ) func init() { @@ -227,11 +228,8 @@ func (c *Component) Update(args component.Arguments) error { promTargets := make([]promTarget, len(newArgs.Targets)) for i, target := range newArgs.Targets { - var labels = make(model.LabelSet) - for k, v := range target { - labels[model.LabelName(k)] = model.LabelValue(v) - } - promTargets[i] = promTarget{labels: labels, fingerPrint: labels.Fingerprint()} + labelsCopy := target.LabelSet() + promTargets[i] = promTarget{labels: labelsCopy, fingerPrint: labelsCopy.Fingerprint()} } // Sorting the targets before filtering ensures consistent filtering of targets diff --git a/internal/component/loki/source/file/file.go b/internal/component/loki/source/file/file.go index b6e7e46e8f..38645cf03f 100644 --- a/internal/component/loki/source/file/file.go +++ b/internal/component/loki/source/file/file.go @@ -5,10 +5,14 @@ import ( "fmt" "os" "path/filepath" - "strings" "sync" "time" + "github.com/grafana/tail/watch" + "github.com/prometheus/common/model" + + "go.uber.org/atomic" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/common/loki" "github.com/grafana/alloy/internal/component/common/loki/positions" @@ -16,9 +20,6 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runner" "github.com/grafana/alloy/internal/runtime/logging/level" - "github.com/grafana/tail/watch" - "github.com/prometheus/common/model" - "go.uber.org/atomic" ) func init() { @@ -192,15 +193,9 @@ func (c *Component) Update(args component.Arguments) error { } for _, target := range newArgs.Targets { - path := target[pathLabel] + path, _ := target.Get(pathLabel) - labels := make(model.LabelSet) - for k, v := range target { - if strings.HasPrefix(k, model.ReservedLabelPrefix) { - continue - } - labels[model.LabelName(k)] = model.LabelValue(v) - } + labels := target.NonReservedLabelSet() // Deduplicate targets which have the same public label set. readersKey := positions.Entry{Path: path, Labels: labels.String()} diff --git a/internal/component/loki/source/file/file_test.go b/internal/component/loki/source/file/file_test.go index 643f2d915c..5c38d4bb97 100644 --- a/internal/component/loki/source/file/file_test.go +++ b/internal/component/loki/source/file/file_test.go @@ -11,16 +11,17 @@ import ( "testing" "time" - "github.com/grafana/alloy/internal/component" - "github.com/grafana/alloy/internal/component/common/loki" - "github.com/grafana/alloy/internal/component/discovery" - "github.com/grafana/alloy/internal/runtime/componenttest" - "github.com/grafana/alloy/internal/util" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "go.uber.org/goleak" "golang.org/x/text/encoding/unicode" + + "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/component/common/loki" + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/internal/runtime/componenttest" + "github.com/grafana/alloy/internal/util" ) func Test(t *testing.T) { @@ -41,10 +42,10 @@ func Test(t *testing.T) { go func() { err := ctrl.Run(ctx, Arguments{ - Targets: []discovery.Target{{ + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{ "__path__": f.Name(), "foo": "bar", - }}, + })}, ForwardTo: []loki.LogsReceiver{ch1, ch2}, }) require.NoError(t, err) @@ -91,10 +92,10 @@ func TestFileWatch(t *testing.T) { ch1 := loki.NewLogsReceiver() args := Arguments{ - Targets: []discovery.Target{{ + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{ "__path__": f.Name(), "foo": "bar", - }}, + })}, ForwardTo: []loki.LogsReceiver{ch1}, FileWatch: FileWatch{ MinPollFrequency: time.Millisecond * 500, @@ -150,10 +151,10 @@ func TestUpdate_NoLeak(t *testing.T) { require.NoError(t, err) args := Arguments{ - Targets: []discovery.Target{{ + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{ "__path__": f.Name(), "foo": "bar", - }}, + })}, ForwardTo: []loki.LogsReceiver{}, } @@ -195,8 +196,8 @@ func TestTwoTargets(t *testing.T) { ch1 := loki.NewLogsReceiver() args := Arguments{} args.Targets = []discovery.Target{ - {"__path__": f.Name(), "foo": "bar"}, - {"__path__": f2.Name(), "foo": "bar2"}, + discovery.NewTargetFromMap(map[string]string{"__path__": f.Name(), "foo": "bar"}), + discovery.NewTargetFromMap(map[string]string{"__path__": f2.Name(), "foo": "bar2"}), } args.ForwardTo = []loki.LogsReceiver{ch1} @@ -265,7 +266,7 @@ func TestEncoding(t *testing.T) { ch1 := loki.NewLogsReceiver() args := Arguments{} - args.Targets = []discovery.Target{{"__path__": f.Name(), "lbl1": "val1"}} + args.Targets = []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__path__": f.Name(), "lbl1": "val1"})} args.Encoding = "UTF-16BE" args.ForwardTo = []loki.LogsReceiver{ch1} @@ -334,10 +335,10 @@ func TestDeleteRecreateFile(t *testing.T) { go func() { err := ctrl.Run(ctx, Arguments{ - Targets: []discovery.Target{{ + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{ "__path__": f.Name(), "foo": "bar", - }}, + })}, ForwardTo: []loki.LogsReceiver{ch1}, }) require.NoError(t, err) diff --git a/internal/component/loki/source/kubernetes/kubernetes.go b/internal/component/loki/source/kubernetes/kubernetes.go index dcab367b4b..da1ee8b3f9 100644 --- a/internal/component/loki/source/kubernetes/kubernetes.go +++ b/internal/component/loki/source/kubernetes/kubernetes.go @@ -11,6 +11,8 @@ import ( "time" "github.com/go-kit/log" + "k8s.io/client-go/kubernetes" + "github.com/grafana/alloy/internal/component" commonk8s "github.com/grafana/alloy/internal/component/common/kubernetes" "github.com/grafana/alloy/internal/component/common/loki" @@ -20,7 +22,6 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/service/cluster" - "k8s.io/client-go/kubernetes" ) func init() { @@ -192,7 +193,7 @@ func (c *Component) resyncTargets(targets []discovery.Target) { tailTargets := make([]*kubetail.Target, 0, len(targets)) for _, target := range targets { - lset := target.Labels() + lset := target.PromLabels() processed, err := kubetail.PrepareLabels(lset, c.opts.ID) if err != nil { // TODO(rfratto): should this set the health of the component? diff --git a/internal/component/loki/source/kubernetes/kubernetes_test.go b/internal/component/loki/source/kubernetes/kubernetes_test.go index dd16a4fa3d..703ee63527 100644 --- a/internal/component/loki/source/kubernetes/kubernetes_test.go +++ b/internal/component/loki/source/kubernetes/kubernetes_test.go @@ -1,13 +1,15 @@ package kubernetes import ( + "testing" + "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/loki/source/kubernetes/kubetail" "github.com/grafana/alloy/internal/service/cluster" - "testing" - "github.com/grafana/alloy/syntax" "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/syntax" ) func TestAlloyConfig(t *testing.T) { @@ -55,20 +57,20 @@ func TestClusteringDuplicateAddress(t *testing.T) { true, cluster.Mock(), []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ "__address__": "localhost:9090", "container": "alloy", "pod": "grafana-k8s-monitoring-alloy-0", "job": "integrations/alloy", "namespace": "default", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__address__": "localhost:8080", "container": "alloy", "pod": "grafana-k8s-monitoring-alloy-0", "job": "integrations/alloy", "namespace": "default", - }, + }), }, kubetail.ClusteringLabels, ) diff --git a/internal/component/loki/write/write_test.go b/internal/component/loki/write/write_test.go index 7ee1c95b21..7015878627 100644 --- a/internal/component/loki/write/write_test.go +++ b/internal/component/loki/write/write_test.go @@ -272,7 +272,7 @@ func testMultipleEndpoint(t *testing.T, alterArgs func(arguments *Arguments)) { go func() { err := ctrl.Run(context.Background(), lsf.Arguments{ - Targets: []discovery.Target{{"__path__": f.Name(), "somelbl": "somevalue"}}, + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__path__": f.Name(), "somelbl": "somevalue"})}, ForwardTo: []loki.LogsReceiver{ tc1.Exports().(Exports).Receiver, tc2.Exports().(Exports).Receiver, diff --git a/internal/component/prometheus/exporter/blackbox/blackbox.go b/internal/component/prometheus/exporter/blackbox/blackbox.go index 0803fd6746..7a40ff736b 100644 --- a/internal/component/prometheus/exporter/blackbox/blackbox.go +++ b/internal/component/prometheus/exporter/blackbox/blackbox.go @@ -45,14 +45,15 @@ func buildBlackboxTargets(baseTarget discovery.Target, args component.Arguments) } for _, tgt := range blackboxTargets { - target := make(discovery.Target) + target := make(map[string]string, len(tgt.Labels)+baseTarget.Len()) // Set extra labels first, meaning that any other labels will override for k, v := range tgt.Labels { target[k] = v } - for k, v := range baseTarget { - target[k] = v - } + baseTarget.ForEachLabel(func(key string, value string) bool { + target[key] = value + return true + }) target["job"] = target["job"] + "/" + tgt.Name target["__param_target"] = tgt.Target @@ -60,7 +61,7 @@ func buildBlackboxTargets(baseTarget discovery.Target, args component.Arguments) target["__param_module"] = tgt.Module } - targets = append(targets, target) + targets = append(targets, discovery.NewTargetFromMap(target)) } return targets diff --git a/internal/component/prometheus/exporter/blackbox/blackbox_test.go b/internal/component/prometheus/exporter/blackbox/blackbox_test.go index 59c6fd2d21..3ad36020de 100644 --- a/internal/component/prometheus/exporter/blackbox/blackbox_test.go +++ b/internal/component/prometheus/exporter/blackbox/blackbox_test.go @@ -4,13 +4,14 @@ import ( "testing" "time" - "github.com/grafana/alloy/internal/component" - "github.com/grafana/alloy/internal/component/discovery" - "github.com/grafana/alloy/syntax" blackbox_config "github.com/prometheus/blackbox_exporter/config" "github.com/prometheus/common/model" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/syntax" ) func TestUnmarshalAlloy(t *testing.T) { @@ -259,20 +260,20 @@ func TestBuildBlackboxTargets(t *testing.T) { Targets: TargetBlock{{Name: "target_a", Target: "http://example.com", Module: "http_2xx"}}, ProbeTimeoutOffset: 1.0, } - baseTarget := discovery.Target{ + baseTarget := discovery.NewTargetFromMap(map[string]string{ model.SchemeLabel: "http", model.MetricsPathLabel: "component/prometheus.exporter.blackbox.default/metrics", "instance": "prometheus.exporter.blackbox.default", "job": "integrations/blackbox", "__meta_agent_integration_name": "blackbox", "__meta_agent_integration_instance": "prometheus.exporter.blackbox.default", - } + }) args := component.Arguments(baseArgs) targets := buildBlackboxTargets(baseTarget, args) require.Equal(t, 1, len(targets)) - require.Equal(t, "integrations/blackbox/target_a", targets[0]["job"]) - require.Equal(t, "http://example.com", targets[0]["__param_target"]) - require.Equal(t, "http_2xx", targets[0]["__param_module"]) + requireTargetLabel(t, targets[0], "job", "integrations/blackbox/target_a") + requireTargetLabel(t, targets[0], "__param_target", "http://example.com") + requireTargetLabel(t, targets[0], "__param_module", "http_2xx") } func TestBuildBlackboxTargetsWithExtraLabels(t *testing.T) { @@ -289,23 +290,23 @@ func TestBuildBlackboxTargetsWithExtraLabels(t *testing.T) { }}, ProbeTimeoutOffset: 1.0, } - baseTarget := discovery.Target{ + baseTarget := discovery.NewTargetFromMap(map[string]string{ model.SchemeLabel: "http", model.MetricsPathLabel: "component/prometheus.exporter.blackbox.default/metrics", "instance": "prometheus.exporter.blackbox.default", "job": "integrations/blackbox", "__meta_agent_integration_name": "blackbox", "__meta_agent_integration_instance": "prometheus.exporter.blackbox.default", - } + }) args := component.Arguments(baseArgs) targets := buildBlackboxTargets(baseTarget, args) require.Equal(t, 1, len(targets)) - require.Equal(t, "integrations/blackbox/target_a", targets[0]["job"]) - require.Equal(t, "http://example.com", targets[0]["__param_target"]) - require.Equal(t, "http_2xx", targets[0]["__param_module"]) + requireTargetLabel(t, targets[0], "job", "integrations/blackbox/target_a") + requireTargetLabel(t, targets[0], "__param_target", "http://example.com") + requireTargetLabel(t, targets[0], "__param_module", "http_2xx") - require.Equal(t, "test", targets[0]["env"]) - require.Equal(t, "bar", targets[0]["foo"]) + requireTargetLabel(t, targets[0], "env", "test") + requireTargetLabel(t, targets[0], "foo", "bar") // Check that the extra labels do not override existing labels baseArgs.Targets[0].Labels = map[string]string{ @@ -315,8 +316,8 @@ func TestBuildBlackboxTargetsWithExtraLabels(t *testing.T) { args = component.Arguments(baseArgs) targets = buildBlackboxTargets(baseTarget, args) require.Equal(t, 1, len(targets)) - require.Equal(t, "integrations/blackbox/target_a", targets[0]["job"]) - require.Equal(t, "prometheus.exporter.blackbox.default", targets[0]["instance"]) + requireTargetLabel(t, targets[0], "job", "integrations/blackbox/target_a") + requireTargetLabel(t, targets[0], "instance", "prometheus.exporter.blackbox.default") } // Test convert from TargetsList to []blackbox_exporter.BlackboxTarget @@ -451,3 +452,10 @@ func TestValidateTargetsMutualExclusivity(t *testing.T) { } require.ErrorContains(t, args.Validate(), "the block `target` and the attribute `targets` are mutually exclusive") } + +func requireTargetLabel(t *testing.T, target discovery.Target, label, expectedValue string) { + t.Helper() + actual, ok := target.Get(label) + require.True(t, ok) + require.Equal(t, expectedValue, actual) +} diff --git a/internal/component/prometheus/exporter/exporter.go b/internal/component/prometheus/exporter/exporter.go index 466af3c3a1..9714ce3624 100644 --- a/internal/component/prometheus/exporter/exporter.go +++ b/internal/component/prometheus/exporter/exporter.go @@ -9,12 +9,13 @@ import ( "strings" "sync" + "github.com/prometheus/common/model" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/runtime/logging/level" http_service "github.com/grafana/alloy/internal/service/http" "github.com/grafana/alloy/internal/static/integrations" - "github.com/prometheus/common/model" ) // Creator is a function provided by an implementation to create a concrete exporter instance. @@ -92,7 +93,9 @@ func (c *Component) Update(args component.Arguments) error { c.mut.Lock() c.exporter = exporter if instanceKey != "" { - c.baseTarget["instance"] = instanceKey + tb := discovery.NewTargetBuilderFrom(c.baseTarget) + tb.Set("instance", instanceKey) + c.baseTarget = tb.Target() } var targets []discovery.Target @@ -142,7 +145,7 @@ func newExporter(creator Creator, name string, targetBuilderFunc func(discovery. componentName = opts.ID } - c.baseTarget = discovery.Target{ + c.baseTarget = discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: httpData.MemoryListenAddr, model.SchemeLabel: "http", model.MetricsPathLabel: path.Join(httpData.HTTPPathForComponent(opts.ID), "metrics"), @@ -150,8 +153,7 @@ func newExporter(creator Creator, name string, targetBuilderFunc func(discovery. "job": jobName, "__meta_component_name": componentName, "__meta_component_id": opts.ID, - } - + }) // Call to Update() to set the output once at the start. if err := c.Update(args); err != nil { return nil, err diff --git a/internal/component/prometheus/exporter/kafka/kafka.go b/internal/component/prometheus/exporter/kafka/kafka.go index 26e64a7421..6c64821f6a 100644 --- a/internal/component/prometheus/exporter/kafka/kafka.go +++ b/internal/component/prometheus/exporter/kafka/kafka.go @@ -4,6 +4,8 @@ import ( "fmt" "github.com/IBM/sarama" + "github.com/prometheus/common/config" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/prometheus/exporter" @@ -11,7 +13,6 @@ import ( "github.com/grafana/alloy/internal/static/integrations" "github.com/grafana/alloy/internal/static/integrations/kafka_exporter" "github.com/grafana/alloy/syntax/alloytypes" - "github.com/prometheus/common/config" ) var DefaultArguments = Arguments{ @@ -92,13 +93,13 @@ func (a *Arguments) Validate() error { func customizeTarget(baseTarget discovery.Target, args component.Arguments) []discovery.Target { a := args.(Arguments) - target := baseTarget + targetBuilder := discovery.NewTargetBuilderFrom(baseTarget) if len(a.KafkaURIs) > 1 { - target["instance"] = a.Instance + targetBuilder.Set("instance", a.Instance) } else { - target["instance"] = a.KafkaURIs[0] + targetBuilder.Set("instance", a.KafkaURIs[0]) } - return []discovery.Target{target} + return []discovery.Target{targetBuilder.Target()} } func createExporter(opts component.Options, args component.Arguments, defaultInstanceKey string) (integrations.Integration, string, error) { diff --git a/internal/component/prometheus/exporter/kafka/kafka_test.go b/internal/component/prometheus/exporter/kafka/kafka_test.go index 58f165402f..d257ad6068 100644 --- a/internal/component/prometheus/exporter/kafka/kafka_test.go +++ b/internal/component/prometheus/exporter/kafka/kafka_test.go @@ -3,10 +3,11 @@ package kafka import ( "testing" + "github.com/stretchr/testify/require" + "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/static/integrations/kafka_exporter" "github.com/grafana/alloy/syntax" - "github.com/stretchr/testify/require" ) func TestAlloyUnmarshal(t *testing.T) { @@ -116,7 +117,9 @@ func TestCustomizeTarget(t *testing.T) { baseTarget := discovery.Target{} newTargets := customizeTarget(baseTarget, args) require.Equal(t, 1, len(newTargets)) - require.Equal(t, "example", newTargets[0]["instance"]) + val, ok := newTargets[0].Get("instance") + require.True(t, ok) + require.Equal(t, "example", val) } func TestSASLPassword(t *testing.T) { // #6044 diff --git a/internal/component/prometheus/exporter/snmp/snmp.go b/internal/component/prometheus/exporter/snmp/snmp.go index 0f89195828..6b2712f37d 100644 --- a/internal/component/prometheus/exporter/snmp/snmp.go +++ b/internal/component/prometheus/exporter/snmp/snmp.go @@ -6,6 +6,9 @@ import ( "slices" "time" + snmp_config "github.com/prometheus/snmp_exporter/config" + "gopkg.in/yaml.v2" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/prometheus/exporter" @@ -13,8 +16,6 @@ import ( "github.com/grafana/alloy/internal/static/integrations" "github.com/grafana/alloy/internal/static/integrations/snmp_exporter" "github.com/grafana/alloy/syntax/alloytypes" - snmp_config "github.com/prometheus/snmp_exporter/config" - "gopkg.in/yaml.v2" ) func init() { @@ -35,6 +36,7 @@ func createExporter(opts component.Options, args component.Arguments, defaultIns // buildSNMPTargets creates the exporter's discovery targets based on the defined SNMP targets. func buildSNMPTargets(baseTarget discovery.Target, args component.Arguments) []discovery.Target { + // TODO: This implementation of targets manipulation may not be optimal. If it's a hot spot, we should optimise it. var targets []discovery.Target snmpTargets := args.(Arguments).Targets @@ -44,14 +46,15 @@ func buildSNMPTargets(baseTarget discovery.Target, args component.Arguments) []d } for _, tgt := range snmpTargets { - target := make(discovery.Target) + target := make(map[string]string, len(tgt.Labels)+baseTarget.Len()) // Set extra labels first, meaning that any other labels will override for k, v := range tgt.Labels { target[k] = v } - for k, v := range baseTarget { - target[k] = v - } + baseTarget.ForEachLabel(func(key string, value string) bool { + target[key] = value + return true + }) target["job"] = target["job"] + "/" + tgt.Name target["__param_target"] = tgt.Target @@ -69,7 +72,7 @@ func buildSNMPTargets(baseTarget discovery.Target, args component.Arguments) []d target["__param_auth"] = tgt.Auth } - targets = append(targets, target) + targets = append(targets, discovery.NewTargetFromMap(target)) } return targets diff --git a/internal/component/prometheus/exporter/snmp/snmp_test.go b/internal/component/prometheus/exporter/snmp/snmp_test.go index 1f8e34279e..da6bc09a78 100644 --- a/internal/component/prometheus/exporter/snmp/snmp_test.go +++ b/internal/component/prometheus/exporter/snmp/snmp_test.go @@ -344,22 +344,22 @@ func TestBuildSNMPTargets(t *testing.T) { WalkParams: "public", Auth: "public_v2"}}, WalkParams: WalkParams{{Name: "public", Retries: 2}}, } - baseTarget := discovery.Target{ + baseTarget := discovery.NewTargetFromMap(map[string]string{ model.SchemeLabel: "http", model.MetricsPathLabel: "component/prometheus.exporter.snmp.default/metrics", "instance": "prometheus.exporter.snmp.default", "job": "integrations/snmp", "__meta_agent_integration_name": "snmp", "__meta_agent_integration_instance": "prometheus.exporter.snmp.default", - } + }) args := component.Arguments(baseArgs) targets := buildSNMPTargets(baseTarget, args) require.Equal(t, 1, len(targets)) - require.Equal(t, "integrations/snmp/network_switch_1", targets[0]["job"]) - require.Equal(t, "192.168.1.2", targets[0]["__param_target"]) - require.Equal(t, "if_mib", targets[0]["__param_module"]) - require.Equal(t, "public", targets[0]["__param_walk_params"]) - require.Equal(t, "public_v2", targets[0]["__param_auth"]) + requireTargetLabel(t, targets[0], "job", "integrations/snmp/network_switch_1") + requireTargetLabel(t, targets[0], "__param_target", "192.168.1.2") + requireTargetLabel(t, targets[0], "__param_module", "if_mib") + requireTargetLabel(t, targets[0], "__param_walk_params", "public") + requireTargetLabel(t, targets[0], "__param_auth", "public_v2") } func TestUnmarshalAlloyWithInlineConfig(t *testing.T) { @@ -465,3 +465,10 @@ func TestUnmarshalAlloyWithInvalidInlineConfig(t *testing.T) { }) } } + +func requireTargetLabel(t *testing.T, target discovery.Target, label, expectedValue string) { + t.Helper() + actual, ok := target.Get(label) + require.True(t, ok) + require.Equal(t, expectedValue, actual) +} diff --git a/internal/component/prometheus/scrape/scrape.go b/internal/component/prometheus/scrape/scrape.go index 665c45ca32..f433898c64 100644 --- a/internal/component/prometheus/scrape/scrape.go +++ b/internal/component/prometheus/scrape/scrape.go @@ -336,7 +336,7 @@ func (c *Component) distributeTargets( newLocalTargets := newDistTargets.LocalTargets() c.targetsGauge.Set(float64(len(newLocalTargets))) - promNewTargets := c.componentTargetsToPromTargetGroups(jobName, newLocalTargets) + promNewTargets := discovery.ComponentTargetsToPromTargetGroups(jobName, newLocalTargets) movedTargets := newDistTargets.MovedToRemoteInstance(oldDistributedTargets) c.movedTargetsCounter.Add(float64(len(movedTargets))) @@ -483,34 +483,25 @@ func (c *Component) DebugInfo() interface{} { } } -func (c *Component) componentTargetsToPromTargetGroups(jobName string, tgs []discovery.Target) map[string][]*targetgroup.Group { - promGroup := &targetgroup.Group{Source: jobName} - for _, tg := range tgs { - promGroup.Targets = append(promGroup.Targets, convertLabelSet(tg)) - } - - return map[string][]*targetgroup.Group{jobName: {promGroup}} -} - func (c *Component) populatePromLabels(targets []discovery.Target, jobName string, args Arguments) []*scrape.Target { - lb := labels.NewBuilder(labels.EmptyLabels()) - promTargets, errs := scrape.TargetsFromGroup( - c.componentTargetsToPromTargetGroups(jobName, targets)[jobName][0], - getPromScrapeConfigs(c.opts.ID, args), - false, /* noDefaultScrapePort - always false in this component */ - make([]*scrape.Target, len(targets)), /* targets slice to reuse */ - lb, - ) - for _, err := range errs { - level.Warn(c.opts.Logger).Log("msg", "error while populating labels of targets using prom config", "err", err) + // We need to call scrape.TargetsFromGroup to reuse the rather complex logic of populating labels on targets. + allTargets := make([]*scrape.Target, 0, len(targets)) + groups := discovery.ComponentTargetsToPromTargetGroups(jobName, targets) + for _, tgs := range groups { + for _, tg := range tgs { + promTargets, errs := scrape.TargetsFromGroup( + tg, + getPromScrapeConfigs(jobName, args), + false, /* noDefaultScrapePort - always false in this component */ + make([]*scrape.Target, len(targets)), /* targets slice to reuse */ + labels.NewBuilder(labels.EmptyLabels()), + ) + for _, err := range errs { + level.Warn(c.opts.Logger).Log("msg", "error while populating labels of targets using prom config", "err", err) + } + allTargets = append(allTargets, promTargets...) + } } - return promTargets -} -func convertLabelSet(tg discovery.Target) model.LabelSet { - lset := make(model.LabelSet, len(tg)) - for k, v := range tg { - lset[model.LabelName(k)] = model.LabelValue(v) - } - return lset + return allTargets } diff --git a/internal/component/prometheus/scrape/scrape_clustering_test.go b/internal/component/prometheus/scrape/scrape_clustering_test.go index e735a9f816..d51c393c9b 100644 --- a/internal/component/prometheus/scrape/scrape_clustering_test.go +++ b/internal/component/prometheus/scrape/scrape_clustering_test.go @@ -38,7 +38,7 @@ var ( // There is a race condition in prometheus where calls to NewManager can race over a package-global variable when // calling targetMetadataCache.registerManager(m). This is a workaround to prevent this for now. - //TODO(thampiotr): Open an issue in prometheus to fix this? + // TODO(thampiotr): Open an issue in prometheus to fix this? promManagerMutex sync.Mutex ) @@ -260,7 +260,7 @@ func setUpClusterLookup(fakeCluster *fakeCluster, assignment map[peer.Peer][]int fakeCluster.lookupMap = make(map[shard.Key][]peer.Peer) for owningPeer, ownedTargets := range assignment { for _, id := range ownedTargets { - fakeCluster.lookupMap[shard.Key(targets[id].Target().NonMetaLabels().Hash())] = []peer.Peer{owningPeer} + fakeCluster.lookupMap[shard.Key(targets[id].Target().NonMetaLabelsHash())] = []peer.Peer{owningPeer} } } } diff --git a/internal/component/pyroscope/ebpf/ebpf_linux.go b/internal/component/pyroscope/ebpf/ebpf_linux.go index cf51ea64a6..843447b330 100644 --- a/internal/component/pyroscope/ebpf/ebpf_linux.go +++ b/internal/component/pyroscope/ebpf/ebpf_linux.go @@ -10,16 +10,17 @@ import ( "sync" "time" - "github.com/grafana/alloy/internal/component" - "github.com/grafana/alloy/internal/component/pyroscope" - "github.com/grafana/alloy/internal/featuregate" - "github.com/grafana/alloy/internal/runtime/logging/level" ebpfspy "github.com/grafana/pyroscope/ebpf" demangle2 "github.com/grafana/pyroscope/ebpf/cpp/demangle" "github.com/grafana/pyroscope/ebpf/pprof" "github.com/grafana/pyroscope/ebpf/sd" "github.com/grafana/pyroscope/ebpf/symtab" "github.com/oklog/run" + + "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/component/pyroscope" + "github.com/grafana/alloy/internal/featuregate" + "github.com/grafana/alloy/internal/runtime/logging/level" ) func init() { @@ -221,7 +222,7 @@ func (c *Component) updateDebugInfo() { func targetsOptionFromArgs(args Arguments) sd.TargetsOptions { targets := make([]sd.DiscoveryTarget, 0, len(args.Targets)) for _, t := range args.Targets { - targets = append(targets, sd.DiscoveryTarget(t)) + targets = append(targets, t.AsMap()) } return sd.TargetsOptions{ Targets: targets, diff --git a/internal/component/pyroscope/ebpf/ebpf_linux_test.go b/internal/component/pyroscope/ebpf/ebpf_linux_test.go index 394db0a67d..c59fc901d5 100644 --- a/internal/component/pyroscope/ebpf/ebpf_linux_test.go +++ b/internal/component/pyroscope/ebpf/ebpf_linux_test.go @@ -198,10 +198,10 @@ forward_to = [] expected: func() Arguments { x := NewDefaultArguments() x.Targets = []discovery.Target{ - map[string]string{ + discovery.NewTargetFromMap(map[string]string{ "container_id": "cid", "service_name": "foo", - }, + }), } x.ForwardTo = []pyroscope.Appendable{} return x @@ -224,10 +224,10 @@ collect_kernel_profile = false`, expected: func() Arguments { x := NewDefaultArguments() x.Targets = []discovery.Target{ - map[string]string{ + discovery.NewTargetFromMap(map[string]string{ "container_id": "cid", "service_name": "foo", - }, + }), } x.ForwardTo = []pyroscope.Appendable{} x.CollectInterval = time.Second * 3 diff --git a/internal/component/pyroscope/java/java.go b/internal/component/pyroscope/java/java.go index 129bafe411..c7e76f7112 100644 --- a/internal/component/pyroscope/java/java.go +++ b/internal/component/pyroscope/java/java.go @@ -126,9 +126,14 @@ func (j *javaComponent) updateTargets(args Arguments) { active := make(map[int]struct{}) for _, target := range args.Targets { - pid64, err := strconv.ParseInt(target[labelProcessID], 10, 32) + pidStr, ok := target.Get(labelProcessID) + if !ok { + _ = level.Error(j.opts.Logger).Log("msg", "could not find PID label", "pid", pidStr) + continue + } + pid64, err := strconv.ParseInt(pidStr, 10, 32) if err != nil { - _ = level.Error(j.opts.Logger).Log("msg", "could not convert process ID to a 32 bit integer", "pid", target[labelProcessID], "err", err) + _ = level.Error(j.opts.Logger).Log("msg", "could not convert process ID to a 32 bit integer", "pid", pidStr, "err", err) continue } pid := int(pid64) diff --git a/internal/component/pyroscope/java/loop.go b/internal/component/pyroscope/java/loop.go index 03664e4939..ff6acf9297 100644 --- a/internal/component/pyroscope/java/loop.go +++ b/internal/component/pyroscope/java/loop.go @@ -13,14 +13,15 @@ import ( "time" "github.com/go-kit/log" - "github.com/grafana/alloy/internal/component/discovery" - "github.com/grafana/alloy/internal/component/pyroscope" - "github.com/grafana/alloy/internal/component/pyroscope/java/asprof" - "github.com/grafana/alloy/internal/runtime/logging/level" jfrpprof "github.com/grafana/jfr-parser/pprof" jfrpprofPyroscope "github.com/grafana/jfr-parser/pprof/pyroscope" "github.com/prometheus/prometheus/model/labels" gopsutil "github.com/shirou/gopsutil/v3/process" + + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/internal/component/pyroscope" + "github.com/grafana/alloy/internal/component/pyroscope/java/asprof" + "github.com/grafana/alloy/internal/runtime/logging/level" ) const spyName = "alloy.java" @@ -155,7 +156,7 @@ func (p *profilingLoop) push(jfrBytes []byte, startTime time.Time, endTime time. sz := req.Profile.SizeVT() l := log.With(p.logger, "metric", metric, "sz", sz) ls := labels.NewBuilder(nil) - for _, l := range jfrpprofPyroscope.Labels(target, profiles.JFREvent, req.Metric, "", spyName) { + for _, l := range jfrpprofPyroscope.Labels(target.AsMap(), profiles.JFREvent, req.Metric, "", spyName) { ls.Set(l.Name, l.Value) } if ls.Get(labelServiceName) == "" { diff --git a/internal/component/pyroscope/java/target.go b/internal/component/pyroscope/java/target.go index cffdf967b5..3cda7ea0ef 100644 --- a/internal/component/pyroscope/java/target.go +++ b/internal/component/pyroscope/java/target.go @@ -12,23 +12,22 @@ const ( ) func inferServiceName(target discovery.Target) string { - k8sServiceName := target[labelServiceNameK8s] - if k8sServiceName != "" { + if k8sServiceName, ok := target.Get(labelServiceNameK8s); ok { return k8sServiceName } - k8sNamespace := target["__meta_kubernetes_namespace"] - k8sContainer := target["__meta_kubernetes_pod_container_name"] - if k8sNamespace != "" && k8sContainer != "" { + k8sNamespace, nsOk := target.Get("__meta_kubernetes_namespace") + k8sContainer, contOk := target.Get("__meta_kubernetes_pod_container_name") + if nsOk && contOk { return fmt.Sprintf("java/%s/%s", k8sNamespace, k8sContainer) } - dockerContainer := target["__meta_docker_container_name"] - if dockerContainer != "" { + + if dockerContainer, ok := target.Get("__meta_docker_container_name"); ok { return dockerContainer } - if swarmService := target["__meta_dockerswarm_container_label_service_name"]; swarmService != "" { + if swarmService, ok := target.Get("__meta_dockerswarm_container_label_service_name"); ok { return swarmService } - if swarmService := target["__meta_dockerswarm_service_name"]; swarmService != "" { + if swarmService, ok := target.Get("__meta_dockerswarm_service_name"); ok { return swarmService } return "unspecified" diff --git a/internal/component/pyroscope/scrape/scrape.go b/internal/component/pyroscope/scrape/scrape.go index 59ad720504..337e2f515a 100644 --- a/internal/component/pyroscope/scrape/scrape.go +++ b/internal/component/pyroscope/scrape/scrape.go @@ -8,7 +8,6 @@ import ( "time" config_util "github.com/prometheus/common/config" - "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/grafana/alloy/internal/component/pyroscope" @@ -322,7 +321,7 @@ func (c *Component) Run(ctx context.Context) error { // NOTE(@tpaschalis) First approach, manually building the // 'clustered' targets implementation every time. ct := discovery.NewDistributedTargets(clusteringEnabled, c.cluster, tgs) - promTargets := c.componentTargetsToProm(jobName, ct.LocalTargets()) + promTargets := discovery.ComponentTargetsToPromTargetGroups(jobName, ct.LocalTargets()) select { case targetSetsChan <- promTargets: @@ -374,23 +373,6 @@ func (c *Component) NotifyClusterChange() { } } -func (c *Component) componentTargetsToProm(jobName string, tgs []discovery.Target) map[string][]*targetgroup.Group { - promGroup := &targetgroup.Group{Source: jobName} - for _, tg := range tgs { - promGroup.Targets = append(promGroup.Targets, convertLabelSet(tg)) - } - - return map[string][]*targetgroup.Group{jobName: {promGroup}} -} - -func convertLabelSet(tg discovery.Target) model.LabelSet { - lset := make(model.LabelSet, len(tg)) - for k, v := range tg { - lset[model.LabelName(k)] = model.LabelValue(v) - } - return lset -} - // DebugInfo implements component.DebugComponent. func (c *Component) DebugInfo() interface{} { var res []scrape.TargetStatus diff --git a/internal/component/pyroscope/scrape/scrape_loop_test.go b/internal/component/pyroscope/scrape/scrape_loop_test.go index 0a0f9fbdce..6251f4c276 100644 --- a/internal/component/pyroscope/scrape/scrape_loop_test.go +++ b/internal/component/pyroscope/scrape/scrape_loop_test.go @@ -11,9 +11,6 @@ import ( "time" "github.com/go-kit/log" - "github.com/grafana/alloy/internal/component/discovery" - "github.com/grafana/alloy/internal/component/pyroscope" - "github.com/grafana/alloy/internal/util" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -21,6 +18,10 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" "go.uber.org/goleak" + + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/internal/component/pyroscope" + "github.com/grafana/alloy/internal/util" ) func TestScrapePool(t *testing.T) { @@ -28,7 +29,7 @@ func TestScrapePool(t *testing.T) { args := NewDefaultArguments() args.Targets = []discovery.Target{ - {"instance": "foo"}, + discovery.NewTargetFromMap(map[string]string{"instance": "foo"}), } args.ProfilingConfig.Block.Enabled = false args.ProfilingConfig.Goroutine.Enabled = false diff --git a/internal/component/pyroscope/scrape/scrape_test.go b/internal/component/pyroscope/scrape/scrape_test.go index 21a3d93731..a2fb4796ef 100644 --- a/internal/component/pyroscope/scrape/scrape_test.go +++ b/internal/component/pyroscope/scrape/scrape_test.go @@ -10,6 +10,12 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + "go.uber.org/goleak" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/component/prometheus/scrape" @@ -18,11 +24,6 @@ import ( http_service "github.com/grafana/alloy/internal/service/http" "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/syntax" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - "go.uber.org/atomic" - "go.uber.org/goleak" ) func TestComponent(t *testing.T) { @@ -50,14 +51,14 @@ func TestComponent(t *testing.T) { arg.ForwardTo = []pyroscope.Appendable{pyroscope.NoopAppendable} arg.Targets = []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: "foo", serviceNameLabel: "s", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: "bar", serviceNameK8SLabel: "k", - }, + }), } c.Update(arg) @@ -99,10 +100,10 @@ func TestUnmarshalConfig(t *testing.T) { expected: func() Arguments { r := NewDefaultArguments() r.Targets = []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ "__address__": "localhost:9090", "foo": "bar", - }, + }), } return r }, @@ -131,14 +132,14 @@ func TestUnmarshalConfig(t *testing.T) { expected: func() Arguments { r := NewDefaultArguments() r.Targets = []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ "__address__": "localhost:9090", "foo": "bar", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ "__address__": "localhost:8080", "foo": "buzz", - }, + }), } r.ProfilingConfig.Block.Enabled = false r.ProfilingConfig.Custom = append(r.ProfilingConfig.Custom, CustomProfilingTarget{ @@ -266,16 +267,16 @@ func TestUpdateWhileScraping(t *testing.T) { go c.Run(ctx) args.Targets = []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: address, serviceNameLabel: "s", "foo": "bar", - }, - { + }), + discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: address, serviceNameK8SLabel: "k", "foo": "buz", - }, + }), } c.Update(args) @@ -290,11 +291,11 @@ func TestUpdateWhileScraping(t *testing.T) { go func() { for i := 0; i < 100; i++ { args.Targets = []discovery.Target{ - { + discovery.NewTargetFromMap(map[string]string{ model.AddressLabel: address, serviceNameLabel: "s", "foo": fmt.Sprintf("%d", i), - }, + }), } require.NoError(t, c.Update(args)) c.scraper.reload() diff --git a/internal/converter/internal/common/convert_targets.go b/internal/converter/internal/common/convert_targets.go index a6a79690c7..5b49c12d60 100644 --- a/internal/converter/internal/common/convert_targets.go +++ b/internal/converter/internal/common/convert_targets.go @@ -22,7 +22,7 @@ func NewDiscoveryExports(expr string) discovery.Exports { // as a component export string rather than the standard [discovery.Target] // AlloyTokenize. func NewDiscoveryTargets(expr string) []discovery.Target { - return []discovery.Target{map[string]string{"__expr__": expr}} + return []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__expr__": expr})} } // ConvertTargets implements [builder.Tokenizer]. This allows us to set @@ -52,9 +52,9 @@ func (f ConvertTargets) AlloyTokenize() []builder.Token { toks = append(toks, builder.Token{Tok: token.LITERAL, Lit: "\n"}) } - for ix, targetMap := range f.Targets { + for ix, target := range f.Targets { keyValMap := map[string]string{} - for key, val := range targetMap { + target.ForEachLabel(func(key string, val string) bool { // __expr__ is a special key used by the converter code to specify // we should tokenize the value instead of tokenizing the map normally. // An alternative strategy would have been to add a new property for @@ -68,7 +68,8 @@ func (f ConvertTargets) AlloyTokenize() []builder.Token { } else { keyValMap[key] = val } - } + return true + }) if len(keyValMap) > 0 { expr.SetValue([]map[string]string{keyValMap}) diff --git a/internal/converter/internal/common/convert_targets_test.go b/internal/converter/internal/common/convert_targets_test.go index 2135fe20f7..1d33ef8eda 100644 --- a/internal/converter/internal/common/convert_targets_test.go +++ b/internal/converter/internal/common/convert_targets_test.go @@ -3,10 +3,11 @@ package common_test import ( "testing" + "github.com/stretchr/testify/require" + "github.com/grafana/alloy/internal/component/discovery" "github.com/grafana/alloy/internal/converter/internal/common" "github.com/grafana/alloy/syntax/token/builder" - "github.com/stretchr/testify/require" ) func TestOptionalSecret_Write(t *testing.T) { @@ -25,14 +26,14 @@ func TestOptionalSecret_Write(t *testing.T) { { name: "empty", value: common.ConvertTargets{ - Targets: []discovery.Target{{}}, + Targets: []discovery.Target{}, }, - expect: ``, + expect: `[]`, }, { name: "__address__ key", value: common.ConvertTargets{ - Targets: []discovery.Target{{"__address__": "testing"}}, + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__address__": "testing"})}, }, expect: `[{ __address__ = "testing", @@ -41,7 +42,7 @@ func TestOptionalSecret_Write(t *testing.T) { { name: "__address__ key label", value: common.ConvertTargets{ - Targets: []discovery.Target{{"__address__": "testing", "label": "value"}}, + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__address__": "testing", "label": "value"})}, }, expect: `[{ __address__ = "testing", @@ -52,8 +53,8 @@ func TestOptionalSecret_Write(t *testing.T) { name: "multiple __address__ key label", value: common.ConvertTargets{ Targets: []discovery.Target{ - {"__address__": "testing", "label": "value"}, - {"__address__": "testing2", "label": "value"}, + discovery.NewTargetFromMap(map[string]string{"__address__": "testing", "label": "value"}), + discovery.NewTargetFromMap(map[string]string{"__address__": "testing2", "label": "value"}), }, }, expect: `array.concat( @@ -70,14 +71,17 @@ func TestOptionalSecret_Write(t *testing.T) { { name: "__expr__ key", value: common.ConvertTargets{ - Targets: []discovery.Target{{"__expr__": "testing"}}, + Targets: []discovery.Target{discovery.NewTargetFromMap(map[string]string{"__expr__": "testing"})}, }, expect: `testing`, }, { name: "multiple __expr__ key", value: common.ConvertTargets{ - Targets: []discovery.Target{{"__expr__": "testing"}, {"__expr__": "testing2"}}, + Targets: []discovery.Target{ + discovery.NewTargetFromMap(map[string]string{"__expr__": "testing"}), + discovery.NewTargetFromMap(map[string]string{"__expr__": "testing2"}), + }, }, expect: `array.concat( testing, @@ -87,7 +91,10 @@ func TestOptionalSecret_Write(t *testing.T) { { name: "both key types", value: common.ConvertTargets{ - Targets: []discovery.Target{{"__address__": "testing", "label": "value"}, {"__expr__": "testing2"}}, + Targets: []discovery.Target{ + discovery.NewTargetFromMap(map[string]string{"__address__": "testing", "label": "value"}), + discovery.NewTargetFromMap(map[string]string{"__expr__": "testing2"}), + }, }, expect: `array.concat( [{ diff --git a/internal/converter/internal/prometheusconvert/component/scrape.go b/internal/converter/internal/prometheusconvert/component/scrape.go index 13022f5774..1f63d652fe 100644 --- a/internal/converter/internal/prometheusconvert/component/scrape.go +++ b/internal/converter/internal/prometheusconvert/component/scrape.go @@ -90,7 +90,7 @@ func getScrapeTargets(staticConfig prom_discovery.StaticConfig) []discovery.Targ targetMap[string(labelName)] = string(labelValue) newMap := map[string]string{} maps.Copy(newMap, targetMap) - targets = append(targets, newMap) + targets = append(targets, discovery.NewTargetFromMap(newMap)) } } } diff --git a/internal/converter/internal/promtailconvert/internal/build/service_discovery.go b/internal/converter/internal/promtailconvert/internal/build/service_discovery.go index c55587c6e3..2f20c9f1a3 100644 --- a/internal/converter/internal/promtailconvert/internal/build/service_discovery.go +++ b/internal/converter/internal/promtailconvert/internal/build/service_discovery.go @@ -26,7 +26,7 @@ func (s *ScrapeConfigBuilder) AppendSDs() { targetLiterals := make([]discovery.Target, 0) for _, target := range targets { - if expr, ok := target["__expr__"]; ok { + if expr, ok := target.Get("__expr__"); ok { // use the expression if __expr__ is set s.allTargetsExps = append(s.allTargetsExps, expr) } else { diff --git a/internal/converter/internal/test_common/testing.go b/internal/converter/internal/test_common/testing.go index a315f0362c..7bc06364dd 100644 --- a/internal/converter/internal/test_common/testing.go +++ b/internal/converter/internal/test_common/testing.go @@ -12,6 +12,9 @@ import ( "strings" "testing" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "github.com/grafana/alloy/internal/converter/diag" "github.com/grafana/alloy/internal/featuregate" alloy_runtime "github.com/grafana/alloy/internal/runtime" @@ -20,9 +23,8 @@ import ( cluster_service "github.com/grafana/alloy/internal/service/cluster" http_service "github.com/grafana/alloy/internal/service/http" "github.com/grafana/alloy/internal/service/labelstore" + "github.com/grafana/alloy/internal/service/livedebugging" remotecfg_service "github.com/grafana/alloy/internal/service/remotecfg" - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" ) const ( @@ -214,6 +216,7 @@ func attemptLoadingAlloyConfig(t *testing.T, bb []byte) { clusterService, labelstore.New(nil, prometheus.DefaultRegisterer), remotecfgService, + livedebugging.New(), }, EnableCommunityComps: true, }) diff --git a/internal/runtime/componenttest/componenttest.go b/internal/runtime/componenttest/componenttest.go index dc5c1b9e2d..4526ba00c9 100644 --- a/internal/runtime/componenttest/componenttest.go +++ b/internal/runtime/componenttest/componenttest.go @@ -5,19 +5,21 @@ import ( "context" "fmt" "os" - "reflect" "sync" "time" - "github.com/grafana/alloy/internal/service/labelstore" - "github.com/grafana/alloy/internal/service/livedebugging" "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" + "github.com/grafana/alloy/internal/runtime/equality" + "github.com/grafana/alloy/internal/service/labelstore" + "github.com/grafana/alloy/internal/service/livedebugging" + "github.com/go-kit/log" + "go.opentelemetry.io/otel/trace/noop" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/runtime/logging" - "go.opentelemetry.io/otel/trace/noop" ) // A Controller is a testing controller which controls a single component. @@ -66,7 +68,7 @@ func NewControllerFromReg(l log.Logger, reg component.Registration) *Controller func (c *Controller) onStateChange(e component.Exports) { c.exportsMut.Lock() - changed := !reflect.DeepEqual(c.exports, e) + changed := !equality.DeepEqual(c.exports, e) c.exports = e c.exportsMut.Unlock() diff --git a/internal/runtime/equality/equality.go b/internal/runtime/equality/equality.go new file mode 100644 index 0000000000..65b6b506fd --- /dev/null +++ b/internal/runtime/equality/equality.go @@ -0,0 +1,139 @@ +package equality + +import ( + "reflect" +) + +var customEqualityType = reflect.TypeOf((*CustomEquality)(nil)).Elem() + +// CustomEquality allows to define custom Equals implementation. This can be used, for example, with exported types, +// so that the Runtime can short-circuit propagating updates when it is not necessary. If a struct is passed to +// DeepEqual, the `other` supplied to Equals will be a pointer. See tests for reference. +type CustomEquality interface { + Equals(other any) bool +} + +// DeepEqual is a wrapper around reflect.DeepEqual, which first checks if arguments implement CustomEquality. If they +// do, their Equals method is used for comparison instead of reflect.DeepEqual. +// For simplicity, DeepEqual requires x and y to be of the same type before calling CustomEquality.Equals. +// NOTE: structs, slices, maps and arrays that contain a mix of values implementing CustomEquality and not implementing it +// are not supported. Unexported fields are not supported either. In those cases, implement CustomEquality on higher +// level of your object hierarchy. +func DeepEqual(x, y any) bool { + if x == nil || y == nil { + return x == y + } + v1 := reflect.ValueOf(x) + v2 := reflect.ValueOf(y) + + // See if we can compare them using CustomEquality + if r := deepCustomEqual(v1, v2); r.compared { + return r.isEqual + } + // Otherwise fall back to reflect.DeepEqual + return reflect.DeepEqual(x, y) +} + +type result struct { + compared bool + isEqual bool +} + +func successfulCompare(isEqual bool) result { return result{compared: true, isEqual: isEqual} } + +var ( + couldNotCompare = result{compared: false, isEqual: false} + comparedAndEqual = result{compared: true, isEqual: true} +) + +func deepCustomEqual(v1, v2 reflect.Value) result { + if !v1.IsValid() || !v2.IsValid() { + return couldNotCompare + } + + if v1.Type() != v2.Type() { + return couldNotCompare + } + + pointerOrStruct := v1.Type().Kind() == reflect.Ptr || v1.Type().Kind() == reflect.Struct + if pointerOrStruct && v1.CanInterface() && v1.Type().Implements(customEqualityType) { + if v2Ptr := getAddr(v2); v2Ptr.CanInterface() { + return successfulCompare(v1.Interface().(CustomEquality).Equals(v2Ptr.Interface())) + } + } + + switch v1.Kind() { + case reflect.Array: + for i := 0; i < v1.Len(); i++ { + partResult := deepCustomEqual(v1.Index(i), v2.Index(i)) + if !partResult.compared || !partResult.isEqual { + return partResult + } + } + return comparedAndEqual + case reflect.Slice: + if v1.IsNil() != v2.IsNil() { + return couldNotCompare + } + if v1.Len() != v2.Len() { + return couldNotCompare + } + for i := 0; i < v1.Len(); i++ { + partResult := deepCustomEqual(v1.Index(i), v2.Index(i)) + if !partResult.compared || !partResult.isEqual { + return partResult + } + } + return comparedAndEqual + case reflect.Interface, reflect.Pointer: + if v1.IsNil() || v2.IsNil() { + return couldNotCompare + } + return deepCustomEqual(v1.Elem(), v2.Elem()) + case reflect.Struct: + for i, n := 0, v1.NumField(); i < n; i++ { + partResult := deepCustomEqual(v1.Field(i), v2.Field(i)) + if !partResult.compared || !partResult.isEqual { + return partResult + } + } + return comparedAndEqual + case reflect.Map: + if v1.IsNil() != v2.IsNil() { + return couldNotCompare + } + if v1.Len() != v2.Len() { + return couldNotCompare + } + iter := v1.MapRange() + for iter.Next() { + val1 := iter.Value() + val2 := v2.MapIndex(iter.Key()) + if !val1.IsValid() || !val2.IsValid() { + return couldNotCompare + } + partResult := deepCustomEqual(val1, val2) + if !partResult.compared || !partResult.isEqual { + return partResult + } + } + return comparedAndEqual + default: + return couldNotCompare + } +} + +// getAddr grabs an address (if needed) to v. This function must be called with either a pointer or struct value. +func getAddr(v reflect.Value) reflect.Value { + if v.Type().Kind() == reflect.Ptr { // already pointer + return v + } + // otherwise it's a struct + if v.CanAddr() { + return v.Addr() + } + // if not addressable, we'll need to make a copy + newVal := reflect.New(v.Type()).Elem() + newVal.Set(v) + return newVal.Addr() +} diff --git a/internal/runtime/equality/equality_test.go b/internal/runtime/equality/equality_test.go new file mode 100644 index 0000000000..7b4be15ba1 --- /dev/null +++ b/internal/runtime/equality/equality_test.go @@ -0,0 +1,282 @@ +package equality + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeepEqual(t *testing.T) { + type args struct { + x any + y any + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "numbers equal", + args: args{x: 123, y: 123}, + want: true, + }, + { + name: "strings equal", + args: args{x: "123", y: "123"}, + want: true, + }, + { + name: "strings unequal", + args: args{x: "not 123", y: "123"}, + want: false, + }, + { + name: "structs equal", + args: args{x: justAStruct{123, 13.6}, y: justAStruct{123, 13.6}}, + want: true, + }, + { + name: "structs unequal", + args: args{x: justAStruct{123, 13.6}, y: justAStruct{123, 3.14}}, + want: false, + }, + { + name: "strings unequal", + args: args{x: "not 123", y: "123"}, + want: false, + }, + { + name: "strings slice equal", + args: args{x: []string{"123", "567"}, y: []string{"123", "567"}}, + want: true, + }, + { + name: "strings slice unequal", + args: args{x: []string{"123", "567"}, y: []string{"1234", "5678"}}, + want: false, + }, + { + name: "custom struct equal", + args: args{x: equalsTester{"same__"}, y: equalsTester{"length"}}, + want: true, // note: equalsTester defined equality as length of its string matching! + }, + { + name: "custom struct unequal", + args: args{x: equalsTester{"different"}, y: equalsTester{"lengths"}}, + want: false, + }, + { + name: "custom struct pointer equal", + args: args{x: &equalsTester{"same__"}, y: &equalsTester{"length"}}, + want: true, + }, + { + name: "custom struct pointer unequal", + args: args{x: &equalsTester{"different"}, y: &equalsTester{"lengths"}}, + want: false, + }, + { + name: "custom struct pointer unequal with nil", + args: args{x: &equalsTester{"different"}, y: nil}, + want: false, + }, + { + // Even though values are equal, we're comparing different types and this is not supported. This behaviour + // matches reflect.DeepEqual. + name: "custom struct and pointer cannot be equal", + args: args{x: &equalsTester{"same__"}, y: equalsTester{"length"}}, + want: false, + }, + { + name: "custom struct array equal", + args: args{x: [2]equalsTester{{"1"}, {"2"}}, y: [2]equalsTester{{"a"}, {"b"}}}, + want: true, + }, + { + name: "custom struct array unequal", + args: args{x: [2]equalsTester{{"1"}, {"2"}}, y: [2]equalsTester{{"different_length"}, {"b"}}}, + want: false, + }, + { + name: "custom pointer array equal", + args: args{x: [2]*equalsTester{{"1"}, {"2"}}, y: [2]*equalsTester{{"a"}, {"b"}}}, + want: true, + }, + { + name: "custom pointer array unequal", + args: args{x: [2]*equalsTester{{"1"}, {"2"}}, y: [2]*equalsTester{{"different_length"}, {"b"}}}, + want: false, + }, + { + name: "custom struct slice equal", + args: args{x: []equalsTester{{"1"}, {"2"}}, y: []equalsTester{{"a"}, {"b"}}}, + want: true, + }, + { + name: "custom struct slice unequal", + args: args{x: []equalsTester{{"1"}, {"2"}}, y: []equalsTester{{"different_length"}, {"b"}}}, + want: false, + }, + { + name: "custom pointer slice equal", + args: args{x: []*equalsTester{{"1"}, {"2"}}, y: []*equalsTester{{"a"}, {"b"}}}, + want: true, + }, + { + name: "custom pointer slice unequal", + args: args{x: []*equalsTester{{"1"}, {"2"}}, y: []*equalsTester{{"different_length"}, {"b"}}}, + want: false, + }, + { + name: "custom pointer slice unequal lengths", + args: args{x: []*equalsTester{{"1"}, {"2"}}, y: []*equalsTester{}}, + want: false, + }, + { + name: "custom pointer slice with nil", + args: args{x: []*equalsTester{{"1"}, {"2"}}, y: nil}, + want: false, + }, + { + name: "mixed fields structs are not supported", + args: args{ + x: mixedFieldsStruct{ + i: 123, + e1: equalsTester{"1234"}, + e2: equalsTester{"ab"}, + s: justAStruct{567, 3.14}, + }, + y: mixedFieldsStruct{ + i: 123, + e1: equalsTester{"abcd"}, + e2: equalsTester{"12"}, + s: justAStruct{567, 3.14}, + }, + }, + want: false, + }, + { + name: "all custom unexported fields struct not supported", + args: args{ + x: allCustomFieldsStructUnexported{ + e1: equalsTester{"1234"}, + e2: equalsTester{"ab"}, + }, + y: allCustomFieldsStructUnexported{ + e1: equalsTester{"abcd"}, + e2: equalsTester{"12"}, + }, + }, + want: false, + }, + { + name: "all custom exported fields struct equal", + args: args{ + x: allCustomFieldsStructExported{ + E1: equalsTester{"1234"}, + E2: equalsTester{"ab"}, + }, + y: allCustomFieldsStructExported{ + E1: equalsTester{"abcd"}, + E2: equalsTester{"12"}, + }, + }, + want: true, + }, + { + name: "all custom exported fields struct unequal", + args: args{ + x: allCustomFieldsStructExported{ + E1: equalsTester{"1234"}, + E2: equalsTester{"ab"}, + }, + y: allCustomFieldsStructExported{ + E1: equalsTester{"abcd"}, + E2: equalsTester{"this is too long"}, + }, + }, + want: false, + }, + { + name: "custom struct map equal", + args: args{ + x: map[string]equalsTester{"a": {"1"}, "b": {"12"}}, + y: map[string]equalsTester{"a": {"a"}, "b": {"ab"}}, + }, + want: true, + }, + { + name: "custom struct map unequal", + args: args{ + x: map[string]equalsTester{"a": {"1"}, "b": {"12"}}, + y: map[string]equalsTester{"a": {"a"}, "b": {"too long"}}, + }, + want: false, + }, + { + name: "custom struct map unequal different keys", + args: args{ + x: map[string]equalsTester{"a": {"1"}, "b": {"12"}}, + y: map[string]equalsTester{"a": {"a"}, "c": {"ab"}}, + }, + want: false, + }, + { + name: "custom struct map unequal different length", + args: args{ + x: map[string]equalsTester{"a": {"1"}, "b": {"12"}}, + y: map[string]equalsTester{"a": {"a"}, "b": {"ab"}, "c": {""}}, + }, + want: false, + }, + { + name: "custom struct map unequal nil", + args: args{ + x: map[string]equalsTester{"a": {"1"}, "b": {"12"}}, + y: nil, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, DeepEqual(tt.args.x, tt.args.y)) + assert.Equal(t, tt.want, DeepEqual(tt.args.y, tt.args.x)) + }) + } +} + +type justAStruct struct { + i int + f float64 +} + +type mixedFieldsStruct struct { + i int + e1 equalsTester + e2 equalsTester + s justAStruct +} + +type allCustomFieldsStructUnexported struct { + e1 equalsTester + e2 equalsTester +} + +type allCustomFieldsStructExported struct { + E1 equalsTester + E2 equalsTester +} + +type equalsTester struct { + s string +} + +func (t equalsTester) Equals(other any) bool { + if o, ok := other.(*equalsTester); ok { + // Special kind of equals - considered equal if lengths of strings are the same. + return len(t.s) == len(o.s) + } + return false +} diff --git a/internal/runtime/internal/controller/node_builtin_component.go b/internal/runtime/internal/controller/node_builtin_component.go index f1df4ff422..5241e2ddc2 100644 --- a/internal/runtime/internal/controller/node_builtin_component.go +++ b/internal/runtime/internal/controller/node_builtin_component.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/featuregate" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/runtime/logging" "github.com/grafana/alloy/internal/runtime/tracing" "github.com/grafana/alloy/syntax/ast" @@ -291,7 +292,7 @@ func (cn *BuiltinComponentNode) evaluate(scope *vm.Scope) error { return nil } - if reflect.DeepEqual(cn.args, argsCopyValue) { + if equality.DeepEqual(cn.args, argsCopyValue) { // Ignore components which haven't changed. This reduces the cost of // calling evaluate for components where evaluation is expensive (e.g., if // re-evaluating requires re-starting some internal logic). @@ -380,7 +381,7 @@ func (cn *BuiltinComponentNode) setExports(e component.Exports) { var changed bool cn.exportsMut.Lock() - if !reflect.DeepEqual(cn.exports, e) { + if !equality.DeepEqual(cn.exports, e) { changed = true cn.exports = e } diff --git a/internal/runtime/internal/controller/node_custom_component.go b/internal/runtime/internal/controller/node_custom_component.go index df98803857..938af8b14f 100644 --- a/internal/runtime/internal/controller/node_custom_component.go +++ b/internal/runtime/internal/controller/node_custom_component.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "path" - "reflect" "strings" "sync" "time" @@ -12,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/vm" ) @@ -264,7 +264,7 @@ func (cn *CustomComponentNode) setExports(e component.Exports) { var changed bool cn.exportsMut.Lock() - if !reflect.DeepEqual(cn.exports, e) { + if !equality.DeepEqual(cn.exports, e) { changed = true cn.exports = e } diff --git a/internal/runtime/internal/controller/node_service.go b/internal/runtime/internal/controller/node_service.go index 85619b2d34..899cb10c47 100644 --- a/internal/runtime/internal/controller/node_service.go +++ b/internal/runtime/internal/controller/node_service.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/service" "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/vm" @@ -104,7 +105,7 @@ func (sn *ServiceNode) Evaluate(scope *vm.Scope) error { // since services expect a non-pointer. argsCopyValue := reflect.ValueOf(argsPointer).Elem().Interface() - if reflect.DeepEqual(sn.args, argsCopyValue) { + if equality.DeepEqual(sn.args, argsCopyValue) { // Ignore arguments which haven't changed. This reduces the cost of calling // evaluate for services where evaluation is expensive (e.g., if // re-evaluating requires re-starting some internal logic). diff --git a/internal/runtime/internal/controller/value_cache.go b/internal/runtime/internal/controller/value_cache.go index 009c592d74..f1319db82b 100644 --- a/internal/runtime/internal/controller/value_cache.go +++ b/internal/runtime/internal/controller/value_cache.go @@ -2,10 +2,10 @@ package controller import ( "fmt" - "reflect" "sync" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/syntax/vm" ) @@ -96,7 +96,7 @@ func (vc *valueCache) CacheModuleExportValue(name string, value any) { v, found := vc.moduleExports[name] if !found { vc.moduleChangedIndex++ - } else if !reflect.DeepEqual(v, value) { + } else if !equality.DeepEqual(v, value) { vc.moduleChangedIndex++ } diff --git a/internal/runtime/internal/importsource/import_file.go b/internal/runtime/internal/importsource/import_file.go index e4691d9ed5..d0e10e02c6 100644 --- a/internal/runtime/internal/importsource/import_file.go +++ b/internal/runtime/internal/importsource/import_file.go @@ -7,14 +7,15 @@ import ( "io/fs" "os" "path/filepath" - "reflect" "strings" "sync" "time" "github.com/go-kit/log" + "github.com/grafana/alloy/internal/component" filedetector "github.com/grafana/alloy/internal/filedetector" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/syntax/vm" @@ -84,7 +85,7 @@ func (im *ImportFile) Evaluate(scope *vm.Scope) error { return fmt.Errorf("decoding configuration: %w", err) } - if reflect.DeepEqual(im.args, arguments) { + if equality.DeepEqual(im.args, arguments) { return nil } im.args = arguments diff --git a/internal/runtime/internal/importsource/import_git.go b/internal/runtime/internal/importsource/import_git.go index 02ee1ed675..29f3ef8b22 100644 --- a/internal/runtime/internal/importsource/import_git.go +++ b/internal/runtime/internal/importsource/import_git.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "path/filepath" - "reflect" "strings" "sync" "time" @@ -13,6 +12,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/vcs" "github.com/grafana/alloy/syntax" @@ -93,7 +93,7 @@ func (im *ImportGit) Evaluate(scope *vm.Scope) error { return fmt.Errorf("decoding configuration: %w", err) } - if reflect.DeepEqual(im.args, arguments) { + if equality.DeepEqual(im.args, arguments) { return nil } @@ -208,7 +208,7 @@ func (im *ImportGit) Update(args component.Arguments) (err error) { // Create or update the repo field. // Failure to update repository makes the module loader temporarily use cached contents on disk - if im.repo == nil || !reflect.DeepEqual(repoOpts, im.repoOpts) { + if im.repo == nil || !equality.DeepEqual(repoOpts, im.repoOpts) { r, err := vcs.NewGitRepo(context.Background(), im.repoPath, repoOpts) if err != nil { if errors.As(err, &vcs.UpdateFailedError{}) { diff --git a/internal/runtime/internal/importsource/import_http.go b/internal/runtime/internal/importsource/import_http.go index 815748f8f6..73f19eced0 100644 --- a/internal/runtime/internal/importsource/import_http.go +++ b/internal/runtime/internal/importsource/import_http.go @@ -5,12 +5,12 @@ import ( "fmt" "net/http" "path" - "reflect" "time" "github.com/grafana/alloy/internal/component" common_config "github.com/grafana/alloy/internal/component/common/config" remote_http "github.com/grafana/alloy/internal/component/remote/http" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/syntax/vm" ) @@ -84,7 +84,7 @@ func (im *ImportHTTP) Evaluate(scope *vm.Scope) error { im.arguments = arguments } - if reflect.DeepEqual(im.arguments, arguments) { + if equality.DeepEqual(im.arguments, arguments) { return nil } diff --git a/internal/runtime/internal/importsource/import_string.go b/internal/runtime/internal/importsource/import_string.go index a8a1249fc4..e1ec0644eb 100644 --- a/internal/runtime/internal/importsource/import_string.go +++ b/internal/runtime/internal/importsource/import_string.go @@ -3,9 +3,9 @@ package importsource import ( "context" "fmt" - "reflect" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/syntax/alloytypes" "github.com/grafana/alloy/syntax/vm" ) @@ -37,7 +37,7 @@ func (im *ImportString) Evaluate(scope *vm.Scope) error { return fmt.Errorf("decoding configuration: %w", err) } - if reflect.DeepEqual(im.arguments, arguments) { + if equality.DeepEqual(im.arguments, arguments) { return nil } im.arguments = arguments diff --git a/internal/runtime/internal/testcomponents/module/git/git.go b/internal/runtime/internal/testcomponents/module/git/git.go index b4cf2fb984..12100d2bfb 100644 --- a/internal/runtime/internal/testcomponents/module/git/git.go +++ b/internal/runtime/internal/testcomponents/module/git/git.go @@ -5,13 +5,14 @@ import ( "context" "errors" "path/filepath" - "reflect" "sync" "time" "github.com/go-kit/log" + "github.com/grafana/alloy/internal/component" "github.com/grafana/alloy/internal/featuregate" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/runtime/internal/testcomponents/module" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/vcs" @@ -199,7 +200,7 @@ func (c *Component) Update(args component.Arguments) (err error) { // Create or update the repo field. // Failure to update repository makes the module loader temporarily use cached contents on disk - if c.repo == nil || !reflect.DeepEqual(repoOpts, c.repoOpts) { + if c.repo == nil || !equality.DeepEqual(repoOpts, c.repoOpts) { r, err := vcs.NewGitRepo(context.Background(), repoPath, repoOpts) if err != nil { if errors.As(err, &vcs.UpdateFailedError{}) { diff --git a/internal/runtime/internal/testcomponents/module/module.go b/internal/runtime/internal/testcomponents/module/module.go index f7a420b2c4..f9d3c81d1f 100644 --- a/internal/runtime/internal/testcomponents/module/module.go +++ b/internal/runtime/internal/testcomponents/module/module.go @@ -3,11 +3,11 @@ package module import ( "context" "fmt" - "reflect" "sync" "time" "github.com/grafana/alloy/internal/component" + "github.com/grafana/alloy/internal/runtime/equality" "github.com/grafana/alloy/internal/runtime/logging/level" ) @@ -44,7 +44,7 @@ func NewModuleComponent(o component.Options) (*ModuleComponent, error) { // It will set the component health in addition to return the error so that the consumer can rely on either or both. // If the content is the same as the last time it was successfully loaded, it will not be reloaded. func (c *ModuleComponent) LoadAlloySource(args map[string]any, contentValue string) error { - if reflect.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() { + if equality.DeepEqual(args, c.getLatestArgs()) && contentValue == c.getLatestContent() { return nil } diff --git a/internal/service/ui/ui.go b/internal/service/ui/ui.go index ea69448a82..2832f642bf 100644 --- a/internal/service/ui/ui.go +++ b/internal/service/ui/ui.go @@ -8,6 +8,7 @@ import ( "path" "github.com/gorilla/mux" + "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/service" http_service "github.com/grafana/alloy/internal/service/http" diff --git a/internal/static/traces/promsdprocessor/consumer/consumer.go b/internal/static/traces/promsdprocessor/consumer/consumer.go index c96b79e024..3bddc9862b 100644 --- a/internal/static/traces/promsdprocessor/consumer/consumer.go +++ b/internal/static/traces/promsdprocessor/consumer/consumer.go @@ -10,13 +10,14 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/grafana/alloy/internal/component/discovery" "github.com/prometheus/common/model" "go.opentelemetry.io/collector/client" otelconsumer "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" semconv "go.opentelemetry.io/collector/semconv/v1.5.0" + + "github.com/grafana/alloy/internal/component/discovery" ) const ( @@ -27,7 +28,7 @@ const ( // OperationTypeUpsert does both of above OperationTypeUpsert = "upsert" - //TODO: It'd be cleaner to get these from the otel semver package? + // TODO: It'd be cleaner to get these from the otel semver package? // Not all are in semver though. E.g. "k8s.pod.ip" is internal inside the k8sattributesprocessor. PodAssociationIPLabel = "ip" PodAssociationOTelIPLabel = "net.host.ip" @@ -160,20 +161,21 @@ func (c *Consumer) processAttributes(ctx context.Context, attrs pcommon.Map) { return } - for _, label := range labels.Labels() { + labels.ForEachLabel(func(label string, value string) bool { switch c.opts.OperationType { case OperationTypeUpsert: - attrs.PutStr(label.Name, label.Value) + attrs.PutStr(label, value) case OperationTypeInsert: - if _, ok := attrs.Get(label.Name); !ok { - attrs.PutStr(label.Name, label.Value) + if _, ok := attrs.Get(label); !ok { + attrs.PutStr(label, value) } case OperationTypeUpdate: - if toVal, ok := attrs.Get(label.Name); ok { - toVal.SetStr(label.Value) + if toVal, ok := attrs.Get(label); ok { + toVal.SetStr(value) } } - } + return true + }) } func (c *Consumer) getPodIP(ctx context.Context, attrs pcommon.Map) string { @@ -230,9 +232,9 @@ func (c *Consumer) getConnectionIP(ctx context.Context) string { } func GetHostFromLabels(labels discovery.Target) (string, error) { - address, ok := labels[model.AddressLabel] + address, ok := labels.Get(model.AddressLabel) if !ok { - return "", fmt.Errorf("unable to find address in labels %q", labels.Labels()) + return "", fmt.Errorf("unable to find address in labels %q", labels) } host := address @@ -247,12 +249,6 @@ func GetHostFromLabels(labels discovery.Target) (string, error) { return host, nil } -func NewTargetsWithNonInternalLabels(labels discovery.Target) discovery.Target { - res := make(discovery.Target) - for k, v := range labels { - if !strings.HasPrefix(k, "__") { - res[k] = v - } - } - return res +func NewTargetsWithNonInternalLabels(target discovery.Target) discovery.Target { + return discovery.NewTargetFromLabelSet(target.NonReservedLabelSet()) } diff --git a/internal/static/traces/promsdprocessor/consumer/consumer_test.go b/internal/static/traces/promsdprocessor/consumer/consumer_test.go index 758b4095e5..4d9ad80f92 100644 --- a/internal/static/traces/promsdprocessor/consumer/consumer_test.go +++ b/internal/static/traces/promsdprocessor/consumer/consumer_test.go @@ -5,14 +5,15 @@ import ( "net" "testing" - "github.com/grafana/alloy/internal/component/discovery" - "github.com/grafana/alloy/internal/util" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/client" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/pcommon" semconv "go.opentelemetry.io/collector/semconv/v1.5.0" "gotest.tools/assert" + + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/internal/util" ) func TestOperationType(t *testing.T) { @@ -84,9 +85,9 @@ func TestOperationType(t *testing.T) { } consumerOpts := Options{ HostLabels: map[string]discovery.Target{ - attrIP: { + attrIP: discovery.NewTargetFromMap(map[string]string{ attrKey: tc.newValue, - }, + }), }, OperationType: tc.operationType, PodAssociations: podAssociations, diff --git a/internal/static/traces/promsdprocessor/prom_sd_processor.go b/internal/static/traces/promsdprocessor/prom_sd_processor.go index b916705581..92ad2a218d 100644 --- a/internal/static/traces/promsdprocessor/prom_sd_processor.go +++ b/internal/static/traces/promsdprocessor/prom_sd_processor.go @@ -178,11 +178,7 @@ func (p *promServiceDiscoProcessor) syncTargets(jobName string, group *targetgro continue } - var labels = make(discovery.Target) - for k, v := range processedLabels.Map() { - labels[k] = v - } - + var labels = discovery.NewTargetFromModelLabels(processedLabels) host, err := promsdconsumer.GetHostFromLabels(labels) if err != nil { level.Warn(p.logger).Log("msg", "ignoring target, unable to find address", "err", err) diff --git a/internal/static/traces/promsdprocessor/prom_sd_processor_test.go b/internal/static/traces/promsdprocessor/prom_sd_processor_test.go index 3641168149..3113a42306 100644 --- a/internal/static/traces/promsdprocessor/prom_sd_processor_test.go +++ b/internal/static/traces/promsdprocessor/prom_sd_processor_test.go @@ -4,11 +4,12 @@ import ( "testing" "github.com/go-kit/log" - "github.com/grafana/alloy/internal/component/discovery" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/assert" + + "github.com/grafana/alloy/internal/component/discovery" ) func TestSyncGroups(t *testing.T) { @@ -38,7 +39,7 @@ func TestSyncGroups(t *testing.T) { }, }, expected: map[string]discovery.Target{ - "127.0.0.1": {}, + "127.0.0.1": discovery.EmptyTarget, }, }, { @@ -54,9 +55,9 @@ func TestSyncGroups(t *testing.T) { }, }, expected: map[string]discovery.Target{ - "127.0.0.1": { + "127.0.0.1": discovery.NewTargetFromMap(map[string]string{ "label": "val", - }, + }), }, }, { @@ -72,9 +73,9 @@ func TestSyncGroups(t *testing.T) { }, }, expected: map[string]discovery.Target{ - "127.0.0.1": { + "127.0.0.1": discovery.NewTargetFromMap(map[string]string{ "label": "val", - }, + }), }, }, { @@ -90,7 +91,7 @@ func TestSyncGroups(t *testing.T) { }, }, expected: map[string]discovery.Target{ - "127.0.0.1": {}, + "127.0.0.1": discovery.EmptyTarget, }, }, } diff --git a/internal/util/testtarget/test_target.go b/internal/util/testtarget/test_target.go index 0dab941635..12afbff640 100644 --- a/internal/util/testtarget/test_target.go +++ b/internal/util/testtarget/test_target.go @@ -45,9 +45,9 @@ func (t *TestTarget) AddHistogram(opts prometheus.HistogramOpts) prometheus.Hist } func (t *TestTarget) Target() discovery.Target { - return discovery.Target{ + return discovery.NewTargetFromMap(map[string]string{ "__address__": t.server.Listener.Addr().String(), - } + }) } func (t *TestTarget) Registry() *prometheus.Registry { diff --git a/syntax/encoding/alloyjson/alloyjson.go b/syntax/encoding/alloyjson/alloyjson.go index 28ab68506b..054aa22c6b 100644 --- a/syntax/encoding/alloyjson/alloyjson.go +++ b/syntax/encoding/alloyjson/alloyjson.go @@ -281,34 +281,46 @@ func buildJSONValue(v value.Value) jsonValue { return jsonValue{Type: "array", Value: elements} case value.TypeObject: - keys := v.Keys() - - // If v isn't an ordered object (i.e., a go map), sort the keys so they - // have a deterministic print order. - if !v.OrderedKeys() { - sort.Strings(keys) - } - - fields := []jsonObjectField{} - - for i := 0; i < len(keys); i++ { - field, _ := v.Key(keys[i]) - - fields = append(fields, jsonObjectField{ - Key: keys[i], - Value: buildJSONValue(field), - }) - } - - return jsonValue{Type: "object", Value: fields} + return tokenizeObject(v) case value.TypeFunction: return jsonValue{Type: "function", Value: v.Describe()} case value.TypeCapsule: + if v.Implements(reflect.TypeFor[value.ConvertibleIntoCapsule]()) { + // Check if this capsule can be converted into Alloy object for more detailed description: + newVal := make(map[string]value.Value) + if err := v.ReflectAddr().Interface().(value.ConvertibleIntoCapsule).ConvertInto(&newVal); err == nil { + return tokenizeObject(value.Encode(newVal)) + } + } + // Otherwise, describe the value return jsonValue{Type: "capsule", Value: v.Describe()} default: panic(fmt.Sprintf("syntax/encoding/alloyjson: unrecognized value type %q", v.Type())) } } + +func tokenizeObject(v value.Value) jsonValue { + keys := v.Keys() + + // If v isn't an ordered object (i.e., a go map), sort the keys so they + // have a deterministic print order. + if !v.OrderedKeys() { + sort.Strings(keys) + } + + fields := []jsonObjectField{} + + for i := 0; i < len(keys); i++ { + field, _ := v.Key(keys[i]) + + fields = append(fields, jsonObjectField{ + Key: keys[i], + Value: buildJSONValue(field), + }) + } + + return jsonValue{Type: "object", Value: fields} +} diff --git a/syntax/encoding/alloyjson/alloyjson_test.go b/syntax/encoding/alloyjson/alloyjson_test.go index 2e9bf65269..36aa1786e2 100644 --- a/syntax/encoding/alloyjson/alloyjson_test.go +++ b/syntax/encoding/alloyjson/alloyjson_test.go @@ -1,12 +1,14 @@ package alloyjson_test import ( + "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/grafana/alloy/syntax" "github.com/grafana/alloy/syntax/alloytypes" "github.com/grafana/alloy/syntax/encoding/alloyjson" - "github.com/stretchr/testify/require" ) func TestValues(t *testing.T) { @@ -89,6 +91,20 @@ func TestValues(t *testing.T) { input: alloytypes.Secret("foo"), expectJSON: `{ "type": "capsule", "value": "(secret)" }`, }, + { + name: "mappable capsule", + input: capsuleConvertibleToObject{ + name: "Scrooge McDuck", + address: "Duckburg, Killmotor Hill", + }, + expectJSON: `{ + "type": "object", + "value": [ + { "key": "address", "value": { "type": "string", "value": "Duckburg, Killmotor Hill" }}, + { "key": "name", "value": { "type": "string", "value": "Scrooge McDuck" }} + ] + }`, + }, { // nil arrays and objects must always be [] instead of null as that's // what the API definition says they should be. @@ -361,3 +377,28 @@ func TestRawMap_Capsule(t *testing.T) { require.NoError(t, err) require.JSONEq(t, expect, string(bb)) } + +type capsuleConvertibleToObject struct { + name string + address string +} + +func (c capsuleConvertibleToObject) ConvertInto(dst interface{}) error { + switch dst := dst.(type) { + case *map[string]syntax.Value: + result := map[string]syntax.Value{ + "name": syntax.ValueFromString(c.name), + "address": syntax.ValueFromString(c.address), + } + *dst = result + return nil + } + return fmt.Errorf("capsuleConvertibleToObject: conversion to '%T' is not supported", dst) +} + +func (c capsuleConvertibleToObject) AlloyCapsule() {} + +var ( + _ syntax.Capsule = capsuleConvertibleToObject{} + _ syntax.ConvertibleIntoCapsule = capsuleConvertibleToObject{} +) diff --git a/syntax/internal/value/value.go b/syntax/internal/value/value.go index 829449370e..69599c0a29 100644 --- a/syntax/internal/value/value.go +++ b/syntax/internal/value/value.go @@ -199,17 +199,10 @@ func (v Value) Text() string { panic("syntax/value: Text called on non-string type") } - // Attempt to get an address to v.rv for interface checking. - // - // The normal v.rv value is used for other checks. - addrRV := v.rv - if addrRV.CanAddr() { - addrRV = addrRV.Addr() - } switch { - case addrRV.Type().Implements(goTextMarshaler): + case v.Implements(goTextMarshaler): // TODO(rfratto): what should we do if this fails? - text, _ := addrRV.Interface().(encoding.TextMarshaler).MarshalText() + text, _ := v.ReflectAddr().Interface().(encoding.TextMarshaler).MarshalText() return string(text) case v.rv.Type() == goDuration: @@ -221,6 +214,10 @@ func (v Value) Text() string { } } +func (v Value) IsString() bool { + return v.Type() == TypeString +} + // Len returns the length of v. Panics if v is not an array or object. func (v Value) Len() int { switch v.ty { @@ -258,9 +255,23 @@ func (v Value) Interface() interface{} { return v.rv.Interface() } +func (v Value) Implements(t reflect.Type) bool { + return v.ReflectAddr().Type().Implements(t) +} + // Reflect returns the raw reflection value backing v. func (v Value) Reflect() reflect.Value { return v.rv } +// ReflectAddr is like Reflect, but attempts to get an address of the raw reflection value where possible. +func (v Value) ReflectAddr() reflect.Value { + // Attempt to get an address to v.rv + addrRV := v.rv + if addrRV.CanAddr() { + addrRV = addrRV.Addr() + } + return addrRV +} + // makeValue converts a reflect value into a Value, dereferencing any pointers or // interface{} values. func makeValue(v reflect.Value) Value { diff --git a/syntax/token/builder/builder_test.go b/syntax/token/builder/builder_test.go index 47b095dd61..35e351ee13 100644 --- a/syntax/token/builder/builder_test.go +++ b/syntax/token/builder/builder_test.go @@ -6,11 +6,13 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/syntax" "github.com/grafana/alloy/syntax/parser" "github.com/grafana/alloy/syntax/printer" "github.com/grafana/alloy/syntax/token" "github.com/grafana/alloy/syntax/token/builder" - "github.com/stretchr/testify/require" ) func TestBuilder_File(t *testing.T) { @@ -43,6 +45,31 @@ func TestBuilder_File(t *testing.T) { require.Equal(t, expect, string(f.Bytes())) } +type capsuleConvertibleToObject struct { + name string + address string +} + +func (c capsuleConvertibleToObject) ConvertInto(dst interface{}) error { + switch dst := dst.(type) { + case *map[string]syntax.Value: + result := map[string]syntax.Value{ + "name": syntax.ValueFromString(c.name), + "address": syntax.ValueFromString(c.address), + } + *dst = result + return nil + } + return fmt.Errorf("capsuleConvertibleToObject: conversion to '%T' is not supported", dst) +} + +func (c capsuleConvertibleToObject) AlloyCapsule() {} + +var ( + _ syntax.Capsule = capsuleConvertibleToObject{} + _ syntax.ConvertibleIntoCapsule = capsuleConvertibleToObject{} +) + func TestBuilder_GoEncode(t *testing.T) { f := builder.NewFile() @@ -55,7 +82,17 @@ func TestBuilder_GoEncode(t *testing.T) { f.Body().SetAttributeValue("bool", true) f.Body().SetAttributeValue("list", []int{0, 1, 2}) f.Body().SetAttributeValue("func", func(int, int) int { return 0 }) + f.Body().AppendTokens([]builder.Token{{token.LITERAL, "\n"}}) + f.Body().SetAttributeValue("capsule", make(chan int)) + f.Body().SetAttributeValue("mappable_capsule", capsuleConvertibleToObject{ + name: "Bert", + address: "11a Sesame St", + }) + f.Body().SetAttributeValue("mappable_capsule_ptr", &capsuleConvertibleToObject{ + name: "Ernie", + address: "11b Sesame St", + }) f.Body().AppendTokens([]builder.Token{{token.LITERAL, "\n"}}) f.Body().SetAttributeValue("map", map[string]interface{}{"foo": "bar"}) @@ -73,12 +110,21 @@ func TestBuilder_GoEncode(t *testing.T) { // Hello, world! null_value = null - num = 15 - string = "Hello, world!" - bool = true - list = [0, 1, 2] - func = function - capsule = capsule("chan int") + num = 15 + string = "Hello, world!" + bool = true + list = [0, 1, 2] + func = function + + capsule = capsule("chan int") + mappable_capsule = { + address = "11a Sesame St", + name = "Bert", + } + mappable_capsule_ptr = { + address = "11b Sesame St", + name = "Ernie", + } map = { foo = "bar", diff --git a/syntax/token/builder/value_tokens.go b/syntax/token/builder/value_tokens.go index e07b187a39..ca872246d2 100644 --- a/syntax/token/builder/value_tokens.go +++ b/syntax/token/builder/value_tokens.go @@ -2,6 +2,7 @@ package builder import ( "fmt" + "reflect" "sort" "github.com/grafana/alloy/syntax/internal/value" @@ -57,35 +58,25 @@ func valueTokens(v value.Value) []Token { toks = append(toks, Token{token.RBRACK, ""}) case value.TypeObject: - toks = append(toks, Token{token.LCURLY, ""}, Token{token.LITERAL, "\n"}) - - keys := v.Keys() - - // If v isn't an ordered object (i.e., a go map), sort the keys so they - // have a deterministic print order. - if !v.OrderedKeys() { - sort.Strings(keys) - } - - for i := 0; i < len(keys); i++ { - if scanner.IsValidIdentifier(keys[i]) { - toks = append(toks, Token{token.IDENT, keys[i]}) - } else { - toks = append(toks, Token{token.STRING, fmt.Sprintf("%q", keys[i])}) - } - - field, _ := v.Key(keys[i]) - toks = append(toks, Token{token.ASSIGN, ""}) - toks = append(toks, valueTokens(field)...) - toks = append(toks, Token{token.COMMA, ""}, Token{token.LITERAL, "\n"}) - } - toks = append(toks, Token{token.RCURLY, ""}) + toks = objectTokens(v) case value.TypeFunction: toks = append(toks, Token{token.LITERAL, v.Describe()}) case value.TypeCapsule: - toks = append(toks, Token{token.LITERAL, v.Describe()}) + done := false + if v.Implements(reflect.TypeFor[value.ConvertibleIntoCapsule]()) { + // Check if this capsule can be converted into Alloy object for more detailed description: + newVal := make(map[string]value.Value) + if err := v.ReflectAddr().Interface().(value.ConvertibleIntoCapsule).ConvertInto(&newVal); err == nil { + toks = tokenEncode(newVal) + done = true + } + } + if !done { + // Default to Describe() for capsules that don't support other representation. + toks = append(toks, Token{token.LITERAL, v.Describe()}) + } default: panic(fmt.Sprintf("syntax/token/builder: unrecognized value type %q", v.Type())) @@ -93,3 +84,30 @@ func valueTokens(v value.Value) []Token { return toks } + +func objectTokens(v value.Value) []Token { + toks := []Token{{token.LCURLY, ""}, {token.LITERAL, "\n"}} + + keys := v.Keys() + + // If v isn't an ordered object (i.e. it is a go map), sort the keys so they + // have a deterministic print order. + if !v.OrderedKeys() { + sort.Strings(keys) + } + + for i := 0; i < len(keys); i++ { + if scanner.IsValidIdentifier(keys[i]) { + toks = append(toks, Token{token.IDENT, keys[i]}) + } else { + toks = append(toks, Token{token.STRING, fmt.Sprintf("%q", keys[i])}) + } + + field, _ := v.Key(keys[i]) + toks = append(toks, Token{token.ASSIGN, ""}) + toks = append(toks, valueTokens(field)...) + toks = append(toks, Token{token.COMMA, ""}, Token{token.LITERAL, "\n"}) + } + toks = append(toks, Token{token.RCURLY, ""}) + return toks +} diff --git a/syntax/types.go b/syntax/types.go index 41afef0d7b..8d39c6119f 100644 --- a/syntax/types.go +++ b/syntax/types.go @@ -95,3 +95,9 @@ type ConvertibleIntoCapsule interface { // available. Other errors are treated as an Alloy decoding error. ConvertInto(dst interface{}) error } + +// Value represents an Alloy value. See the value.Value for more details. +type Value = value.Value + +// ValueFromString creates a new Value from a given string. +var ValueFromString = value.String From dd72bee6597f5a792af291b05ea1433215d8b9fb Mon Sep 17 00:00:00 2001 From: Marc Sanmiquel Date: Thu, 13 Feb 2025 17:59:50 +0100 Subject: [PATCH 05/14] fix(pyroscope.scrape): windows tests path (#2719) * fix(pyroscope.scrape): windows tests path --- internal/component/pyroscope/scrape/target.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/component/pyroscope/scrape/target.go b/internal/component/pyroscope/scrape/target.go index e9ecae1a16..1478037c16 100644 --- a/internal/component/pyroscope/scrape/target.go +++ b/internal/component/pyroscope/scrape/target.go @@ -19,7 +19,6 @@ import ( "hash/fnv" "net" "net/url" - "path/filepath" "slices" "strconv" "strings" @@ -119,9 +118,8 @@ func urlFromTarget(lbls labels.Labels, params url.Values) string { return (&url.URL{ Scheme: lbls.Get(model.SchemeLabel), Host: lbls.Get(model.AddressLabel), - Path: filepath.Join(lbls.Get(ProfilePathPrefix), lbls.Get(ProfilePath)), // faster than URL.JoinPath RawQuery: newParams.Encode(), - }).String() + }).JoinPath(lbls.Get(ProfilePathPrefix), lbls.Get(ProfilePath)).String() } func (t *Target) String() string { From 35285403abe40587b4fa64f0ce02a77491b0a2bd Mon Sep 17 00:00:00 2001 From: mattdurham Date: Thu, 13 Feb 2025 13:35:23 -0500 Subject: [PATCH 06/14] Add walqueue sharding (#2665) * Add parralelism block. * fix attr * Update to include context cancelled logic. * Update CHANGELOG.md Co-authored-by: Piotr <17101802+thampiotr@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Piotr <17101802+thampiotr@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Piotr <17101802+thampiotr@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Piotr <17101802+thampiotr@users.noreply.github.com> * Update to include better naming from walqueue. * Reword verbiage. * Fix naming in test. * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * pr feedbackh * Merge suggestions and committed. * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Update docs/sources/reference/components/prometheus/prometheus.write.queue.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --------- Co-authored-by: Piotr <17101802+thampiotr@users.noreply.github.com> Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 4 + .../prometheus/prometheus.write.queue.md | 49 +++++++++--- go.mod | 4 +- go.sum | 8 +- .../component/prometheus/write/queue/types.go | 52 +++++++++++- .../prometheus/write/queue/types_test.go | 79 ++++++++++++++++++- 6 files changed, 180 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2713a5e720..afdaf0444d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ internal API changes are not present. Main (unreleased) ----------------- +### Breaking changes + +- (_Experimental_) In `prometheus.write.queue` changed `parallelism` from attribute to a block to allow for dynamic scaling. (@mattdurham) + ### Features - (_Experimental_) Add a `stage.windowsevent` block in the `loki.process` component. This aims to replace the existing `stage.eventlogmessage`. (@wildum) diff --git a/docs/sources/reference/components/prometheus/prometheus.write.queue.md b/docs/sources/reference/components/prometheus/prometheus.write.queue.md index fe15e2bce4..746428c50e 100644 --- a/docs/sources/reference/components/prometheus/prometheus.write.queue.md +++ b/docs/sources/reference/components/prometheus/prometheus.write.queue.md @@ -48,13 +48,13 @@ The following arguments are supported: The following blocks are supported inside the definition of `prometheus.write.queue`: - Hierarchy | Block | Description | Required ------------------------|------------------|----------------------------------------------------------|---------- - persistence | [persistence][] | Configuration for persistence | no - endpoint | [endpoint][] | Location to send metrics to. | no - endpoint > basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no - endpoint > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no - + Hierarchy | Block | Description | Required +-----------------------|-----------------|-----------------------------------------------------------|---------- + persistence | [persistence][] | Configuration for persistence | no + endpoint | [endpoint][] | Location to send metrics to. | no + endpoint > basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no + endpoint > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no + endpoint > parallelism | [parallelism][] | Configure parallelism for the endpoint. | no The `>` symbol indicates deeper levels of nesting. For example, `endpoint > basic_auth` refers to a `basic_auth` block defined inside an @@ -64,7 +64,7 @@ basic_auth` refers to a `basic_auth` block defined inside an [basic_auth]: #basic_auth-block [persistence]: #persistence-block [tls_config]: #tls_config-block - +[parallelism]: #parallelism ### persistence block @@ -96,7 +96,6 @@ The following arguments are supported: `max_retry_attempts` | `uint` | Maximum number of retries before dropping the batch. | `0` | no `batch_count` | `uint` | How many series to queue in each queue. | `1000` | no `flush_interval` | `duration` | How long to wait until sending if `batch_count` is not trigger. | `1s` | no - `parallelism` | `uint` | How many parallel batches to write. | 10 | no `external_labels` | `map(string)` | Labels to add to metrics sent over the network. | | no `enable_round_robin` | `bool` | Use round robin load balancing when there are multiple IPs for a given endpoint. | `false` | no @@ -117,6 +116,38 @@ Name | Type | Description `insecure_skip_verify` | `bool` | Disables validation of the server certificate. | | no `key_pem` | `secret` | Key PEM-encoded text for client authentication. | | no + +### parallelism block + +| Name | Type | Description | Default | Required | +|----------------------------------|------------|------------------------------------------------------------------------------------------------------------------------------------|---------|----------| +| `drift_scale_up` | `duration` | The maximum amount of time between the timestamps of incoming signals and outgoing signals before increasing desired connections. | `60` | no | +| `drift_scale_down` | `duration` | The minimum amount of time between the timestamps of incoming signals and outgoing signals before descreasing desired connections. | `30` | no | +| `max_connections` | `uint` | The maximum number of desired connections. | `50` | no | +| `min_connections` | `uint` | The minimum number of desired connections. | `2` | no | +| `network_flush_interval` | `duration` | The length of time that network successes and failures are kept for determining desired connections. | `1m` | no | +| `desired_connections_lookback` | `duration` | The length of time that previous desired connections are kept for determining desired connections. | `5m` | no | +| `desired_check_interval` | `duration` | The length of time between checking for desired connections. | `5s` | no | +| `allowed_network_error_fraction` | `float` | The allowed error rate before scaling down. For example `0.50` allows 50% error rate. | `0.50` | no | + +Parallelism determines when to scale up or down the number of desired connections. + +The drift between the incoming and outgoing timestamps determines whether to increase or decrease the desired connections. +The value stays the same if the drift is between `drift_scale_up_seconds` and `drift_scale_down_seconds`. + +Network successes and failures are recorded and kept in memory. +This data helps determine the nature of the drift. +For example, if the drift is increasing and the network failures are increasing, the desired connections should not increase because that would increase the load on the endpoint. + +The `desired_check_interval` prevents connection flapping. +Each time a desired connection is calculated, the connection is added to a lookback buffer. +Before increasing or decreasing the desired connections, `prometheus.write.queue` chooses the highest value in the lookback buffer. +For example, for the past 5 minutes, the desired connections have been: [2,1,1]. +The check determines that the desired connections are 1, and the number of desired connections will not change because the value `2` is still in the lookback buffer. +On the next check, the desired connections are [1,1,1]. +Since the `2` value has expired, the desired connections will change to 1. +In general, the system is fast to increase and slow to decrease the desired connections. + ## Exported fields The following fields are exported and can be referenced by other components: diff --git a/go.mod b/go.mod index db4057a6a6..51d9f7a77c 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0 github.com/grafana/vmware_exporter v0.0.5-beta - github.com/grafana/walqueue v0.0.0-20250113171943-e5fe545d1408 + github.com/grafana/walqueue v0.0.0-20250211154548-6435b3242458 github.com/hashicorp/consul/api v1.30.0 github.com/hashicorp/go-discover v0.0.0-20230724184603-e89ebd1b2f65 github.com/hashicorp/go-multierror v1.1.1 @@ -882,8 +882,10 @@ require ( github.com/containers/common v0.61.0 // indirect github.com/deneonet/benc v1.1.2 // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect + github.com/panjf2000/ants/v2 v2.11.0 // indirect github.com/pires/go-proxyproto v0.7.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + golang.design/x/chann v0.1.2 // indirect ) // NOTE: replace directives below must always be *temporary*. diff --git a/go.sum b/go.sum index d4b08f7c3f..2df083018c 100644 --- a/go.sum +++ b/go.sum @@ -1922,8 +1922,8 @@ github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0 h1:bjh0PVYSVVFxzINqPF github.com/grafana/tail v0.0.0-20230510142333-77b18831edf0/go.mod h1:7t5XR+2IA8P2qggOAHTj/GCZfoLBle3OvNSYh1VkRBU= github.com/grafana/vmware_exporter v0.0.5-beta h1:2JCqzIWJzns8FN78wPsueC9rT3e3kZo2OUoL5kGMjdM= github.com/grafana/vmware_exporter v0.0.5-beta/go.mod h1:1CecUZII0zVsVcHtNfNeTTcxK7EksqAsAn/TCCB0Mh4= -github.com/grafana/walqueue v0.0.0-20250113171943-e5fe545d1408 h1:TGoFWafEVwzy4Wg0L2PPQonwTKYaqa8kMpiAxiEvXW8= -github.com/grafana/walqueue v0.0.0-20250113171943-e5fe545d1408/go.mod h1:Hisxv1n+PxFQEkayynKy+B4AiJiJVRKHKT/8ng6jgOM= +github.com/grafana/walqueue v0.0.0-20250211154548-6435b3242458 h1:KUa/teNk/VYBDSSjsgdLI37gnFXTz1+h513CmNWJeU4= +github.com/grafana/walqueue v0.0.0-20250211154548-6435b3242458/go.mod h1:rnU7r397nvQCTyVbcODlD3P6DIbQlidxPDweV+4ab2M= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grobie/gomemcache v0.0.0-20230213081705-239240bbc445 h1:FlKQKUYPZ5yDCN248M3R7x8yu2E3yEZ0H7aLomE4EoE= github.com/grobie/gomemcache v0.0.0-20230213081705-239240bbc445/go.mod h1:L69/dBlPQlWkcnU76WgcppK5e4rrxzQdi6LhLnK/ytA= @@ -2810,6 +2810,8 @@ github.com/ovh/go-ovh v1.6.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c h1:vwpFWvAO8DeIZfFeqASzZfsxuWPno9ncAebBEP0N3uE= github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otzZQXgoO96RTzDB/Hycg0qZcXZsWJGJRSXbmEIJ+4M= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/panjf2000/ants/v2 v2.11.0 h1:sHrqEwTBQTQ2w6PMvbMfvBtVUuhsaYPzUmAYDLYmJPg= +github.com/panjf2000/ants/v2 v2.11.0/go.mod h1:V9HhTupTWxcaRmIglJvGwvzqXUTnIZW9uO6q4hAfApw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -3564,6 +3566,8 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org/netipx v0.0.0-20230125063823-8449b0a6169f h1:ketMxHg+vWm3yccyYiq+uK8D3fRmna2Fcj+awpQp84s= go4.org/netipx v0.0.0-20230125063823-8449b0a6169f/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc= +golang.design/x/chann v0.1.2 h1:eHF9wjuQnpp+j4ryWhyxC/pFuYzbvMAkudA/I5ALovY= +golang.design/x/chann v0.1.2/go.mod h1:Rh5KhCAp+0qu9+FfKPymHpu8onmjl89sFwMeiw3SK14= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/internal/component/prometheus/write/queue/types.go b/internal/component/prometheus/write/queue/types.go index e07d9e6b47..020011a8c4 100644 --- a/internal/component/prometheus/write/queue/types.go +++ b/internal/component/prometheus/write/queue/types.go @@ -50,7 +50,16 @@ func defaultEndpointConfig() EndpointConfig { MaxRetryAttempts: 0, BatchCount: 1_000, FlushInterval: 1 * time.Second, - Parallelism: 4, + Parallelism: ParallelismConfig{ + DriftScaleUp: 60 * time.Second, + DriftScaleDown: 30 * time.Second, + MaxConnections: 50, + MinConnections: 2, + NetworkFlushInterval: 1 * time.Minute, + DesiredConnectionsLookback: 5 * time.Minute, + DesiredCheckInterval: 5 * time.Second, + AllowedNetworkErrorFraction: 0.50, + }, } } @@ -66,6 +75,23 @@ func (r *Arguments) Validate() error { if conn.FlushInterval < 1*time.Second { return fmt.Errorf("flush_interval must be greater or equal to 1s, the internal timers resolution is 1s") } + if conn.Parallelism.MaxConnections < conn.Parallelism.MinConnections { + return fmt.Errorf("max_connections less than min_connections") + } + if conn.Parallelism.MinConnections == 0 { + return fmt.Errorf("min_connections must be greater than 0") + } + if conn.Parallelism.DriftScaleUp <= conn.Parallelism.DriftScaleDown { + return fmt.Errorf("drift_scale_up_seconds less than or equal drift_scale_down_seconds") + } + // Any lower than 1 second and you spend a fair amount of time churning on the draining and + // refilling the write buffers. + if conn.Parallelism.DesiredCheckInterval < 1*time.Second { + return fmt.Errorf("desired_check_interval must be greater than or equal to 1 second") + } + if conn.Parallelism.AllowedNetworkErrorFraction < 0 || conn.Parallelism.AllowedNetworkErrorFraction > 1 { + return fmt.Errorf("allowed_network_error_percent must be between 0.00 and 1.00") + } } return nil @@ -87,7 +113,7 @@ type EndpointConfig struct { // How long to wait before sending regardless of batch count. FlushInterval time.Duration `alloy:"flush_interval,attr,optional"` // How many concurrent queues to have. - Parallelism uint `alloy:"parallelism,attr,optional"` + Parallelism ParallelismConfig `alloy:"parallelism,block,optional"` ExternalLabels map[string]string `alloy:"external_labels,attr,optional"` TLSConfig *TLSConfig `alloy:"tls_config,block,optional"` RoundRobin bool `alloy:"enable_round_robin,attr,optional"` @@ -100,6 +126,17 @@ type TLSConfig struct { InsecureSkipVerify bool `alloy:"insecure_skip_verify,attr,optional"` } +type ParallelismConfig struct { + DriftScaleUp time.Duration `alloy:"drift_scale_up,attr,optional"` + DriftScaleDown time.Duration `alloy:"drift_scale_down,attr,optional"` + MaxConnections uint `alloy:"max_connections,attr,optional"` + MinConnections uint `alloy:"min_connections,attr,optional"` + NetworkFlushInterval time.Duration `alloy:"network_flush_interval,attr,optional"` + DesiredConnectionsLookback time.Duration `alloy:"desired_connections_lookback,attr,optional"` + DesiredCheckInterval time.Duration `alloy:"desired_check_interval,attr,optional"` + AllowedNetworkErrorFraction float64 `alloy:"allowed_network_error_fraction,attr,optional"` +} + var UserAgent = fmt.Sprintf("Alloy/%s", version.Version) func (cc EndpointConfig) ToNativeType() types.ConnectionConfig { @@ -113,8 +150,17 @@ func (cc EndpointConfig) ToNativeType() types.ConnectionConfig { BatchCount: cc.BatchCount, FlushInterval: cc.FlushInterval, ExternalLabels: cc.ExternalLabels, - Connections: cc.Parallelism, UseRoundRobin: cc.RoundRobin, + Parallelism: types.ParallelismConfig{ + AllowedDrift: cc.Parallelism.DriftScaleUp, + MinimumScaleDownDrift: cc.Parallelism.DriftScaleDown, + MaxConnections: cc.Parallelism.MaxConnections, + MinConnections: cc.Parallelism.MinConnections, + ResetInterval: cc.Parallelism.NetworkFlushInterval, + Lookback: cc.Parallelism.DesiredConnectionsLookback, + CheckInterval: cc.Parallelism.DesiredCheckInterval, + AllowedNetworkErrorFraction: cc.Parallelism.AllowedNetworkErrorFraction, + }, } if cc.BasicAuth != nil { tcc.BasicAuth = &types.BasicAuth{ diff --git a/internal/component/prometheus/write/queue/types_test.go b/internal/component/prometheus/write/queue/types_test.go index d2e767bef3..c5fbd17088 100644 --- a/internal/component/prometheus/write/queue/types_test.go +++ b/internal/component/prometheus/write/queue/types_test.go @@ -1,9 +1,11 @@ package queue import ( + "testing" + "time" + "github.com/grafana/alloy/syntax" "github.com/stretchr/testify/require" - "testing" ) func TestParsingTLSConfig(t *testing.T) { @@ -24,3 +26,78 @@ func TestParsingTLSConfig(t *testing.T) { require.NoError(t, err) } + +func TestParralelismConfig_Validate(t *testing.T) { + testCases := []struct { + name string + config func(cfg ParallelismConfig) ParallelismConfig + expectedErrMsg string + }{ + { + name: "default config is valid", + config: func(cfg ParallelismConfig) ParallelismConfig { + return cfg + }, + }, + { + name: "positive drift scale up seconds is invalid", + config: func(cfg ParallelismConfig) ParallelismConfig { + cfg.DriftScaleUp = 10 * time.Second + cfg.DriftScaleDown = 10 * time.Second + return cfg + }, + expectedErrMsg: "drift_scale_up_seconds less than or equal drift_scale_down_seconds", + }, + { + name: "max less than min", + config: func(cfg ParallelismConfig) ParallelismConfig { + cfg.MaxConnections = 1 + cfg.MinConnections = 2 + return cfg + }, + expectedErrMsg: "max_connections less than min_connections", + }, + { + name: "to low desired check", + config: func(cfg ParallelismConfig) ParallelismConfig { + cfg.DesiredCheckInterval = (1 * time.Second) - (50 * time.Millisecond) + return cfg + }, + expectedErrMsg: "desired_check_interval must be greater than or equal to 1 second", + }, + { + name: "invalid network error percentage low", + config: func(cfg ParallelismConfig) ParallelismConfig { + cfg.AllowedNetworkErrorFraction = -0.01 + return cfg + }, + expectedErrMsg: "allowed_network_error_percent must be between 0.00 and 1.00", + }, + { + name: "invalid network error percentage high", + config: func(cfg ParallelismConfig) ParallelismConfig { + cfg.AllowedNetworkErrorFraction = 1.01 + return cfg + }, + expectedErrMsg: "allowed_network_error_percent must be between 0.00 and 1.00", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cfg := defaultEndpointConfig() + cfg.Parallelism = tc.config(cfg.Parallelism) + args := &Arguments{ + Endpoints: []EndpointConfig{cfg}, + } + err := args.Validate() + + if tc.expectedErrMsg == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedErrMsg) + } + }) + } +} From 1abbb590745ba18265ba0abbb5588c6b3b285c9a Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 13 Feb 2025 22:13:22 +0200 Subject: [PATCH 07/14] remotecfg: bump protocol version, use GET requests (#2668) * Bump alloy-remote-config to v0.0.10 Signed-off-by: Paschalis Tsilias * Update ConnectRPC client to use new field name, and enable GET requests Signed-off-by: Paschalis Tsilias * Trigger CI Signed-off-by: Paschalis Tsilias --------- Signed-off-by: Paschalis Tsilias --- go.mod | 2 +- go.sum | 4 ++-- internal/service/remotecfg/remotecfg.go | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 51d9f7a77c..7cbc4d741c 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/google/renameio/v2 v2.0.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 - github.com/grafana/alloy-remote-config v0.0.9 + github.com/grafana/alloy-remote-config v0.0.10 github.com/grafana/alloy/syntax v0.1.0 github.com/grafana/beyla v1.10.0-alloy // custom beyla 1.10 branch without git lfs github.com/grafana/catchpoint-prometheus-exporter v0.0.0-20240606062944-e55f3668661d diff --git a/go.sum b/go.sum index 2df083018c..47e037c920 100644 --- a/go.sum +++ b/go.sum @@ -1863,8 +1863,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gosnmp/gosnmp v1.38.0 h1:I5ZOMR8kb0DXAFg/88ACurnuwGwYkXWq3eLpJPHMEYc= github.com/gosnmp/gosnmp v1.38.0/go.mod h1:FE+PEZvKrFz9afP9ii1W3cprXuVZ17ypCcyyfYuu5LY= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= -github.com/grafana/alloy-remote-config v0.0.9 h1:gy34SxZ8Iq/HrDTIFZi80+8BlT+FnJhKiP9mryHNEUE= -github.com/grafana/alloy-remote-config v0.0.9/go.mod h1:kHE1usYo2WAVCikQkIXuoG1Clz8BSdiz3kF+DZSCQ4k= +github.com/grafana/alloy-remote-config v0.0.10 h1:1Ge7lz2mjXI1rd6SmiZpFHyXeLehBuCi43+XTkdqgV4= +github.com/grafana/alloy-remote-config v0.0.10/go.mod h1:kHE1usYo2WAVCikQkIXuoG1Clz8BSdiz3kF+DZSCQ4k= github.com/grafana/beyla v1.10.0-alloy h1:kGyZtBSS/Br2qdhbvzu8sVYZHuE9a3OzWpbp6gN55EY= github.com/grafana/beyla v1.10.0-alloy/go.mod h1:CRWu15fkScScSYBlYUtdJu2Ak8ojGvnuwHToGGkaOXY= github.com/grafana/cadvisor v0.0.0-20240729082359-1f04a91701e2 h1:ju6EcY2aEobeBg185ETtFCKj5WzaQ48qfkbsSRRQrF4= diff --git a/internal/service/remotecfg/remotecfg.go b/internal/service/remotecfg/remotecfg.go index f3c21a4424..c4b03eecb7 100644 --- a/internal/service/remotecfg/remotecfg.go +++ b/internal/service/remotecfg/remotecfg.go @@ -316,6 +316,7 @@ func (s *Service) Update(newConfig any) error { s.asClient = collectorv1connect.NewCollectorServiceClient( httpClient, newArgs.URL, + connect.WithHTTPGet(), ) } // Combine the new attributes on top of the system attributes @@ -354,9 +355,9 @@ func (s *Service) fetch() { func (s *Service) registerCollector() error { req := connect.NewRequest(&collectorv1.RegisterCollectorRequest{ - Id: s.args.ID, - Attributes: s.attrs, - Name: s.args.Name, + Id: s.args.ID, + LocalAttributes: s.attrs, + Name: s.args.Name, }) client := s.asClient @@ -428,9 +429,9 @@ func (s *Service) fetchLocal() { func (s *Service) getAPIConfig() ([]byte, error) { s.mut.RLock() req := connect.NewRequest(&collectorv1.GetConfigRequest{ - Id: s.args.ID, - Attributes: s.attrs, - Hash: s.remoteHash, + Id: s.args.ID, + LocalAttributes: s.attrs, + Hash: s.remoteHash, }) client := s.asClient s.mut.RUnlock() From cd33426b1d4be1377e067fced63353a9818503b7 Mon Sep 17 00:00:00 2001 From: Erik Baranowski <39704712+erikbaranowski@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:26:24 -0500 Subject: [PATCH 08/14] update the stability level of remotecfg to generally available (#2723) * update the stability level of remotecfg to generally available Signed-off-by: Erik Baranowski * Update docs/sources/reference/config-blocks/remotecfg.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --------- Signed-off-by: Erik Baranowski Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 2 ++ docs/sources/reference/config-blocks/remotecfg.md | 6 ++---- internal/service/remotecfg/remotecfg.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afdaf0444d..cc99425b4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,8 @@ Main (unreleased) In previous versions of Alloy, native histogram support has also been enabled by default as long as `scrape_protocols` starts with `PrometheusProto`. + - Change the stability of the `remotecfg` feature from "public preview" to "generally available". (@erikbaranowski) + v1.6.1 ----------------- diff --git a/docs/sources/reference/config-blocks/remotecfg.md b/docs/sources/reference/config-blocks/remotecfg.md index fba4534700..46b12d0fde 100644 --- a/docs/sources/reference/config-blocks/remotecfg.md +++ b/docs/sources/reference/config-blocks/remotecfg.md @@ -2,15 +2,13 @@ canonical: https://grafana.com/docs/alloy/latest/reference/config-blocks/remotecfg/ description: Learn about the remotecfg configuration block menuTitle: remotecfg +labels: + stage: general-availability title: remotecfg block --- -Public preview - # remotecfg block -{{< docs/shared lookup="stability/public_preview.md" source="alloy" version="" >}} - `remotecfg` is an optional configuration block that enables {{< param "PRODUCT_NAME" >}} to fetch and load the configuration from a remote endpoint. `remotecfg` is specified without a label and can only be provided once per configuration file. diff --git a/internal/service/remotecfg/remotecfg.go b/internal/service/remotecfg/remotecfg.go index c4b03eecb7..782fda4f2e 100644 --- a/internal/service/remotecfg/remotecfg.go +++ b/internal/service/remotecfg/remotecfg.go @@ -250,7 +250,7 @@ func (s *Service) Definition() service.Definition { Name: ServiceName, ConfigType: Arguments{}, DependsOn: nil, // remotecfg has no dependencies. - Stability: featuregate.StabilityPublicPreview, + Stability: featuregate.StabilityGenerallyAvailable, } } From 3ae862c3edef6fc259c253df4c06cf261c8552ee Mon Sep 17 00:00:00 2001 From: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> Date: Thu, 13 Feb 2025 13:04:46 -0800 Subject: [PATCH 09/14] Update the Alloy Linux and FreeBSD version support (#2724) * Update the Linux version support * Update FreeBSD info * Update wording for lifecycle --- docs/sources/introduction/supported-platforms.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sources/introduction/supported-platforms.md b/docs/sources/introduction/supported-platforms.md index b79fde02af..c55e22b2e2 100644 --- a/docs/sources/introduction/supported-platforms.md +++ b/docs/sources/introduction/supported-platforms.md @@ -12,8 +12,9 @@ The following operating systems and hardware architecture are supported. ## Linux -* Minimum version: kernel 2.6.32 or later +* Minimum version: kernel 4.x or later * Architectures: AMD64, ARM64 +* Within the Linux distribution lifecycle ## Windows @@ -27,5 +28,5 @@ The following operating systems and hardware architecture are supported. ## FreeBSD -* Minimum version: FreeBSD 10 or later +* Within the FreeBSD lifecycle * Architectures: AMD64 From 512db4365b97b947fe7ee1b53eb39b6cfd666bc7 Mon Sep 17 00:00:00 2001 From: Cristian Greco Date: Fri, 14 Feb 2025 09:39:28 +0100 Subject: [PATCH 10/14] database_observability: don't error out for `insert` with `values` (#2716) Statements like `insert into table values (...)` were logging an error even though the tables were correctly parsed. --- .../database_observability/mysql/collector/query_sample.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/component/database_observability/mysql/collector/query_sample.go b/internal/component/database_observability/mysql/collector/query_sample.go index 2e83c9ec1d..40505d483c 100644 --- a/internal/component/database_observability/mysql/collector/query_sample.go +++ b/internal/component/database_observability/mysql/collector/query_sample.go @@ -226,6 +226,8 @@ func (c QuerySample) tablesFromQuery(digest string, stmt sqlparser.Statement) [] case *sqlparser.Insert: parsedTables = []string{c.parseTableName(stmt.Table)} switch insRowsStmt := stmt.Rows.(type) { + case sqlparser.Values: + // ignore raw values case *sqlparser.Select: parsedTables = append(parsedTables, c.tablesFromQuery(digest, insRowsStmt)...) case *sqlparser.Union: From 4573fa54d091cc8020c83663c624bf51fa58a60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Madar=C3=A1sz?= Date: Fri, 14 Feb 2025 11:51:00 +0100 Subject: [PATCH 11/14] otelcol.processor.cumulativetodelta: new component (#2689) * Initial implementation of cumulativetodelta * Apply suggestions from code review Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Fix cumulativetodelta stage in docs * Make the metrics and match_type fields optional in the include/exclude blocks * Another experimental to public preview fix * Apply suggestions from code review Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * Add cumulativetodelta tests --------- Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- CHANGELOG.md | 3 + .../sources/reference/compatibility/_index.md | 2 + .../otelcol.processor.cumulativetodelta.md | 169 ++++++++++++++ go.mod | 1 + go.sum | 2 + internal/component/all/all.go | 1 + .../cumulativetodelta/cumulativetodelta.go | 206 ++++++++++++++++++ .../cumulativetodelta_test.go | 199 +++++++++++++++++ .../converter_cumulativetodeltaprocessor.go | 72 ++++++ .../testdata/cumulativetodelta.alloy | 38 ++++ .../testdata/cumulativetodelta.yaml | 30 +++ 11 files changed, 723 insertions(+) create mode 100644 docs/sources/reference/components/otelcol/otelcol.processor.cumulativetodelta.md create mode 100644 internal/component/otelcol/processor/cumulativetodelta/cumulativetodelta.go create mode 100644 internal/component/otelcol/processor/cumulativetodelta/cumulativetodelta_test.go create mode 100644 internal/converter/internal/otelcolconvert/converter_cumulativetodeltaprocessor.go create mode 100644 internal/converter/internal/otelcolconvert/testdata/cumulativetodelta.alloy create mode 100644 internal/converter/internal/otelcolconvert/testdata/cumulativetodelta.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index cc99425b4f..62e2923c50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ Main (unreleased) ### Features +- (_Public preview_) Add a `otelcol.processor.cumulativetodelta` component to convert metrics from + cumulative temporality to delta. (@madaraszg-tulip) + - (_Experimental_) Add a `stage.windowsevent` block in the `loki.process` component. This aims to replace the existing `stage.eventlogmessage`. (@wildum) - Add `pyroscope.relabel` component to modify or filter profiles using Prometheus relabeling rules. (@marcsanmi) diff --git a/docs/sources/reference/compatibility/_index.md b/docs/sources/reference/compatibility/_index.md index 3a22b11990..6ec6afc335 100644 --- a/docs/sources/reference/compatibility/_index.md +++ b/docs/sources/reference/compatibility/_index.md @@ -310,6 +310,7 @@ The following components, grouped by namespace, _export_ OpenTelemetry `otelcol. - [otelcol.exporter.syslog](../components/otelcol/otelcol.exporter.syslog) - [otelcol.processor.attributes](../components/otelcol/otelcol.processor.attributes) - [otelcol.processor.batch](../components/otelcol/otelcol.processor.batch) +- [otelcol.processor.cumulativetodelta](../components/otelcol/otelcol.processor.cumulativetodelta) - [otelcol.processor.deltatocumulative](../components/otelcol/otelcol.processor.deltatocumulative) - [otelcol.processor.discovery](../components/otelcol/otelcol.processor.discovery) - [otelcol.processor.filter](../components/otelcol/otelcol.processor.filter) @@ -348,6 +349,7 @@ The following components, grouped by namespace, _consume_ OpenTelemetry `otelcol - [otelcol.connector.spanmetrics](../components/otelcol/otelcol.connector.spanmetrics) - [otelcol.processor.attributes](../components/otelcol/otelcol.processor.attributes) - [otelcol.processor.batch](../components/otelcol/otelcol.processor.batch) +- [otelcol.processor.cumulativetodelta](../components/otelcol/otelcol.processor.cumulativetodelta) - [otelcol.processor.deltatocumulative](../components/otelcol/otelcol.processor.deltatocumulative) - [otelcol.processor.discovery](../components/otelcol/otelcol.processor.discovery) - [otelcol.processor.filter](../components/otelcol/otelcol.processor.filter) diff --git a/docs/sources/reference/components/otelcol/otelcol.processor.cumulativetodelta.md b/docs/sources/reference/components/otelcol/otelcol.processor.cumulativetodelta.md new file mode 100644 index 0000000000..43721fd6e7 --- /dev/null +++ b/docs/sources/reference/components/otelcol/otelcol.processor.cumulativetodelta.md @@ -0,0 +1,169 @@ +--- +canonical: https://grafana.com/docs/alloy/latest/reference/components/otelcol/otelcol.processor.cumulativetodelta/ +aliases: + - ../otelcol.processor.cumulativetodelta/ # /docs/alloy/latest/reference/otelcol.processor.cumulativetodelta/ +description: Learn about otelcol.processor.cumulativetodelta +labels: + stage: public-preview +title: otelcol.processor.cumulativetodelta +--- + +# `otelcol.processor.cumulativetodelta` + +{{< docs/shared lookup="stability/public_preview.md" source="alloy" version="" >}} + +`otelcol.processor.cumulativetodelta` accepts metrics from other `otelcol` components and converts metrics with the cumulative temporality to delta. + +{{< admonition type="note" >}} +`otelcol.processor.cumulativetodelta` is a wrapper over the upstream OpenTelemetry Collector `cumulativetodelta` processor. +Bug reports or feature requests will be redirected to the upstream repository, if necessary. +{{< /admonition >}} + +You can specify multiple `otelcol.processor.cumulativetodelta` components by giving them different labels. + +## Usage + +```alloy +otelcol.processor.cumulativetodelta "