Target allocator support for the Agent (#245)

Co-authored-by: David Ashpole <[email protected]>
Co-authored-by: Hyunsoo Kim <[email protected]>

.chloggen/refactor-target-allocator.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: prometheusreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Move the TargetAllocator configuration struct to an internal directory
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33146]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

receiver/prometheusreceiver/config.go

@@ -6,19 +6,17 @@ package prometheusreceiver // import "github.com/open-telemetry/opentelemetry-co
 import (
 	"errors"
 	"fmt"
-	"net/url"
 	"os"
 	"sort"
 	"strings"
-	"time"
 
 	commonconfig "github.com/prometheus/common/config"
 	promconfig "github.com/prometheus/prometheus/config"
-	promHTTP "github.com/prometheus/prometheus/discovery/http"
 	"github.com/prometheus/prometheus/discovery/kubernetes"
-	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/confmap"
 	"gopkg.in/yaml.v2"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/targetallocator"
 )
 
 // Config defines configuration for Prometheus receiver.
@@ -37,7 +35,7 @@ type Config struct {
 	// ReportExtraScrapeMetrics - enables reporting of additional metrics for Prometheus client like scrape_body_size_bytes
 	ReportExtraScrapeMetrics bool `mapstructure:"report_extra_scrape_metrics"`
 
-	TargetAllocator *TargetAllocator `mapstructure:"target_allocator"`
+	TargetAllocator *targetallocator.Config `mapstructure:"target_allocator"`
 }
 
 // Validate checks the receiver configuration is valid.
@@ -48,32 +46,12 @@ func (cfg *Config) Validate() error {
 	return nil
 }
 
-type TargetAllocator struct {
-	confighttp.ClientConfig `mapstructure:",squash"`
-	Interval                time.Duration         `mapstructure:"interval"`
-	CollectorID             string                `mapstructure:"collector_id"`
-	HTTPSDConfig            *PromHTTPSDConfig     `mapstructure:"http_sd_config"`
-	HTTPScrapeConfig        *PromHTTPClientConfig `mapstructure:"http_scrape_config"`
-}
-
-func (cfg *TargetAllocator) Validate() error {
-	// ensure valid endpoint
-	if _, err := url.ParseRequestURI(cfg.Endpoint); err != nil {
-		return fmt.Errorf("TargetAllocator endpoint is not valid: %s", cfg.Endpoint)
-	}
-	// ensure valid collectorID without variables
-	if cfg.CollectorID == "" || strings.Contains(cfg.CollectorID, "${") {
-		return fmt.Errorf("CollectorID is not a valid ID")
-	}
-
-	return nil
-}
-
 // PromConfig is a redeclaration of promconfig.Config because we need custom unmarshaling
 // as prometheus "config" uses `yaml` tags.
 type PromConfig promconfig.Config
 
 var _ confmap.Unmarshaler = (*PromConfig)(nil)
+var _ confmap.Marshaler = (*PromConfig)(nil)
 
 func (cfg *PromConfig) Unmarshal(componentParser *confmap.Conf) error {
 	cfgMap := componentParser.ToStringMap()
@@ -83,6 +61,20 @@ func (cfg *PromConfig) Unmarshal(componentParser *confmap.Conf) error {
 	return unmarshalYAML(cfgMap, (*promconfig.Config)(cfg))
 }
 
+func (cfg PromConfig) Marshal(componentParser *confmap.Conf) error {
+	yamlOut, err := yaml.Marshal(cfg)
+	if err != nil {
+		return fmt.Errorf("prometheus receiver: failed to marshal config to yaml: %w", err)
+	}
+
+	var result map[string]any
+	err = yaml.UnmarshalStrict(yamlOut, &result)
+	if err != nil {
+		return fmt.Errorf("prometheus receiver: failed to unmarshal yaml to prometheus config object: %w", err)
+	}
+	return componentParser.Merge(confmap.NewFromStringMap(result))
+}
+
 func (cfg *PromConfig) Validate() error {
 	// Reject features that Prometheus supports but that the receiver doesn't support:
 	// See:
@@ -126,43 +118,6 @@ func (cfg *PromConfig) Validate() error {
 	return nil
 }
 
-// PromHTTPSDConfig is a redeclaration of promHTTP.SDConfig because we need custom unmarshaling
-// as prometheus "config" uses `yaml` tags.
-type PromHTTPSDConfig promHTTP.SDConfig
-
-var _ confmap.Unmarshaler = (*PromHTTPSDConfig)(nil)
-
-func (cfg *PromHTTPSDConfig) Unmarshal(componentParser *confmap.Conf) error {
-	cfgMap := componentParser.ToStringMap()
-	if len(cfgMap) == 0 {
-		return nil
-	}
-	cfgMap["url"] = "http://placeholder" // we have to set it as else marshaling will fail
-	return unmarshalYAML(cfgMap, (*promHTTP.SDConfig)(cfg))
-}
-
-type PromHTTPClientConfig commonconfig.HTTPClientConfig
-
-var _ confmap.Unmarshaler = (*PromHTTPClientConfig)(nil)
-
-func (cfg *PromHTTPClientConfig) Unmarshal(componentParser *confmap.Conf) error {
-	cfgMap := componentParser.ToStringMap()
-	if len(cfgMap) == 0 {
-		return nil
-	}
-	return unmarshalYAML(cfgMap, (*commonconfig.HTTPClientConfig)(cfg))
-}
-
-func (cfg *PromHTTPClientConfig) Validate() error {
-	httpCfg := (*commonconfig.HTTPClientConfig)(cfg)
-	if err := validateHTTPClientConfig(httpCfg); err != nil {
-		return err
-	}
-	// Prometheus UnmarshalYaml implementation by default calls Validate,
-	// but it is safer to do it here as well.
-	return httpCfg.Validate()
-}
-
 func unmarshalYAML(in map[string]any, out any) error {
 	yamlOut, err := yaml.Marshal(in)
 	if err != nil {

receiver/prometheusreceiver/config_test.go

@@ -104,6 +104,22 @@ func TestLoadTargetAllocatorConfig(t *testing.T) {
 	assert.Equal(t, 1, len(r2.PrometheusConfig.ScrapeConfigs))
 	assert.Equal(t, "demo", r2.PrometheusConfig.ScrapeConfigs[0].JobName)
 	assert.Equal(t, promModel.Duration(5*time.Second), r2.PrometheusConfig.ScrapeConfigs[0].ScrapeInterval)
+
+	sub, err = cm.Sub(component.NewIDWithName(metadata.Type, "k8Setup").String())
+	require.NoError(t, err)
+
+	t.Setenv("POD_NAME", "collector-1")
+	cfg = factory.CreateDefaultConfig()
+	require.NoError(t, sub.Unmarshal(cfg))
+	require.NoError(t, component.ValidateConfig(cfg))
+
+	r3 := cfg.(*Config)
+	assert.Equal(t, "https://target-allocator-service:80", r3.TargetAllocator.Endpoint)
+	assert.Equal(t, 30*time.Second, r3.TargetAllocator.Interval)
+	assert.Equal(t, "collector-1", r3.TargetAllocator.CollectorID)
+	assert.Equal(t, promModel.Duration(15*time.Second), r3.PrometheusConfig.GlobalConfig.ScrapeInterval)
+	assert.Equal(t, promModel.Duration(10*time.Second), r3.PrometheusConfig.GlobalConfig.ScrapeTimeout)
+
 }
 
 func TestLoadConfigFailsOnUnknownSection(t *testing.T) {
@@ -341,34 +357,3 @@ func TestFileSDConfigWithoutSDFile(t *testing.T) {
 
 	require.NoError(t, component.ValidateConfig(cfg))
 }
-
-func TestPromHTTPClientConfigValidateAuthorization(t *testing.T) {
-	cfg := PromHTTPClientConfig{}
-	require.NoError(t, component.ValidateConfig(cfg))
-	cfg.Authorization = &promConfig.Authorization{}
-	require.NoError(t, component.ValidateConfig(cfg))
-	cfg.Authorization.CredentialsFile = "none"
-	require.Error(t, component.ValidateConfig(cfg))
-	cfg.Authorization.CredentialsFile = filepath.Join("testdata", "dummy-tls-cert-file")
-	require.NoError(t, component.ValidateConfig(cfg))
-}
-
-func TestPromHTTPClientConfigValidateTLSConfig(t *testing.T) {
-	cfg := PromHTTPClientConfig{}
-	require.NoError(t, component.ValidateConfig(cfg))
-	cfg.TLSConfig.CertFile = "none"
-	require.Error(t, component.ValidateConfig(cfg))
-	cfg.TLSConfig.CertFile = filepath.Join("testdata", "dummy-tls-cert-file")
-	cfg.TLSConfig.KeyFile = "none"
-	require.Error(t, component.ValidateConfig(cfg))
-	cfg.TLSConfig.KeyFile = filepath.Join("testdata", "dummy-tls-key-file")
-	require.NoError(t, component.ValidateConfig(cfg))
-}
-
-func TestPromHTTPClientConfigValidateMain(t *testing.T) {
-	cfg := PromHTTPClientConfig{}
-	require.NoError(t, component.ValidateConfig(cfg))
-	cfg.BearerToken = "foo"
-	cfg.BearerTokenFile = filepath.Join("testdata", "dummy-tls-key-file")
-	require.Error(t, component.ValidateConfig(cfg))
-}

receiver/prometheusreceiver/go.mod

@@ -17,6 +17,8 @@ require (
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.103.0
 	go.opentelemetry.io/collector/config/confighttp v0.103.0
+	go.opentelemetry.io/collector/config/configopaque v1.10.0
+	go.opentelemetry.io/collector/config/configtls v0.103.0
 	go.opentelemetry.io/collector/confmap v0.103.0
 	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.103.0
 	go.opentelemetry.io/collector/consumer v0.103.0
@@ -165,10 +167,8 @@ require (
 	go.opentelemetry.io/collector v0.103.0 // indirect
 	go.opentelemetry.io/collector/config/configauth v0.103.0 // indirect
 	go.opentelemetry.io/collector/config/configcompression v1.10.0 // indirect
-	go.opentelemetry.io/collector/config/configopaque v1.10.0 // indirect
 	go.opentelemetry.io/collector/config/configretry v0.103.0 // indirect
 	go.opentelemetry.io/collector/config/configtelemetry v0.103.0 // indirect
-	go.opentelemetry.io/collector/config/configtls v0.103.0 // indirect
 	go.opentelemetry.io/collector/config/internal v0.103.0 // indirect
 	go.opentelemetry.io/collector/confmap/converter/expandconverter v0.103.0 // indirect
 	go.opentelemetry.io/collector/confmap/provider/envprovider v0.103.0 // indirect