From bde6a2aea2321f2dc26773f445d1cb19e2bc98e7 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 22 Nov 2023 12:58:59 -0500 Subject: [PATCH] Adding TLS connection between node and OTEL collector for traces (#11278) * feature/tracing-credentials: adding TLS connection between node and OTEL collector for traces * feature/tracing-credentials: adding Tracing.Mode to configure insecure credentials explicitly * feature/tracing-credentials: relaxing conditions for Tracing.Mode and Tracing.TLSCertPath * feature/tracing-credentials: moving make file changes to a separate PR * feature/tracing-credentials: bumping relay and lint * feature/tracing-credentials: 'secure' --> 'tls' and 'insecure' --> 'unencrypted' * feature/tracing-credentials: minor docs refactoring * feature/tracing-credentials: adding changelog --- .github/tracing/README.md | 9 +- core/config/docs/core.toml | 10 +- core/config/toml/types.go | 63 ++++- core/config/toml/types_test.go | 230 +++++++++++++++++- core/config/tracing_config.go | 4 +- core/services/chainlink/config.go | 21 ++ core/services/chainlink/config_general.go | 2 +- core/services/chainlink/config_test.go | 82 ++++++- core/services/chainlink/config_tracing.go | 16 +- .../services/chainlink/config_tracing_test.go | 8 + .../testdata/config-empty-effective.toml | 2 + .../chainlink/testdata/config-full.toml | 2 + .../config-multi-chain-effective.toml | 2 + .../testdata/config-empty-effective.toml | 2 + core/web/resolver/testdata/config-full.toml | 2 + .../config-multi-chain-effective.toml | 2 + docs/CHANGELOG.md | 1 + docs/CONFIG.md | 26 +- integration-tests/types/config/node/core.go | 5 +- plugins/loop_registry.go | 3 +- plugins/loop_registry_test.go | 11 +- testdata/scripts/node/db/help.txtar | 2 +- testdata/scripts/node/validate/default.txtar | 2 + .../disk-based-logging-disabled.txtar | 2 + .../validate/disk-based-logging-no-dir.txtar | 2 + .../node/validate/disk-based-logging.txtar | 2 + testdata/scripts/node/validate/invalid.txtar | 2 + testdata/scripts/node/validate/valid.txtar | 2 + testdata/scripts/node/validate/warnings.txtar | 21 +- 29 files changed, 497 insertions(+), 41 deletions(-) diff --git a/.github/tracing/README.md b/.github/tracing/README.md index eb757384295..feba31feb65 100644 --- a/.github/tracing/README.md +++ b/.github/tracing/README.md @@ -30,7 +30,14 @@ Another way to generate traces is by enabling traces for PRs. This will instrume 8. Run `sh replay.sh` to replay those traces to the otel-collector container that was spun up in the last step. 9. navigate to `localhost:3000/explore` in a web browser to query for traces -The artifact is not json encoded - each individual line is a well formed and complete json object. +The artifact is not json encoded - each individual line is a well formed and complete json object. + + +## Production and NOPs environments + +In a production environment, we suggest coupling the lifecycle of nodes and otel-collectors. A best practice is to deploy the otel-collector alongside your node, using infrastructure as code (IAC) to automate deployments and certificate lifecycles. While there are valid use cases for using `Tracing.Mode = unencrypted`, we have set the default encryption setting to `Tracing.Mode = tls`. Externally deployed otel-collectors can not be used with `Tracing.Mode = unencrypted`. i.e. If `Tracing.Mode = unencrypted` and an external URI is detected for `Tracing.CollectorTarget` node configuration will fail to validate and the node will not boot. The node requires a valid encryption mode and collector target to send traces. + +Once traces reach the otel-collector, the rest of the observability pipeline is flexible. We recommend deploying (through automation) centrally managed Grafana Tempo and Grafana UI instances to receive from one or many otel-collector instances. Always use networking best practices and encrypt trace data, especially at network boundaries. ## Configuration This folder contains the following config files: diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 148de90cd95..79801c2c52b 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -587,13 +587,17 @@ DisableRateLimiting = false # Default # Enabled turns trace collection on or off. On requires an OTEL Tracing Collector. Enabled = false # Default # CollectorTarget is the logical address of the OTEL Tracing Collector. -CollectorTarget = "localhost:4317" # Example +CollectorTarget = 'localhost:4317' # Example # NodeID is an unique name for this node relative to any other node traces are collected for. -NodeID = "NodeID" # Example +NodeID = 'NodeID' # Example # SamplingRatio is the ratio of traces to sample for this node. SamplingRatio = 1.0 # Example +# Mode is a string value. `tls` or `unencrypted` are the only values allowed. If set to `unencrypted`, `TLSCertPath` can be unset, meaning traces will be sent over plaintext to the collector. +Mode = 'tls' # Default +# TLSCertPath is the file path to the TLS certificate used for secure communication with an OTEL Tracing Collector. +TLSCertPath = '/path/to/cert.pem' # Example # Tracing.Attributes are user specified key-value pairs to associate in the context of the traces [Tracing.Attributes] # env is an example user specified key-value pair -env = "test" # Example +env = 'test' # Example diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 61962d43e5f..31076c1f1de 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -1455,6 +1455,8 @@ type Tracing struct { CollectorTarget *string NodeID *string SamplingRatio *float64 + Mode *string + TLSCertPath *string Attributes map[string]string `toml:",omitempty"` } @@ -1474,6 +1476,12 @@ func (t *Tracing) setFrom(f *Tracing) { if v := f.SamplingRatio; v != nil { t.SamplingRatio = f.SamplingRatio } + if v := f.Mode; v != nil { + t.Mode = f.Mode + } + if v := f.TLSCertPath; v != nil { + t.TLSCertPath = f.TLSCertPath + } } func (t *Tracing) ValidateConfig() (err error) { @@ -1487,10 +1495,39 @@ func (t *Tracing) ValidateConfig() (err error) { } } - if t.CollectorTarget != nil { - ok := isValidURI(*t.CollectorTarget) - if !ok { - err = multierr.Append(err, configutils.ErrInvalid{Name: "CollectorTarget", Value: *t.CollectorTarget, Msg: "must be a valid URI"}) + if t.Mode != nil { + switch *t.Mode { + case "tls": + // TLSCertPath must be set + if t.TLSCertPath == nil { + err = multierr.Append(err, configutils.ErrMissing{Name: "TLSCertPath", Msg: "must be set when Tracing.Mode is tls"}) + } else { + ok := isValidFilePath(*t.TLSCertPath) + if !ok { + err = multierr.Append(err, configutils.ErrInvalid{Name: "TLSCertPath", Value: *t.TLSCertPath, Msg: "must be a valid file path"}) + } + } + case "unencrypted": + // no-op + default: + // Mode must be either "tls" or "unencrypted" + err = multierr.Append(err, configutils.ErrInvalid{Name: "Mode", Value: *t.Mode, Msg: "must be either 'tls' or 'unencrypted'"}) + } + } + + if t.CollectorTarget != nil && t.Mode != nil { + switch *t.Mode { + case "tls": + if !isValidURI(*t.CollectorTarget) { + err = multierr.Append(err, configutils.ErrInvalid{Name: "CollectorTarget", Value: *t.CollectorTarget, Msg: "must be a valid URI"}) + } + case "unencrypted": + // Unencrypted traces can not be sent to external networks + if !isValidLocalURI(*t.CollectorTarget) { + err = multierr.Append(err, configutils.ErrInvalid{Name: "CollectorTarget", Value: *t.CollectorTarget, Msg: "must be a valid local URI"}) + } + default: + // no-op } } @@ -1499,15 +1536,19 @@ func (t *Tracing) ValidateConfig() (err error) { var hostnameRegex = regexp.MustCompile(`^[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*$`) +// Validates uri is valid external or local URI func isValidURI(uri string) bool { if strings.Contains(uri, "://") { - // Standard URI check - _, _ = url.ParseRequestURI(uri) - // TODO: BCF-2703. Handle error. All external addresses currently fail validation until we have secure transport to external networks. - return false + _, err := url.ParseRequestURI(uri) + return err == nil } - // For URIs like "otel-collector:4317" + return isValidLocalURI(uri) +} + +// isValidLocalURI returns true if uri is a valid local URI +// External URIs (e.g. http://) are not valid local URIs, and will return false. +func isValidLocalURI(uri string) bool { parts := strings.Split(uri, ":") if len(parts) == 2 { host, port := parts[0], parts[1] @@ -1530,3 +1571,7 @@ func isValidURI(uri string) bool { func isValidHostname(hostname string) bool { return hostnameRegex.MatchString(hostname) } + +func isValidFilePath(path string) bool { + return len(path) > 0 && len(path) < 4096 +} diff --git a/core/config/toml/types_test.go b/core/config/toml/types_test.go index 92aaa024304..e16d3a864da 100644 --- a/core/config/toml/types_test.go +++ b/core/config/toml/types_test.go @@ -185,55 +185,115 @@ func TestTracing_ValidateCollectorTarget(t *testing.T) { tests := []struct { name string collectorTarget *string + mode *string wantErr bool errMsg string }{ { - name: "valid http address", + name: "valid http address in tls mode", + collectorTarget: ptr("https://testing.collector.dev"), + mode: ptr("tls"), + wantErr: false, + }, + { + name: "valid http address in unencrypted mode", collectorTarget: ptr("https://localhost:4317"), - // TODO: BCF-2703. Re-enable when we have secure transport to otel collectors in external networks - wantErr: true, - errMsg: "CollectorTarget: invalid value (https://localhost:4317): must be a valid URI", + mode: ptr("unencrypted"), + wantErr: true, + errMsg: "CollectorTarget: invalid value (https://localhost:4317): must be a valid local URI", }, + // Tracing.Mode = 'tls' { name: "valid localhost address", collectorTarget: ptr("localhost:4317"), + mode: ptr("tls"), wantErr: false, }, { name: "valid docker address", collectorTarget: ptr("otel-collector:4317"), + mode: ptr("tls"), wantErr: false, }, { name: "valid IP address", collectorTarget: ptr("192.168.1.1:4317"), + mode: ptr("tls"), wantErr: false, }, { name: "invalid port", collectorTarget: ptr("localhost:invalid"), wantErr: true, + mode: ptr("tls"), errMsg: "CollectorTarget: invalid value (localhost:invalid): must be a valid URI", }, { name: "invalid address", collectorTarget: ptr("invalid address"), wantErr: true, + mode: ptr("tls"), errMsg: "CollectorTarget: invalid value (invalid address): must be a valid URI", }, { name: "nil CollectorTarget", collectorTarget: ptr(""), wantErr: true, + mode: ptr("tls"), errMsg: "CollectorTarget: invalid value (): must be a valid URI", }, + // Tracing.Mode = 'unencrypted' + { + name: "valid localhost address", + collectorTarget: ptr("localhost:4317"), + mode: ptr("unencrypted"), + wantErr: false, + }, + { + name: "valid docker address", + collectorTarget: ptr("otel-collector:4317"), + mode: ptr("unencrypted"), + wantErr: false, + }, + { + name: "valid IP address", + collectorTarget: ptr("192.168.1.1:4317"), + mode: ptr("unencrypted"), + wantErr: false, + }, + { + name: "invalid port", + collectorTarget: ptr("localhost:invalid"), + wantErr: true, + mode: ptr("unencrypted"), + errMsg: "CollectorTarget: invalid value (localhost:invalid): must be a valid local URI", + }, + { + name: "invalid address", + collectorTarget: ptr("invalid address"), + wantErr: true, + mode: ptr("unencrypted"), + errMsg: "CollectorTarget: invalid value (invalid address): must be a valid local URI", + }, + { + name: "nil CollectorTarget", + collectorTarget: ptr(""), + wantErr: true, + mode: ptr("unencrypted"), + errMsg: "CollectorTarget: invalid value (): must be a valid local URI", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + var tlsCertPath string + if *tt.mode == "tls" { + tlsCertPath = "/path/to/cert.pem" + } tracing := &Tracing{ Enabled: ptr(true), + TLSCertPath: &tlsCertPath, + Mode: tt.mode, CollectorTarget: tt.collectorTarget, } @@ -309,5 +369,167 @@ func TestTracing_ValidateSamplingRatio(t *testing.T) { } } +func TestTracing_ValidateTLSCertPath(t *testing.T) { + // tests for Tracing.Mode = 'tls' + tls_tests := []struct { + name string + tlsCertPath *string + wantErr bool + errMsg string + }{ + { + name: "valid file path", + tlsCertPath: ptr("/etc/ssl/certs/cert.pem"), + wantErr: false, + }, + { + name: "relative file path", + tlsCertPath: ptr("certs/cert.pem"), + wantErr: false, + }, + { + name: "excessively long file path", + tlsCertPath: ptr(strings.Repeat("z", 4097)), + wantErr: true, + errMsg: "TLSCertPath: invalid value (" + strings.Repeat("z", 4097) + "): must be a valid file path", + }, + { + name: "empty file path", + tlsCertPath: ptr(""), + wantErr: true, + errMsg: "TLSCertPath: invalid value (): must be a valid file path", + }, + } + + // tests for Tracing.Mode = 'unencrypted' + unencrypted_tests := []struct { + name string + tlsCertPath *string + wantErr bool + errMsg string + }{ + { + name: "valid file path", + tlsCertPath: ptr("/etc/ssl/certs/cert.pem"), + wantErr: false, + }, + { + name: "relative file path", + tlsCertPath: ptr("certs/cert.pem"), + wantErr: false, + }, + { + name: "excessively long file path", + tlsCertPath: ptr(strings.Repeat("z", 4097)), + wantErr: false, + }, + { + name: "empty file path", + tlsCertPath: ptr(""), + wantErr: false, + }, + } + + for _, tt := range tls_tests { + t.Run(tt.name, func(t *testing.T) { + tracing := &Tracing{ + Mode: ptr("tls"), + TLSCertPath: tt.tlsCertPath, + Enabled: ptr(true), + } + + err := tracing.ValidateConfig() + + if tt.wantErr { + assert.Error(t, err) + assert.Equal(t, tt.errMsg, err.Error()) + } else { + assert.NoError(t, err) + } + }) + } + + for _, tt := range unencrypted_tests { + t.Run(tt.name, func(t *testing.T) { + tracing := &Tracing{ + Mode: ptr("unencrypted"), + TLSCertPath: tt.tlsCertPath, + Enabled: ptr(true), + } + + err := tracing.ValidateConfig() + + if tt.wantErr { + assert.Error(t, err) + assert.Equal(t, tt.errMsg, err.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestTracing_ValidateMode(t *testing.T) { + tests := []struct { + name string + mode *string + tlsCertPath *string + wantErr bool + errMsg string + }{ + { + name: "tls mode with valid TLS path", + mode: ptr("tls"), + tlsCertPath: ptr("/path/to/cert.pem"), + wantErr: false, + }, + { + name: "tls mode without TLS path", + mode: ptr("tls"), + tlsCertPath: nil, + wantErr: true, + errMsg: "TLSCertPath: missing: must be set when Tracing.Mode is tls", + }, + { + name: "unencrypted mode with TLS path", + mode: ptr("unencrypted"), + tlsCertPath: ptr("/path/to/cert.pem"), + wantErr: false, + }, + { + name: "unencrypted mode without TLS path", + mode: ptr("unencrypted"), + tlsCertPath: nil, + wantErr: false, + }, + { + name: "invalid mode", + mode: ptr("unknown"), + tlsCertPath: nil, + wantErr: true, + errMsg: "Mode: invalid value (unknown): must be either 'tls' or 'unencrypted'", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tracing := &Tracing{ + Enabled: ptr(true), + Mode: tt.mode, + TLSCertPath: tt.tlsCertPath, + } + + err := tracing.ValidateConfig() + + if tt.wantErr { + assert.Error(t, err) + assert.Equal(t, tt.errMsg, err.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + // ptr is a utility function for converting a value to a pointer to the value. func ptr[T any](t T) *T { return &t } diff --git a/core/config/tracing_config.go b/core/config/tracing_config.go index 28584a9cde4..307a010c486 100644 --- a/core/config/tracing_config.go +++ b/core/config/tracing_config.go @@ -4,6 +4,8 @@ type Tracing interface { Enabled() bool CollectorTarget() string NodeID() string - Attributes() map[string]string SamplingRatio() float64 + TLSCertPath() string + Mode() string + Attributes() map[string]string } diff --git a/core/services/chainlink/config.go b/core/services/chainlink/config.go index 10598718f97..5d8b1019e8c 100644 --- a/core/services/chainlink/config.go +++ b/core/services/chainlink/config.go @@ -52,6 +52,27 @@ func (c *Config) TOMLString() (string, error) { return string(b), nil } +// warnings aggregates warnings from valueWarnings and deprecationWarnings +func (c *Config) warnings() (err error) { + deprecationErr := c.deprecationWarnings() + warningErr := c.valueWarnings() + err = multierr.Append(deprecationErr, warningErr) + _, list := utils.MultiErrorList(err) + return list +} + +// valueWarnings returns an error if the Config contains values that hint at misconfiguration before defaults are applied. +func (c *Config) valueWarnings() (err error) { + if c.Tracing.Enabled != nil && *c.Tracing.Enabled { + if c.Tracing.Mode != nil && *c.Tracing.Mode == "unencrypted" { + if c.Tracing.TLSCertPath != nil { + err = multierr.Append(err, config.ErrInvalid{Name: "Tracing.TLSCertPath", Value: *c.Tracing.TLSCertPath, Msg: "must be empty when Tracing.Mode is 'unencrypted'"}) + } + } + } + return +} + // deprecationWarnings returns an error if the Config contains deprecated fields. // This is typically used before defaults have been applied, with input from the user. func (c *Config) deprecationWarnings() (err error) { diff --git a/core/services/chainlink/config_general.go b/core/services/chainlink/config_general.go index 6a835e09c89..fff7822a814 100644 --- a/core/services/chainlink/config_general.go +++ b/core/services/chainlink/config_general.go @@ -137,7 +137,7 @@ func (o GeneralConfigOpts) New() (GeneralConfig, error) { return nil, err } - _, warning := utils.MultiErrorList(o.Config.deprecationWarnings()) + _, warning := utils.MultiErrorList(o.Config.warnings()) o.Config.setDefaults() if !o.SkipEnv { diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 891c0a490fb..33fda221285 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -227,11 +227,13 @@ func TestConfig_Marshal(t *testing.T) { Enabled: ptr(true), CollectorTarget: ptr("localhost:4317"), NodeID: ptr("clc-ocr-sol-devnet-node-1"), + SamplingRatio: ptr(1.0), + Mode: ptr("tls"), + TLSCertPath: ptr("/path/to/cert.pem"), Attributes: map[string]string{ "test": "load", "env": "dev", }, - SamplingRatio: ptr(1.0), }, }, } @@ -688,6 +690,8 @@ Enabled = true CollectorTarget = 'localhost:4317' NodeID = 'clc-ocr-sol-devnet-node-1' SamplingRatio = 1.0 +Mode = 'tls' +TLSCertPath = '/path/to/cert.pem' [Tracing.Attributes] env = 'dev' @@ -1537,4 +1541,80 @@ func TestConfig_SetFrom(t *testing.T) { } } +func TestConfig_Warnings(t *testing.T) { + tests := []struct { + name string + config Config + expectedErrors []string + }{ + { + name: "No warnings", + config: Config{}, + expectedErrors: nil, + }, + { + name: "Value warning - unencrypted mode with TLS path set", + config: Config{ + Core: toml.Core{ + Tracing: toml.Tracing{ + Enabled: ptr(true), + Mode: ptr("unencrypted"), + TLSCertPath: ptr("/path/to/cert.pem"), + }, + }, + }, + expectedErrors: []string{"Tracing.TLSCertPath: invalid value (/path/to/cert.pem): must be empty when Tracing.Mode is 'unencrypted'"}, + }, + { + name: "Deprecation warning - P2P.V1 fields set", + config: Config{ + Core: toml.Core{ + P2P: toml.P2P{ + V1: toml.P2PV1{ + Enabled: ptr(true), + }, + }, + }, + }, + expectedErrors: []string{ + "P2P.V1: is deprecated and will be removed in a future version", + }, + }, + { + name: "Value warning and deprecation warning", + config: Config{ + Core: toml.Core{ + P2P: toml.P2P{ + V1: toml.P2PV1{ + Enabled: ptr(true), + }, + }, + Tracing: toml.Tracing{ + Enabled: ptr(true), + Mode: ptr("unencrypted"), + TLSCertPath: ptr("/path/to/cert.pem"), + }, + }, + }, + expectedErrors: []string{ + "Tracing.TLSCertPath: invalid value (/path/to/cert.pem): must be empty when Tracing.Mode is 'unencrypted'", + "P2P.V1: is deprecated and will be removed in a future version", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.config.warnings() + if len(tt.expectedErrors) == 0 { + assert.NoError(t, err) + } else { + for _, expectedErr := range tt.expectedErrors { + assert.Contains(t, err.Error(), expectedErr) + } + } + }) + } +} + func ptr[T any](t T) *T { return &t } diff --git a/core/services/chainlink/config_tracing.go b/core/services/chainlink/config_tracing.go index 390645974f1..36d09ae0564 100644 --- a/core/services/chainlink/config_tracing.go +++ b/core/services/chainlink/config_tracing.go @@ -18,10 +18,18 @@ func (t tracingConfig) NodeID() string { return *t.s.NodeID } -func (t tracingConfig) Attributes() map[string]string { - return t.s.Attributes -} - func (t tracingConfig) SamplingRatio() float64 { return *t.s.SamplingRatio } + +func (t tracingConfig) Mode() string { + return *t.s.Mode +} + +func (t tracingConfig) TLSCertPath() string { + return *t.s.TLSCertPath +} + +func (t tracingConfig) Attributes() map[string]string { + return t.s.Attributes +} diff --git a/core/services/chainlink/config_tracing_test.go b/core/services/chainlink/config_tracing_test.go index 5968bc306a2..37653729cf3 100644 --- a/core/services/chainlink/config_tracing_test.go +++ b/core/services/chainlink/config_tracing_test.go @@ -14,12 +14,16 @@ func TestTracing_Config(t *testing.T) { collectorTarget := "http://localhost:9000" nodeID := "Node1" samplingRatio := 0.5 + mode := "tls" + tlsCertPath := "/path/to/cert.pem" attributes := map[string]string{"key": "value"} tracing := toml.Tracing{ Enabled: &enabled, CollectorTarget: &collectorTarget, NodeID: &nodeID, SamplingRatio: &samplingRatio, + Mode: &mode, + TLSCertPath: &tlsCertPath, Attributes: attributes, } tConfig := tracingConfig{s: tracing} @@ -28,6 +32,8 @@ func TestTracing_Config(t *testing.T) { assert.Equal(t, "http://localhost:9000", tConfig.CollectorTarget()) assert.Equal(t, "Node1", tConfig.NodeID()) assert.Equal(t, 0.5, tConfig.SamplingRatio()) + assert.Equal(t, "tls", tConfig.Mode()) + assert.Equal(t, "/path/to/cert.pem", tConfig.TLSCertPath()) assert.Equal(t, map[string]string{"key": "value"}, tConfig.Attributes()) // Test when all fields are nil @@ -38,5 +44,7 @@ func TestTracing_Config(t *testing.T) { assert.Panics(t, func() { nilConfig.CollectorTarget() }) assert.Panics(t, func() { nilConfig.NodeID() }) assert.Panics(t, func() { nilConfig.SamplingRatio() }) + assert.Panics(t, func() { nilConfig.Mode() }) + assert.Panics(t, func() { nilConfig.TLSCertPath() }) assert.Nil(t, nilConfig.Attributes()) } diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index b897fba7f10..8f3135b34e4 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -232,3 +232,5 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 531c98d7344..eca5f6f96d2 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -238,6 +238,8 @@ Enabled = true CollectorTarget = 'localhost:4317' NodeID = 'clc-ocr-sol-devnet-node-1' SamplingRatio = 1.0 +Mode = 'tls' +TLSCertPath = '/path/to/cert.pem' [Tracing.Attributes] env = 'dev' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index c743601ced8..6a60dfd419a 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -232,6 +232,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index b897fba7f10..8f3135b34e4 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -232,3 +232,5 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 6cd6eaabc3c..7e9872e9554 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -238,6 +238,8 @@ Enabled = false CollectorTarget = 'localhost:4317' NodeID = 'NodeID' SamplingRatio = 1.0 +Mode = 'tls' +TLSCertPath = '/path/to/cert.pem' [Tracing.Attributes] env = 'dev' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index c743601ced8..6a60dfd419a 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -232,6 +232,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8e016347c43..265222c8bec 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added distributed tracing in the OpenTelemetry trace format to the node, currently focused at the LOOPP Plugin development effort. This includes a new set of `Tracing` TOML configurations. The default for collecting traces is off - you must explicitly enable traces and setup a valid OpenTelemetry collector. Refer to `.github/tracing/README.md` for more details. - Added a new, optional WebServer authentication option that supports LDAP as a user identity provider. This enables user login access and user roles to be managed and provisioned via a centralized remote server that supports the LDAP protocol, which can be helpful when running multiple nodes. See the documentation for more information and config setup instructions. There is a new `[WebServer].AuthenticationMethod` config option, when set to `ldap` requires the new `[WebServer.LDAP]` config section to be defined, see the reference `docs/core.toml`. - New prom metrics for mercury: `mercury_transmit_queue_delete_error_count` diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 5b93c7061e8..63c20bdf4a5 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1595,9 +1595,11 @@ DisableRateLimiting skips ratelimiting on asset requests. ```toml [Tracing] Enabled = false # Default -CollectorTarget = "localhost:4317" # Example -NodeID = "NodeID" # Example +CollectorTarget = 'localhost:4317' # Example +NodeID = 'NodeID' # Example SamplingRatio = 1.0 # Example +Mode = 'tls' # Default +TLSCertPath = '/path/to/cert.pem' # Example ``` @@ -1609,13 +1611,13 @@ Enabled turns trace collection on or off. On requires an OTEL Tracing Collector. ### CollectorTarget ```toml -CollectorTarget = "localhost:4317" # Example +CollectorTarget = 'localhost:4317' # Example ``` CollectorTarget is the logical address of the OTEL Tracing Collector. ### NodeID ```toml -NodeID = "NodeID" # Example +NodeID = 'NodeID' # Example ``` NodeID is an unique name for this node relative to any other node traces are collected for. @@ -1625,16 +1627,28 @@ SamplingRatio = 1.0 # Example ``` SamplingRatio is the ratio of traces to sample for this node. +### Mode +```toml +Mode = 'tls' # Default +``` +Mode is a string value. `tls` or `unencrypted` are the only values allowed. If set to `unencrypted`, `TLSCertPath` can be unset, meaning traces will be sent over plaintext to the collector. + +### TLSCertPath +```toml +TLSCertPath = '/path/to/cert.pem' # Example +``` +TLSCertPath is the file path to the TLS certificate used for secure communication with an OTEL Tracing Collector. + ## Tracing.Attributes ```toml [Tracing.Attributes] -env = "test" # Example +env = 'test' # Example ``` Tracing.Attributes are user specified key-value pairs to associate in the context of the traces ### env ```toml -env = "test" # Example +env = 'test' # Example ``` env is an example user specified key-value pair diff --git a/integration-tests/types/config/node/core.go b/integration-tests/types/config/node/core.go index ad85506a04c..b7f2b316aa7 100644 --- a/integration-tests/types/config/node/core.go +++ b/integration-tests/types/config/node/core.go @@ -138,11 +138,12 @@ func WithTracing() NodeConfigOpt { Enabled: ptr.Ptr(true), CollectorTarget: ptr.Ptr("otel-collector:4317"), // ksortable unique id - NodeID: ptr.Ptr(ksuid.New().String()), + NodeID: ptr.Ptr(ksuid.New().String()), + SamplingRatio: ptr.Ptr(1.0), + Mode: ptr.Ptr("unencrypted"), Attributes: map[string]string{ "env": "smoke", }, - SamplingRatio: ptr.Ptr(1.0), } } } diff --git a/plugins/loop_registry.go b/plugins/loop_registry.go index 17ad7cba5ad..7a5274803d6 100644 --- a/plugins/loop_registry.go +++ b/plugins/loop_registry.go @@ -55,8 +55,9 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) { if m.cfgTracing != nil { envCfg.TracingEnabled = m.cfgTracing.Enabled() envCfg.TracingCollectorTarget = m.cfgTracing.CollectorTarget() - envCfg.TracingAttributes = m.cfgTracing.Attributes() envCfg.TracingSamplingRatio = m.cfgTracing.SamplingRatio() + envCfg.TracingTLSCertPath = m.cfgTracing.TLSCertPath() + envCfg.TracingAttributes = m.cfgTracing.Attributes() } m.registry[id] = &RegisteredLoop{Name: id, EnvCfg: envCfg} diff --git a/plugins/loop_registry_test.go b/plugins/loop_registry_test.go index b5da9154b68..b307469e09b 100644 --- a/plugins/loop_registry_test.go +++ b/plugins/loop_registry_test.go @@ -29,13 +29,15 @@ func TestPluginPortManager(t *testing.T) { // Mock tracing config type MockCfgTracing struct{} -func (m *MockCfgTracing) Enabled() bool { return true } -func (m *MockCfgTracing) CollectorTarget() string { return "http://localhost:9000" } func (m *MockCfgTracing) Attributes() map[string]string { return map[string]string{"attribute": "value"} } -func (m *MockCfgTracing) SamplingRatio() float64 { return 0.1 } -func (m *MockCfgTracing) NodeID() string { return "" } +func (m *MockCfgTracing) Enabled() bool { return true } +func (m *MockCfgTracing) NodeID() string { return "" } +func (m *MockCfgTracing) CollectorTarget() string { return "http://localhost:9000" } +func (m *MockCfgTracing) SamplingRatio() float64 { return 0.1 } +func (m *MockCfgTracing) TLSCertPath() string { return "/path/to/cert.pem" } +func (m *MockCfgTracing) Mode() string { return "tls" } func TestLoopRegistry_Register(t *testing.T) { mockCfgTracing := &MockCfgTracing{} @@ -56,4 +58,5 @@ func TestLoopRegistry_Register(t *testing.T) { require.Equal(t, "http://localhost:9000", registeredLoop.EnvCfg.TracingCollectorTarget) require.Equal(t, map[string]string{"attribute": "value"}, registeredLoop.EnvCfg.TracingAttributes) require.Equal(t, 0.1, registeredLoop.EnvCfg.TracingSamplingRatio) + require.Equal(t, "/path/to/cert.pem", registeredLoop.EnvCfg.TracingTLSCertPath) } diff --git a/testdata/scripts/node/db/help.txtar b/testdata/scripts/node/db/help.txtar index ccdf3431786..4f2fe3e0767 100644 --- a/testdata/scripts/node/db/help.txtar +++ b/testdata/scripts/node/db/help.txtar @@ -20,4 +20,4 @@ COMMANDS: OPTIONS: --help, -h show help - + \ No newline at end of file diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index 01e96ac944d..267a760f08c 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -244,6 +244,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' Invalid configuration: invalid secrets: 2 errors: - Database.URL: empty: must be provided and non-empty diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 1f6901e9ffd..e6281e8d221 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -288,6 +288,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 4c1a1c75fc3..65d832aa82e 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -288,6 +288,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 536b7d8ac08..6b9e3d56ce6 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -288,6 +288,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 89f59574fcc..aa2036413c7 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -278,6 +278,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 2d32b39a644..4ceb9d5df35 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -285,6 +285,8 @@ Enabled = false CollectorTarget = '' NodeID = '' SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' [[EVM]] ChainID = '1' diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index e478203e00e..e4ff2aa35ea 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -15,6 +15,12 @@ ListenPort = 0 NewStreamTimeout = '10s' PeerstoreWriteInterval = '5m0s' +[Tracing] +Enabled = true +CollectorTarget = 'otel-collector:4317' +TLSCertPath = 'something' +Mode = 'unencrypted' + -- secrets.toml -- [Database] URL = 'postgresql://user:pass1234567890abcd@localhost:5432/dbname?sslmode=disable' @@ -46,6 +52,12 @@ ListenPort = 0 NewStreamTimeout = '10s' PeerstoreWriteInterval = '5m0s' +[Tracing] +Enabled = true +CollectorTarget = 'otel-collector:4317' +Mode = 'unencrypted' +TLSCertPath = 'something' + # Effective Configuration, with defaults applied: InsecureFastScrypt = false RootDir = '~/.chainlink' @@ -277,13 +289,15 @@ InfiniteDepthQueries = false DisableRateLimiting = false [Tracing] -Enabled = false -CollectorTarget = '' +Enabled = true +CollectorTarget = 'otel-collector:4317' NodeID = '' SamplingRatio = 0.0 +Mode = 'unencrypted' +TLSCertPath = 'something' # Configuration warning: -2 errors: +3 errors: - P2P.V1: is deprecated and will be removed in a future version - P2P.V1: 10 errors: - AnnounceIP: is deprecated and will be removed in a future version @@ -296,4 +310,5 @@ SamplingRatio = 0.0 - ListenPort: is deprecated and will be removed in a future version - NewStreamTimeout: is deprecated and will be removed in a future version - PeerstoreWriteInterval: is deprecated and will be removed in a future version + - Tracing.TLSCertPath: invalid value (something): must be empty when Tracing.Mode is 'unencrypted' Valid configuration.