Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/go_modules/otel-dependencies-40b2…
Browse files Browse the repository at this point in the history
…0a2a27
  • Loading branch information
andrzej-stencel authored May 29, 2024
2 parents 51090f2 + cb7e377 commit 6851f3c
Show file tree
Hide file tree
Showing 21 changed files with 697 additions and 30 deletions.
14 changes: 13 additions & 1 deletion .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ updates:
labels:
- automation
- skip-changelog
- Team:Elastic-Agent
- Team:Elastic-Agent-Control-Plane
groups:
otel-dependencies:
exclude-patterns:
Expand All @@ -24,3 +24,15 @@ updates:
reviewers:
- "elastic/elastic-agent-control-plane"
open-pull-requests-limit: 10

- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"
labels:
- automation
- skip-changelog
- Team:Elastic-Agent-Control-Plane
reviewers:
- "elastic/elastic-agent-control-plane"
open-pull-requests-limit: 10
2 changes: 1 addition & 1 deletion .github/workflows/bump-agent-versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
fetch-depth: 0

- name: Set up Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: 1.21

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/bump-golang.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
- uses: actions/checkout@v4

- name: Install Updatecli in the runner
uses: updatecli/updatecli-action@9a37c7e35598d7b37d8e7568b40ed9538112be01 # v0.76.1
uses: updatecli/updatecli-action@fa41baa922561b436c449de1a0bd1f5bd768248c # v0.76.1

- name: Run Updatecli in Apply mode
run: updatecli apply --config .github/updatecli-bump-golang.yml
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/fragment-in-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
- uses: actions/setup-go@v5
with:
go-version-file: .go-version
- name: check pr-has-fragment
Expand Down
7 changes: 2 additions & 5 deletions .github/workflows/golangci-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-go@v3
- uses: actions/setup-go@v5
with:
go-version-file: .go-version

- name: golangci-lint
uses: golangci/golangci-lint-action@v3
uses: golangci/golangci-lint-action@v6
with:
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
version: v1.55.2
Expand All @@ -39,8 +39,5 @@ jobs:
# into fixing all linting issues in the whole file instead.
args: --timeout=30m --whole-files

# Optional: if set to true then the action will use pre-installed Go.
skip-go-installation: true

# Optional: show only new issues if it's a pull request. The default value is `false`.
only-new-issues: true
2 changes: 1 addition & 1 deletion .github/workflows/post-dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
steps:
- uses: actions/checkout@v3

- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version-file: .go-version

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Kind can be one of:
# - breaking-change: a change to previously-documented behavior
# - deprecation: functionality that is being removed in a later release
# - bug-fix: fixes a problem in a previous version
# - enhancement: extends functionality but does not break or fix existing behavior
# - feature: new functionality
# - known-issue: problems that we are aware of in a given version
# - security: impacts on the security of a product or a user’s deployment.
# - upgrade: important information for someone upgrading from a prior version
# - other: does not fit into any of the other categories
kind: enhancement

# Change summary; a 80ish characters long description of the change.
summary: Fleet Server component now uses policy output configuration to communicate with Elasticsearch

# Long description; in case the summary is not enough to describe the change
# this field accommodate a description without length limits.
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
description: |
Alter how elatic-agent passes the fleet-server output component so that the policy's output is used.
In cases where fleet-server encounters an error when trying to use the policy's output it will use
the configuration specified during enrollment as a fallback. In cases where it uses the fallback
the policy's output is periodically retested and used if it's successful.
# Affected component; a word indicating the component this changeset affects.
component:

# PR URL; optional; the PR number that added the changeset.
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
# Please provide it if you are adding a fragment for a different PR.
pr: https://github.com/elastic/elastic-agent/pull/4643

# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
# If not present is automatically filled by the tooling with the issue linked to the PR number.
issue: https://github.com/elastic/elastic-agent/issue/2784
58 changes: 58 additions & 0 deletions docs/fleet-server-bootstrap.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,61 @@ its API key to use for communication. The new `fleet.yml` still includes the `fl
but this time the `fleet.server.bootstrap: false` is set.
. `enroll` command then either restarts the running Elatic Agent daemon if one was running
from Step 2, or it stops the spawned `run` subprocess and returns.

=== Elasticsearch output

The options passed that are used to specify fleet-server initially connects to elasticsearch are:

- `--fleet-server-es`
- `--fleet-server-es-ca`
- `--fleet-server-es-ca-trusted-fingerprint`
- `--fleet-server-es-insecure`
- `--fleet-server-es-cert`
- `--fleet-server-es-cert-key`
- `--fleet-server-es-service-token`
- `--fleet-server-es-service-token-path`
- `--proxy-url`
- `--proxy-disabled`
- `--proxy-header`

These options are always passed under a `bootstrap` attribute in the output when elastic-agent is passing config to fleet-server.
When the fleet-server recieves an output block, it will inject any keys that are missing from the top level output but are specified in the `bootstrap` block
After injecting the keys from bootstrap, fleet-server will test connecting the Elasticsearch with the output.
If the test fails, the values under the `bootstrap` attribute are used as the output and fleet-server will periodically retest the output in case the error was caused by a temporary network issue.
Note that if `--fleet-server-es-insecure` is specified, and the output in the policy contains one or more CA, or a CA fingerprint, the `--fleet-server-es-insecure` flag is ignored.

An example of this sequence is sequence is:

1) elastic-agent starts fleet-server and sends an output block that looks similar to:
```yaml
output:
bootstrap:
service_token: VALUE
hosts: ["HOST"]
```

2) fleet-server injects attributes into the top level from bootstrap if they are missing, resulting in
```yaml
output:
service_token: VALUE
hosts: ["HOST"]
```

3) fleet-server connects to Elasticsearch with the output block
4) elastic-agent enrolls and recieves its policy
5) elastic-agent sends configuration generated from the policy to fleet-server, this may result in the output as follows:
```yaml
output:
hosts: ["HOST", "HOST2"]
bootstrap:
service_token: VALUE
hosts: ["HOST"]
```

6) fleet-server will inject missing values resulting in:
```yaml
output:
service_token: VALUE
hosts: ["HOST", "HOST2"]
```
7) fleet-server tests and uses the resulting output block.
18 changes: 17 additions & 1 deletion internal/pkg/agent/application/fleet_server_bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,13 @@ func FleetServerComponentModifier(serverCfg *configuration.FleetServerConfig) co
} else {
for j, unit := range comp.Units {
if unit.Type == client.UnitTypeOutput && unit.Config.Type == elasticsearch {
unitCfgMap, err := toMapStr(unit.Config.Source.AsMap(), &serverCfg.Output.Elasticsearch)
unitCfgMap, err := toMapStr(unit.Config.Source.AsMap())
if err != nil {
return nil, err
}
if err := addBootstrapCfg(unitCfgMap, &serverCfg.Output.Elasticsearch); err != nil {
return nil, err
}
fixOutputMap(unitCfgMap)
unitCfg, err := component.ExpectedConfig(unitCfgMap)
if err != nil {
Expand Down Expand Up @@ -100,6 +103,19 @@ func FleetServerComponentModifier(serverCfg *configuration.FleetServerConfig) co
}
}

// addBootrapCfg will transform the passed configuration.Elasticsearch to a map and add it to dst under the bootstrap key.
func addBootstrapCfg(dst map[string]interface{}, es *configuration.Elasticsearch) error {
if es == nil {
return fmt.Errorf("fleet-server bootstrap output config is undefined")
}
mp, err := toMapStr(es)
if err != nil {
return err
}
dst["bootstrap"] = mp
return nil
}

// InjectFleetConfigComponentModifier The modifier that injects the fleet configuration for the components
// that need to be able to connect to fleet server.
func InjectFleetConfigComponentModifier(fleetCfg *configuration.FleetAgentConfig, agentInfo info.Agent) coordinator.ComponentsModifier {
Expand Down
81 changes: 81 additions & 0 deletions internal/pkg/agent/application/fleet_server_bootstrap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,87 @@ func TestFleetServerComponentModifier_NoServerConfig(t *testing.T) {
}
}

func TestFleetServerComponentModifier(t *testing.T) {
tests := []struct {
name string
source map[string]interface{}
expect map[string]interface{}
}{{
name: "empty output component",
source: map[string]interface{}{},
expect: map[string]interface{}{
"bootstrap": map[string]interface{}{
"protocol": "https",
"hosts": []interface{}{"elasticsearch:9200"},
"service_token": "example-token",
},
},
}, {
name: "output component provided",
source: map[string]interface{}{
"protocol": "http",
"hosts": []interface{}{"elasticsearch:9200", "host:9200"},
},
expect: map[string]interface{}{
"protocol": "http",
"hosts": []interface{}{"elasticsearch:9200", "host:9200"},
"bootstrap": map[string]interface{}{
"protocol": "https",
"hosts": []interface{}{"elasticsearch:9200"},
"service_token": "example-token",
},
},
}}
cfg := &configuration.FleetServerConfig{
Output: configuration.FleetServerOutputConfig{
Elasticsearch: configuration.Elasticsearch{
Protocol: "https",
Hosts: []string{"elasticsearch:9200"},
ServiceToken: "example-token",
},
},
}
modifier := FleetServerComponentModifier(cfg)

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
src, err := structpb.NewStruct(tc.source)
require.NoError(t, err)
comps, err := modifier([]component.Component{{
InputSpec: &component.InputRuntimeSpec{
InputType: "fleet-server",
},
Units: []component.Unit{{
Type: client.UnitTypeOutput,
Config: &proto.UnitExpectedConfig{
Type: "elasticsearch",
Source: src,
},
}},
}}, nil)
require.NoError(t, err)

require.Len(t, comps, 1)
require.Len(t, comps[0].Units, 1)
res := comps[0].Units[0].Config.Source.AsMap()
for k, v := range tc.expect {
val, ok := res[k]
require.Truef(t, ok, "expected %q to be in output unit config", k)
if mp, ok := v.(map[string]interface{}); ok {
rMap, ok := val.(map[string]interface{})
require.Truef(t, ok, "expected %q to be map[string]interface{} was %T", k, val)
for kk, vv := range mp {
assert.Contains(t, rMap, kk)
assert.Equal(t, rMap[kk], vv)
}
} else {
assert.Equal(t, v, val)
}
}
})
}
}

func TestInjectFleetConfigComponentModifier(t *testing.T) {
fleetConfig := &configuration.FleetAgentConfig{
Enabled: true,
Expand Down
9 changes: 8 additions & 1 deletion internal/pkg/agent/configuration/fleet_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,14 @@ type FleetServerOutputConfig struct {
Elasticsearch Elasticsearch `config:"elasticsearch" yaml:"elasticsearch"`
}

// Elasticsearch is the configuration for elasticsearch.
// Elasticsearch is the configuration for fleet-server's connection to elasticsearch.
// Note that these keys may be injected into policy output by fleet-server.
// The following TLS options may be set in bootstrap:
// - VerificationMode
// - CAs
// - CATrustedFingerprint
// - CertificateConfig.Certificate AND CertificateConfig.Key
// If an attribute is added to this struct, or another TLS attribute is passed ensure that it is handled as part of the bootstrap config handler in fleet-server/internal/pkg/server/agent.go
type Elasticsearch struct {
Protocol string `config:"protocol" yaml:"protocol"`
Hosts []string `config:"hosts" yaml:"hosts"`
Expand Down
27 changes: 10 additions & 17 deletions internal/pkg/otel/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,19 @@ func newSettings(version string, configPaths []string) (*otelcol.CollectorSettin
Description: buildDescription,
Version: version,
}
converterSet := confmap.ConverterSettings{}
configProviderSettings := otelcol.ConfigProviderSettings{
ResolverSettings: confmap.ResolverSettings{
URIs: configPaths,
Providers: makeMapProvidersMap(
fileprovider.NewWithSettings(confmap.ProviderSettings{}),
envprovider.NewWithSettings(confmap.ProviderSettings{}),
yamlprovider.NewWithSettings(confmap.ProviderSettings{}),
httpprovider.NewWithSettings(confmap.ProviderSettings{}),
httpsprovider.NewWithSettings(confmap.ProviderSettings{}),
),
Converters: []confmap.Converter{expandconverter.New(converterSet)},
ProviderFactories: []confmap.ProviderFactory{
fileprovider.NewFactory(),
envprovider.NewFactory(),
yamlprovider.NewFactory(),
httpprovider.NewFactory(),
httpsprovider.NewFactory(),
},
ConverterFactories: []confmap.ConverterFactory{
expandconverter.NewFactory(),
},
},
}
provider, err := otelcol.NewConfigProvider(configProviderSettings)
Expand All @@ -81,11 +82,3 @@ func newSettings(version string, configPaths []string) (*otelcol.CollectorSettin
DisableGracefulShutdown: true,
}, nil
}

func makeMapProvidersMap(providers ...confmap.Provider) map[string]confmap.Provider {
ret := make(map[string]confmap.Provider, len(providers))
for _, provider := range providers {
ret[provider.Scheme()] = provider
}
return ret
}
Loading

0 comments on commit 6851f3c

Please sign in to comment.