From a20930df7562e1060fcdc41ac83fddb7a7077f2f Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Tue, 29 Aug 2023 10:08:30 -0700 Subject: [PATCH 1/6] fix defaults in fixture --- pkg/testing/fixture.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/testing/fixture.go b/pkg/testing/fixture.go index bad2c84c9fe..bf03a977ba1 100644 --- a/pkg/testing/fixture.go +++ b/pkg/testing/fixture.go @@ -146,6 +146,8 @@ func NewFixture(t *testing.T, version string, opts ...FixtureOpt) (*Fixture, err operatingSystem: runtime.GOOS, architecture: runtime.GOARCH, connectTimout: 5 * time.Second, + // default to elastic-agent, can be changed by a set FixtureOpt below + binaryName: "elastic-agent", } for _, o := range opts { o(f) From 7d5da83fc73b20fb44570dbe3e8dc484ba61e612 Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Tue, 29 Aug 2023 14:16:30 -0700 Subject: [PATCH 2/6] check binary name in test setup --- testing/integration/beat_serverless_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/integration/beat_serverless_test.go b/testing/integration/beat_serverless_test.go index 4883316906a..ab46d689452 100644 --- a/testing/integration/beat_serverless_test.go +++ b/testing/integration/beat_serverless_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// //go:build integration +//go:build integration package integration @@ -62,6 +62,9 @@ func (runner *BeatRunner) SetupSuite() { if runner.testbeatName == "" { runner.T().Fatalf("TEST_BINARY_NAME must be set") } + if runner.testbeatName == "elastic-agent" { + runner.T().Skipf("tests must be run against a beat, not elastic-agent") + } runner.T().Logf("running serverless tests with %s", runner.testbeatName) agentFixture, err := define.NewFixtureWithBinary(runner.T(), define.Version(), runner.testbeatName, "/home/ubuntu", atesting.WithRunLength(time.Minute), atesting.WithAdditionalArgs([]string{"-E", "output.elasticsearch.allow_older_versions=true"})) From 85b4ba5ac10a72c4ed03b26a7a9b84219bde8df0 Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Wed, 6 Sep 2023 07:43:40 -0700 Subject: [PATCH 3/6] allow ilm override in tests --- testing/integration/beat_serverless_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/integration/beat_serverless_test.go b/testing/integration/beat_serverless_test.go index ab46d689452..c258632d124 100644 --- a/testing/integration/beat_serverless_test.go +++ b/testing/integration/beat_serverless_test.go @@ -254,7 +254,7 @@ func (runner *BeatRunner) TestIndexManagementILMEnabledFail() { runner.agentFixture.WorkDir(), "setup", "--index-management", - "-E", "setup.ilm.enabled=true"}) + "-E", "setup.ilm.enabled=true", "-E", "setup.ilm.overwrite=true"}) runner.T().Logf("got response from management setup: %s", string(resp)) assert.Error(runner.T(), err) assert.Contains(runner.T(), string(resp), "not supported") @@ -274,7 +274,7 @@ func (runner *BeatRunner) TestExportILMFail() { resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", runner.agentFixture.WorkDir(), - "export", "ilm-policy"}) + "export", "ilm-policy", "-E", "setup.ilm.overwrite=true"}) runner.T().Logf("got response from management setup: %s", string(resp)) assert.Error(runner.T(), err) assert.Contains(runner.T(), string(resp), "not supported") From 0051120dc6e8144093012e5893a083ca552a2bac Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Wed, 6 Sep 2023 15:07:55 -0700 Subject: [PATCH 4/6] fix filebeat tests, add cleanup --- pkg/testing/tools/elasticsearch.go | 38 +++++++++++++++ pkg/testing/tools/kibana.go | 14 ++++++ testing/integration/beat_serverless_test.go | 52 +++++++++++++++++++-- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/pkg/testing/tools/elasticsearch.go b/pkg/testing/tools/elasticsearch.go index 5ddae293abd..1cdb197ee35 100644 --- a/pkg/testing/tools/elasticsearch.go +++ b/pkg/testing/tools/elasticsearch.go @@ -180,6 +180,30 @@ func GetIndexTemplatesForPattern(ctx context.Context, client elastictransport.In return parsed, nil } +// DeleteIndexTemplatesDataStreams deletes any data streams, then associcated index templates. +func DeleteIndexTemplatesDataStreams(ctx context.Context, client elastictransport.Interface, name string) error { + req := esapi.IndicesDeleteDataStreamRequest{Name: []string{name}, ExpandWildcards: "all,hidden"} + resp, err := req.Do(ctx, client) + if err != nil { + return fmt.Errorf("error deleting data streams: %w", err) + } + _, err = handleResponseRaw(resp) + if err != nil { + return fmt.Errorf("error handling HTTP response for data stream delete: %w", err) + } + + patternReq := esapi.IndicesDeleteIndexTemplateRequest{Name: name} + resp, err = patternReq.Do(ctx, client) + if err != nil { + return fmt.Errorf("error deleting index templates: %w", err) + } + _, err = handleResponseRaw(resp) + if err != nil { + return fmt.Errorf("error handling HTTP response for index template delete: %w", err) + } + return nil +} + // GetPipelines returns a list of installed pipelines that match the given name/pattern func GetPipelines(ctx context.Context, client elastictransport.Interface, name string) (map[string]Pipeline, error) { req := esapi.IngestGetPipelineRequest{PipelineID: name} @@ -200,6 +224,20 @@ func GetPipelines(ctx context.Context, client elastictransport.Interface, name s return parsed, nil } +// DeletePipelines deletes all pipelines that match the given pattern +func DeletePipelines(ctx context.Context, client elastictransport.Interface, name string) error { + req := esapi.IngestDeletePipelineRequest{PipelineID: name} + resp, err := req.Do(ctx, client) + if err != nil { + return fmt.Errorf("error deleting index template") + } + _, err = handleResponseRaw(resp) + if err != nil { + return fmt.Errorf("error handling HTTP response: %w", err) + } + return nil +} + // FindMatchingLogLinesWithContext returns any logs with message fields that match the given line func FindMatchingLogLinesWithContext(ctx context.Context, client elastictransport.Interface, namespace, line string) (Documents, error) { queryRaw := map[string]interface{}{ diff --git a/pkg/testing/tools/kibana.go b/pkg/testing/tools/kibana.go index 6c26798710a..e02e91948be 100644 --- a/pkg/testing/tools/kibana.go +++ b/pkg/testing/tools/kibana.go @@ -30,6 +30,20 @@ type Dashboard struct { Version string `json:"version"` } +// DeleteDashboard removes the selected dashboard +func DeleteDashboard(ctx context.Context, client *kibana.Client, id string) error { + status, resp, err := client.Connection.Request("DELETE", fmt.Sprintf("/api/saved_objects/dashboard/%s", id), nil, nil, nil) + if err != nil { + return fmt.Errorf("error making API request: %w, response: '%s'", err, string(resp)) + } + + if status != 200 { + return fmt.Errorf("non-200 return code: %v, response: '%s'", status, string(resp)) + } + return nil + +} + // GetDashboards returns a list of known dashboards on the system func GetDashboards(ctx context.Context, client *kibana.Client) ([]Dashboard, error) { params := url.Values{} diff --git a/testing/integration/beat_serverless_test.go b/testing/integration/beat_serverless_test.go index c258632d124..7af021d98a9 100644 --- a/testing/integration/beat_serverless_test.go +++ b/testing/integration/beat_serverless_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build integration +// //go:build integration package integration @@ -65,6 +65,10 @@ func (runner *BeatRunner) SetupSuite() { if runner.testbeatName == "elastic-agent" { runner.T().Skipf("tests must be run against a beat, not elastic-agent") } + + if runner.testbeatName != "filebeat" && runner.testbeatName != "metricbeat" { + runner.T().Skip("test only supports metricbeat or filebeat") + } runner.T().Logf("running serverless tests with %s", runner.testbeatName) agentFixture, err := define.NewFixtureWithBinary(runner.T(), define.Version(), runner.testbeatName, "/home/ubuntu", atesting.WithRunLength(time.Minute), atesting.WithAdditionalArgs([]string{"-E", "output.elasticsearch.allow_older_versions=true"})) @@ -80,7 +84,7 @@ func (runner *BeatRunner) SetupSuite() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - mbOutConfig := ` + beatOutConfig := ` output.elasticsearch: hosts: ["%s"] username: %s @@ -95,6 +99,27 @@ processors: fields: test-id: %s ` + if runner.testbeatName == "filebeat" { + beatOutConfig = ` +output.elasticsearch: + hosts: ["%s"] + username: %s + password: %s +setup.kibana: + host: %s +filebeat.config.modules: + - modules: system + syslog: + enabled: true + auth: + enabled: true +processors: + - add_fields: + target: host + fields: + test-id: %s +` + } // beats likes to add standard ports to URLs that don't have them, and ESS will sometimes return a URL without a port, assuming :443 // so try to fix that here @@ -117,14 +142,13 @@ processors: testUuid, err := uuid.NewV4() require.NoError(runner.T(), err) runner.testUuid = testUuid.String() - parsedCfg := fmt.Sprintf(mbOutConfig, fixedESHost, runner.user, runner.pass, fixedKibanaHost, testUuid.String()) + parsedCfg := fmt.Sprintf(beatOutConfig, fixedESHost, runner.user, runner.pass, fixedKibanaHost, testUuid.String()) err = runner.agentFixture.WriteFileToWorkDir(ctx, parsedCfg, fmt.Sprintf("%s.yml", runner.testbeatName)) require.NoError(runner.T(), err) } // run the beat with default metricsets, ensure no errors in logs + data is ingested func (runner *BeatRunner) TestRunAndCheckData() { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*4) defer cancel() err := runner.agentFixture.Run(ctx) @@ -164,6 +188,15 @@ func (runner *BeatRunner) TestSetupDashboards() { require.True(runner.T(), found, fmt.Sprintf("could not find dashboard newer than 5 minutes, out of %d dashboards", len(dashList))) runner.Run("export dashboards", runner.SubtestExportDashboards) + + // cleanup + for _, dash := range dashList { + err = tools.DeleteDashboard(ctx, runner.requirementsInfo.KibanaClient, dash.ID) + if err != nil { + runner.T().Logf("WARNING: could not delete dashboards after test: %s", err) + break + } + } } // tests the [beat] export dashboard command @@ -211,6 +244,11 @@ func (runner *BeatRunner) TestSetupPipelines() { require.NoError(runner.T(), err) require.NotEmpty(runner.T(), pipelines) + /// cleanup + err = tools.DeletePipelines(ctx, runner.requirementsInfo.ESClient, "*filebeat*") + if err != nil { + runner.T().Logf("WARNING: could not clean up pipelines: %s", err) + } } // test beat setup --index-management with ILM disabled @@ -235,6 +273,12 @@ func (runner *BeatRunner) TestIndexManagementNoILM() { runner.Run("export templates", runner.SubtestExportTemplates) runner.Run("export index patterns", runner.SubtestExportIndexPatterns) + + // cleanup + err = tools.DeleteIndexTemplatesDataStreams(ctx, runner.requirementsInfo.ESClient, fmt.Sprintf("*%s*", runner.testbeatName)) + if err != nil { + runner.T().Logf("WARNING: could not clean up index templates/data streams: %s", err) + } } // tests beat setup --index-management with ILM explicitly set From fd88bf25b47ebe7e8b266ebee3e8bf94a750dc60 Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Mon, 11 Sep 2023 14:27:31 -0700 Subject: [PATCH 5/6] tinker with dashboards --- pkg/testing/tools/kibana.go | 9 +++++++-- testing/integration/beat_serverless_test.go | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/testing/tools/kibana.go b/pkg/testing/tools/kibana.go index e02e91948be..df88a3d2691 100644 --- a/pkg/testing/tools/kibana.go +++ b/pkg/testing/tools/kibana.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "net/url" "time" @@ -32,7 +33,9 @@ type Dashboard struct { // DeleteDashboard removes the selected dashboard func DeleteDashboard(ctx context.Context, client *kibana.Client, id string) error { - status, resp, err := client.Connection.Request("DELETE", fmt.Sprintf("/api/saved_objects/dashboard/%s", id), nil, nil, nil) + headers := http.Header{} + headers.Add("x-elastic-internal-origin", "integration-tests") + status, resp, err := client.Connection.Request("DELETE", fmt.Sprintf("/api/saved_objects/dashboard/%s", id), nil, headers, nil) if err != nil { return fmt.Errorf("error making API request: %w, response: '%s'", err, string(resp)) } @@ -53,7 +56,9 @@ func GetDashboards(ctx context.Context, client *kibana.Client) ([]Dashboard, err dashboards := []Dashboard{} page := 1 for { - status, resp, err := client.Connection.Request("GET", "/api/saved_objects/_find", params, nil, nil) + headers := http.Header{} + headers.Add("x-elastic-internal-origin", "integration-tests") + status, resp, err := client.Connection.Request("GET", "/api/saved_objects/_find", params, headers, nil) if err != nil { return nil, fmt.Errorf("error making api request: %w", err) } diff --git a/testing/integration/beat_serverless_test.go b/testing/integration/beat_serverless_test.go index 7af021d98a9..a2460685fc4 100644 --- a/testing/integration/beat_serverless_test.go +++ b/testing/integration/beat_serverless_test.go @@ -209,10 +209,12 @@ func (runner *BeatRunner) SubtestExportDashboards() { require.NoError(runner.T(), err) require.NotEmpty(runner.T(), dashlist) - _, err = runner.agentFixture.Exec(ctx, []string{"--path.home", + exportOut, err := runner.agentFixture.Exec(ctx, []string{"--path.home", runner.agentFixture.WorkDir(), "export", "dashboard", "--folder", outDir, "--id", dashlist[0].ID}) + + runner.T().Logf("got output: %s", exportOut) assert.NoError(runner.T(), err) inFolder, err := os.ReadDir(filepath.Join(outDir, "/_meta/kibana/8/dashboard")) From ec9c8c590639adacf6f47096d11371a8da6243cb Mon Sep 17 00:00:00 2001 From: fearful-symmetry Date: Wed, 13 Sep 2023 10:53:09 -0700 Subject: [PATCH 6/6] fix ilm tests --- pkg/testing/ess/serverless_provision.go | 3 ++- pkg/testing/tools/kibana.go | 1 + testing/integration/beat_serverless_test.go | 28 +++++++++++++-------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pkg/testing/ess/serverless_provision.go b/pkg/testing/ess/serverless_provision.go index ea78fc0eab1..553dd238eb2 100644 --- a/pkg/testing/ess/serverless_provision.go +++ b/pkg/testing/ess/serverless_provision.go @@ -80,7 +80,7 @@ func (prov *ServerlessProvision) Provision(ctx context.Context, requests []runne for _, req := range requests { client := NewServerlessClient(prov.cfg.Region, "observability", prov.cfg.APIKey, prov.log) srvReq := ServerlessRequest{Name: req.ID, RegionID: prov.cfg.Region} - _, err := client.DeployStack(ctx, srvReq) + proj, err := client.DeployStack(ctx, srvReq) if err != nil { return nil, fmt.Errorf("error deploying stack for request %s: %w", req.ID, err) } @@ -95,6 +95,7 @@ func (prov *ServerlessProvision) Provision(ctx context.Context, requests []runne Kibana: client.proj.Endpoints.Kibana, Username: client.proj.Credentials.Username, Password: client.proj.Credentials.Password, + Internal: map[string]interface{}{"ID": proj.ID}, } stacks = append(stacks, newStack) prov.stacksMut.Lock() diff --git a/pkg/testing/tools/kibana.go b/pkg/testing/tools/kibana.go index df88a3d2691..ed9459307b6 100644 --- a/pkg/testing/tools/kibana.go +++ b/pkg/testing/tools/kibana.go @@ -33,6 +33,7 @@ type Dashboard struct { // DeleteDashboard removes the selected dashboard func DeleteDashboard(ctx context.Context, client *kibana.Client, id string) error { + // In the future there should be logic to check if we need this header, waiting for https://github.com/elastic/kibana/pull/164850 headers := http.Header{} headers.Add("x-elastic-internal-origin", "integration-tests") status, resp, err := client.Connection.Request("DELETE", fmt.Sprintf("/api/saved_objects/dashboard/%s", id), nil, headers, nil) diff --git a/testing/integration/beat_serverless_test.go b/testing/integration/beat_serverless_test.go index a2460685fc4..11c4f1b7a0f 100644 --- a/testing/integration/beat_serverless_test.go +++ b/testing/integration/beat_serverless_test.go @@ -262,9 +262,11 @@ func (runner *BeatRunner) TestIndexManagementNoILM() { runner.agentFixture.WorkDir(), "setup", "--index-management", - "-E", "setup.ilm.enabled=false"}) + "--E=setup.ilm.enabled=false"}) runner.T().Logf("got response from management setup: %s", string(resp)) assert.NoError(runner.T(), err) + // we should not print a warning if we've explicitly disabled ILM + assert.NotContains(runner.T(), string(resp), "not supported") tmpls, err := tools.GetIndexTemplatesForPattern(ctx, runner.requirementsInfo.ESClient, fmt.Sprintf("*%s*", runner.testbeatName)) require.NoError(runner.T(), err) @@ -286,7 +288,7 @@ func (runner *BeatRunner) TestIndexManagementNoILM() { // tests beat setup --index-management with ILM explicitly set // On serverless, this should fail. // Will not pass right now, may need to change -func (runner *BeatRunner) TestIndexManagementILMEnabledFail() { +func (runner *BeatRunner) TestIndexManagementILMEnabledWarning() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() info, err := tools.GetPing(ctx, runner.requirementsInfo.ESClient) @@ -300,15 +302,16 @@ func (runner *BeatRunner) TestIndexManagementILMEnabledFail() { runner.agentFixture.WorkDir(), "setup", "--index-management", - "-E", "setup.ilm.enabled=true", "-E", "setup.ilm.overwrite=true"}) + "--E=setup.ilm.enabled=true", "--E=setup.ilm.overwrite=true"}) runner.T().Logf("got response from management setup: %s", string(resp)) - assert.Error(runner.T(), err) + require.NoError(runner.T(), err) assert.Contains(runner.T(), string(resp), "not supported") } // tests beat setup ilm-policy -// On serverless, this should fail -func (runner *BeatRunner) TestExportILMFail() { +// the export command doesn't actually make a network connection, +// so this won't fail +func (runner *BeatRunner) TestExport() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() info, err := tools.GetPing(ctx, runner.requirementsInfo.ESClient) @@ -320,10 +323,15 @@ func (runner *BeatRunner) TestExportILMFail() { resp, err := runner.agentFixture.Exec(ctx, []string{"--path.home", runner.agentFixture.WorkDir(), - "export", "ilm-policy", "-E", "setup.ilm.overwrite=true"}) - runner.T().Logf("got response from management setup: %s", string(resp)) - assert.Error(runner.T(), err) - assert.Contains(runner.T(), string(resp), "not supported") + "export", "ilm-policy", "--E=setup.ilm.overwrite=true"}) + runner.T().Logf("got response from export: %s", string(resp)) + assert.NoError(runner.T(), err) + // check to see if we got a valid output + policy := map[string]interface{}{} + err = json.Unmarshal(resp, &policy) + require.NoError(runner.T(), err) + + require.NotEmpty(runner.T(), policy["policy"]) }