From a2ab85dac7cc4efc4a52fe0265a022a8a14a3281 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:20:16 +0200 Subject: [PATCH 1/5] fix: avoid gotestsum import in libbeat es output (#39835) gotestsum is being imported causing downstream apps to include it in the dependency tree. Drop the import and use the client logger. --- libbeat/outputs/elasticsearch/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index e05c4e0b261f..933d04c789ca 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -26,7 +26,6 @@ import ( "time" "go.elastic.co/apm/v2" - "gotest.tools/gotestsum/log" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" @@ -433,7 +432,7 @@ func (client *Client) bulkCollectPublishFails(bulkResult bulkResult) ([]publishe if client.applyItemStatus(events[i], itemStatus, itemMessage, &stats) { eventsToRetry = append(eventsToRetry, events[i]) - log.Debugf("Bulk item insert failed (i=%v, status=%v): %s", i, itemStatus, itemMessage) + client.log.Debugf("Bulk item insert failed (i=%v, status=%v): %s", i, itemStatus, itemMessage) } } From db9406b50514b8896f08e7bbb176f09c07650566 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:20:31 +0200 Subject: [PATCH 2/5] refactor: replace x/exp/slices with stdlib slices (#39838) Go 1.21 added slices package to stdlib. Replace usage of x/exp/slices with slices and drop x/exp dependency. --- NOTICE.txt | 74 +++++++++++------------ go.mod | 2 +- x-pack/filebeat/input/salesforce/input.go | 2 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 0c501bc8d854..c7bfc6b351bb 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -25187,43 +25187,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : golang.org/x/exp -Version: v0.0.0-20231127185646-65229373498e -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20231127185646-65229373498e/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : golang.org/x/lint Version: v0.0.0-20210508222113-6edffad5e616 @@ -55098,6 +55061,43 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/exp +Version: v0.0.0-20231127185646-65229373498e +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20231127185646-65229373498e/LICENSE: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : golang.org/x/term Version: v0.19.0 diff --git a/go.mod b/go.mod index 9552997c4c30..bf3ff2ad3124 100644 --- a/go.mod +++ b/go.mod @@ -232,7 +232,6 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.6.0 go.elastic.co/apm/v2 v2.6.0 go.mongodb.org/mongo-driver v1.5.1 - golang.org/x/exp v0.0.0-20231127185646-65229373498e golang.org/x/tools/go/vcs v0.1.0-deprecated google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -388,6 +387,7 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect golang.org/x/term v0.19.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/x-pack/filebeat/input/salesforce/input.go b/x-pack/filebeat/input/salesforce/input.go index f2f8ef15c683..12ed4e652a7f 100644 --- a/x-pack/filebeat/input/salesforce/input.go +++ b/x-pack/filebeat/input/salesforce/input.go @@ -14,6 +14,7 @@ import ( "io" "net/http" "os" + "slices" "time" "github.com/g8rswimmer/go-sfdc" @@ -23,7 +24,6 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/hashicorp/go-retryablehttp" "go.uber.org/zap" - "golang.org/x/exp/slices" v2 "github.com/elastic/beats/v7/filebeat/input/v2" inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" From c5b0cec5dd9bbb2d51a13007ab9ee8fdf9aa06ff Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 11 Jun 2024 06:28:39 +0930 Subject: [PATCH 3/5] x-pack/filebeat/input/entityanalytics/provider/{azuread,okta}: add request tracing support (#39821) --- CHANGELOG.next.asciidoc | 1 + .../inputs/input-entity-analytics.asciidoc | 24 +++++++++ .../entityanalytics/provider/azuread/azure.go | 5 +- .../provider/azuread/fetcher/graph/.gitignore | 1 + .../provider/azuread/fetcher/graph/graph.go | 52 ++++++++++++++++++- .../azuread/fetcher/graph/graph_test.go | 25 +++++++-- .../entityanalytics/provider/okta/.gitignore | 1 + .../entityanalytics/provider/okta/conf.go | 20 ++++++- .../entityanalytics/provider/okta/okta.go | 52 +++++++++++++++++-- .../provider/okta/okta_test.go | 14 +++++ 10 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/.gitignore create mode 100644 x-pack/filebeat/input/entityanalytics/provider/okta/.gitignore diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 49b0cf9b2a39..e1214aa0e27a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -279,6 +279,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Update CEL mito extensions to v1.12.2. {pull}39755[39755] - Add support for base64-encoded HMAC headers to HTTP Endpoint. {pull}39655[39655] - Add user group membership support to Okta entity analytics provider. {issue}39814[39814] {pull}39815[39815] +- Add request trace support for Okta and EntraID entity analytics providers. {pull}39821[39821] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc index e099bf3d247c..41c015761231 100644 --- a/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc @@ -509,6 +509,18 @@ This is a list of optional query parameters. The default is `["accountEnabled", "displayName", "operatingSystem", "operatingSystemVersion", "physicalIds", "extensionAttributes", "alternativeSecurityIds"]`. +[float] +==== `tracer.filename` + +It is possible to log HTTP requests and responses to the EntraID API to a local file-system for debugging configurations. +This option is enabled by setting the `tracer.filename` value. Additional options are available to +tune log rotation behavior. + +To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the filename and will be replaced with the input instance id. +For Example, `http-request-trace-*.ndjson`. + +Enabling this option compromises security and should only be used for debugging. + [id="provider-okta"] ==== Okta User Identities (`okta`) @@ -797,6 +809,18 @@ The interval in which incremental updates should occur. The interval must be shorter than the full synchronization interval (`sync_interval`). Expressed as a duration string (e.g., 1m, 3h, 24h). Defaults to `15m` (15 minutes). +[float] +==== `tracer.filename` + +It is possible to log HTTP requests and responses to the Okta API to a local file-system for debugging configurations. +This option is enabled by setting the `tracer.filename` value. Additional options are available to +tune log rotation behavior. + +To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the filename and will be replaced with the input instance id. +For Example, `http-request-trace-*.ndjson`. + +Enabling this option compromises security and should only be used for debugging. + [float] ==== Metrics diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/azure.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/azure.go index 30514352ebaf..d67031753fd5 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/azure.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/azure.go @@ -48,6 +48,8 @@ type azure struct { logger *logp.Logger auth authenticator.Authenticator fetcher fetcher.Fetcher + + ctx v2.Context } // Name returns the name of this provider. @@ -71,6 +73,7 @@ func (p *azure) Test(testCtx v2.TestContext) error { // Run will start data collection on this provider. func (p *azure) Run(inputCtx v2.Context, store *kvstore.Store, client beat.Client) error { p.logger = inputCtx.Logger.With("tenant_id", p.conf.TenantID, "provider", Name) + p.ctx = inputCtx p.auth.SetLogger(p.logger) p.fetcher.SetLogger(p.logger) p.metrics = newMetrics(inputCtx.ID, nil) @@ -575,7 +578,7 @@ func (p *azure) configure(cfg *config.C) (kvstore.Input, error) { if p.auth, err = oauth2.New(cfg, p.Manager.Logger); err != nil { return nil, fmt.Errorf("unable to create authenticator: %w", err) } - if p.fetcher, err = graph.New(cfg, p.Manager.Logger, p.auth); err != nil { + if p.fetcher, err = graph.New(ctxtool.FromCanceller(p.ctx.Cancelation), p.ctx.ID, cfg, p.Manager.Logger, p.auth); err != nil { return nil, fmt.Errorf("unable to create fetcher: %w", err) } diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/.gitignore b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/.gitignore new file mode 100644 index 000000000000..13df6a73f0dd --- /dev/null +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/.gitignore @@ -0,0 +1 @@ +*.ndjson diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go index 558e277d1062..a3104ce0d009 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go @@ -15,13 +15,19 @@ import ( "io" "net/http" "net/url" + "path/filepath" "strings" "github.com/google/uuid" + "go.elastic.co/ecszap" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/internal/collections" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider/azuread/authenticator" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher" + "github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httplog" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -104,6 +110,9 @@ type graphConf struct { Select selection `config:"select"` Transport httpcommon.HTTPTransportSettings `config:",inline"` + + // Tracer allows configuration of request trace logging. + Tracer *lumberjack.Logger `config:"tracer"` } type selection struct { @@ -329,16 +338,22 @@ func (f *graph) doRequest(ctx context.Context, method, url string, body io.Reade } // New creates a new instance of the graph fetcher. -func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) (fetcher.Fetcher, error) { +func New(ctx context.Context, id string, cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) (fetcher.Fetcher, error) { var c graphConf if err := cfg.Unpack(&c); err != nil { return nil, fmt.Errorf("unable to unpack Graph API Fetcher config: %w", err) } + if c.Tracer != nil { + id = sanitizeFileName(id) + c.Tracer.Filename = strings.ReplaceAll(c.Tracer.Filename, "*", id) + } + client, err := c.Transport.Client() if err != nil { return nil, fmt.Errorf("unable to create HTTP client: %w", err) } + client = requestTrace(ctx, client, c, logger) f := graph{ conf: c, @@ -383,6 +398,41 @@ func New(cfg *config.C, logger *logp.Logger, auth authenticator.Authenticator) ( return &f, nil } +// requestTrace decorates cli with an httplog.LoggingRoundTripper if cfg.Tracer +// is non-nil. +func requestTrace(ctx context.Context, cli *http.Client, cfg graphConf, log *logp.Logger) *http.Client { + if cfg.Tracer == nil { + return cli + } + w := zapcore.AddSync(cfg.Tracer) + go func() { + // Close the logger when we are done. + <-ctx.Done() + cfg.Tracer.Close() + }() + core := ecszap.NewCore( + ecszap.NewDefaultEncoderConfig(), + w, + zap.DebugLevel, + ) + traceLogger := zap.New(core) + + const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details. + maxSize := cfg.Tracer.MaxSize * 1e6 + cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, max(0, maxSize-margin), log) + return cli +} + +// sanitizeFileName returns name with ":" and "/" replaced with "_", removing +// repeated instances. The request.tracer.filename may have ":" when an input +// has cursor config and the macOS Finder will treat this as path-separator and +// causes to show up strange filepaths. +func sanitizeFileName(name string) string { + name = strings.ReplaceAll(name, ":", string(filepath.Separator)) + name = filepath.Clean(name) + return strings.ReplaceAll(name, string(filepath.Separator), "_") +} + func formatQuery(name string, query []string, dflt string) string { q := dflt if len(query) != 0 { diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go index f439cc916797..f2fc2effe29e 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph_test.go @@ -7,6 +7,7 @@ package graph import ( "context" "encoding/json" + "flag" "fmt" "net/http" "net/http/httptest" @@ -19,6 +20,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/stretchr/testify/require" + "gopkg.in/natefinch/lumberjack.v2" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/internal/collections" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider/azuread/authenticator/mock" @@ -27,6 +29,8 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) +var trace = flag.Bool("request_trace", false, "enable request tracing during tests") + var usersResponse1 = apiUserResponse{ Users: []userAPI{ { @@ -313,11 +317,16 @@ func TestGraph_Groups(t *testing.T) { rawConf := graphConf{ APIEndpoint: "http://" + testSrv.addr, } + if *trace { + rawConf.Tracer = &lumberjack.Logger{ + Filename: "test_trace-*.ndjson", + } + } c, err := config.NewConfigFrom(&rawConf) require.NoError(t, err) auth := mock.New(mock.DefaultTokenValue) - f, err := New(c, logp.L(), auth) + f, err := New(context.Background(), t.Name(), c, logp.L(), auth) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -372,11 +381,16 @@ func TestGraph_Users(t *testing.T) { rawConf := graphConf{ APIEndpoint: "http://" + testSrv.addr, } + if *trace { + rawConf.Tracer = &lumberjack.Logger{ + Filename: "test_trace-*.ndjson", + } + } c, err := config.NewConfigFrom(&rawConf) require.NoError(t, err) auth := mock.New(mock.DefaultTokenValue) - f, err := New(c, logp.L(), auth) + f, err := New(context.Background(), t.Name(), c, logp.L(), auth) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -477,11 +491,16 @@ func TestGraph_Devices(t *testing.T) { APIEndpoint: "http://" + testSrv.addr, Select: test.selection, } + if *trace { + rawConf.Tracer = &lumberjack.Logger{ + Filename: "test_trace-*.ndjson", + } + } c, err := config.NewConfigFrom(&rawConf) require.NoError(t, err) auth := mock.New(mock.DefaultTokenValue) - f, err := New(c, logp.L(), auth) + f, err := New(context.Background(), t.Name(), c, logp.L(), auth) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/.gitignore b/x-pack/filebeat/input/entityanalytics/provider/okta/.gitignore new file mode 100644 index 000000000000..13df6a73f0dd --- /dev/null +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/.gitignore @@ -0,0 +1 @@ +*.ndjson diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go index eb0906e78d54..873a6195d472 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/conf.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "gopkg.in/natefinch/lumberjack.v2" + "github.com/elastic/elastic-agent-libs/transport/httpcommon" ) @@ -62,6 +64,9 @@ type conf struct { // Request is the configuration for establishing // HTTP requests to the API. Request *requestConfig `config:"request"` + + // Tracer allows configuration of request trace logging. + Tracer *lumberjack.Logger `config:"tracer"` } type requestConfig struct { @@ -163,10 +168,23 @@ func (c *conf) Validate() error { } switch strings.ToLower(c.Dataset) { case "", "all", "users", "devices": - return nil default: return errors.New("dataset must be 'all', 'users', 'devices' or empty") } + + if c.Tracer == nil { + return nil + } + if c.Tracer.Filename == "" { + return errors.New("request tracer must have a filename if used") + } + if c.Tracer.MaxSize == 0 { + // By default Lumberjack caps file sizes at 100MB which + // is excessive for a debugging logger, so default to 1MB + // which is the minimum. + c.Tracer.MaxSize = 1 + } + return nil } func (c *conf) wantUsers() bool { diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go index 70f95d7396e8..0980575df3a0 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta.go @@ -12,10 +12,14 @@ import ( "io" "net/http" "net/url" + "path/filepath" + "strings" "time" "github.com/hashicorp/go-retryablehttp" + "go.elastic.co/ecszap" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "golang.org/x/time/rate" v2 "github.com/elastic/beats/v7/filebeat/input/v2" @@ -23,6 +27,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/internal/kvstore" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta" + "github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httplog" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -105,8 +110,13 @@ func (p *oktaInput) Run(inputCtx v2.Context, store *kvstore.Store, client beat.C // Allow a single fetch operation to obtain limits from the API. p.lim = rate.NewLimiter(1, 1) + if p.cfg.Tracer != nil { + id := sanitizeFileName(inputCtx.ID) + p.cfg.Tracer.Filename = strings.ReplaceAll(p.cfg.Tracer.Filename, "*", id) + } + var err error - p.client, err = newClient(p.cfg, p.logger) + p.client, err = newClient(ctxtool.FromCanceller(inputCtx.Cancelation), p.cfg, p.logger) if err != nil { return err } @@ -152,12 +162,14 @@ func (p *oktaInput) Run(inputCtx v2.Context, store *kvstore.Store, client beat.C } } -func newClient(cfg conf, log *logp.Logger) (*http.Client, error) { +func newClient(ctx context.Context, cfg conf, log *logp.Logger) (*http.Client, error) { c, err := cfg.Request.Transport.Client(clientOptions(cfg.Request.KeepAlive.settings())...) if err != nil { return nil, err } + c = requestTrace(ctx, c, cfg, log) + c.CheckRedirect = checkRedirect(cfg.Request, log) client := &retryablehttp.Client{ @@ -169,10 +181,44 @@ func newClient(cfg conf, log *logp.Logger) (*http.Client, error) { CheckRetry: retryablehttp.DefaultRetryPolicy, Backoff: retryablehttp.DefaultBackoff, } - return client.StandardClient(), nil } +// requestTrace decorates cli with an httplog.LoggingRoundTripper if cfg.Tracer +// is non-nil. +func requestTrace(ctx context.Context, cli *http.Client, cfg conf, log *logp.Logger) *http.Client { + if cfg.Tracer == nil { + return cli + } + w := zapcore.AddSync(cfg.Tracer) + go func() { + // Close the logger when we are done. + <-ctx.Done() + cfg.Tracer.Close() + }() + core := ecszap.NewCore( + ecszap.NewDefaultEncoderConfig(), + w, + zap.DebugLevel, + ) + traceLogger := zap.New(core) + + const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details. + maxSize := cfg.Tracer.MaxSize * 1e6 + cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, max(0, maxSize-margin), log) + return cli +} + +// sanitizeFileName returns name with ":" and "/" replaced with "_", removing +// repeated instances. The request.tracer.filename may have ":" when an input +// has cursor config and the macOS Finder will treat this as path-separator and +// causes to show up strange filepaths. +func sanitizeFileName(name string) string { + name = strings.ReplaceAll(name, ":", string(filepath.Separator)) + name = filepath.Clean(name) + return strings.ReplaceAll(name, string(filepath.Separator), "_") +} + // clientOption returns constructed client configuration options, including // setting up http+unix and http+npipe transports if requested. func clientOptions(keepalive httpcommon.WithKeepaliveSettings) []httpcommon.TransportOption { diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go index da29666712b9..cf3f1423006b 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/okta_test.go @@ -7,6 +7,7 @@ package okta import ( "context" "encoding/json" + "flag" "fmt" "net/http" "net/http/httptest" @@ -17,11 +18,14 @@ import ( "time" "golang.org/x/time/rate" + "gopkg.in/natefinch/lumberjack.v2" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta" "github.com/elastic/elastic-agent-libs/logp" ) +var trace = flag.Bool("request_trace", false, "enable request tracing during tests") + func TestOktaDoFetch(t *testing.T) { tests := []struct { dataset string @@ -153,6 +157,16 @@ func TestOktaDoFetch(t *testing.T) { lim: rate.NewLimiter(1, 1), logger: logp.L(), } + if *trace { + name := test.dataset + if name == "" { + name = "default" + } + a.cfg.Tracer = &lumberjack.Logger{ + Filename: fmt.Sprintf("test_trace_%s.ndjson", name), + } + } + a.client = requestTrace(context.Background(), a.client, a.cfg, a.logger) ss, err := newStateStore(store) if err != nil { From 1fd65c7f980d69886cb894014f4e9737ce05bb71 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 11 Jun 2024 06:28:56 +0930 Subject: [PATCH 4/5] x-pack/filebeat/input/{cel,httpjson}: fix typo in minimum log length (#39834) --- CHANGELOG-developer.next.asciidoc | 1 + x-pack/filebeat/input/cel/input.go | 2 +- x-pack/filebeat/input/httpjson/input.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 58f9cea52119..995b52511b88 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -99,6 +99,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556] - Skip flakey metrics test on windows in filebeat httpjson input. {issue}39676[39676] {pull}39678[39678] - Fix flakey test on Windows 2022 in packetbeat/route. {issue}39698[39698] {pull}39822[39822] +- Fix bug in minimum length for request trace logging. {pull}39834[39834] ==== Added diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 7e8a7584c280..c70941a25a53 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -754,7 +754,7 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger, reg *monitorin ) traceLogger := zap.New(core) - const margin = 1e3 // 1OkB ought to be enough room for all the remainder of the trace details. + const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details. maxSize := cfg.Resource.Tracer.MaxSize * 1e6 trace = httplog.NewLoggingRoundTripper(c.Transport, traceLogger, max(0, maxSize-margin), log) c.Transport = trace diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 0764259619b7..cb91723f464a 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -259,7 +259,7 @@ func newNetHTTPClient(ctx context.Context, cfg *requestConfig, log *logp.Logger, ) traceLogger := zap.New(core) - const margin = 1e3 // 1OkB ought to be enough room for all the remainder of the trace details. + const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details. maxSize := cfg.Tracer.MaxSize*1e6 - margin if maxSize < 0 { maxSize = 0 From 3c9f4d952bfd20b1898cfeb59916a2239b667988 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Mon, 10 Jun 2024 23:46:23 +0200 Subject: [PATCH 5/5] refactor: replace urso/sderr with stdlib errors (#39839) * refactor: replace urso/sderr with stdlib errors Go 1.20 added multiple errors wrapping so we can migrate to stdlib errors and drop the additional dependency on github.com/urso/sderr * refactor: avoid wrapping and unwrapping * Update copytruncate_prospector.go * Update statestore.go --- NOTICE.txt | 422 +++++++++--------- .../filestream/copytruncate_prospector.go | 5 +- .../internal/input-logfile/manager.go | 4 +- filebeat/input/filestream/prospector.go | 5 +- filebeat/input/journald/input.go | 4 +- .../input/journald/pkg/journalread/reader.go | 9 +- filebeat/input/v2/input-cursor/input.go | 7 +- filebeat/input/v2/input-cursor/manager.go | 5 +- go.mod | 2 +- .../provider/azuread/statestore.go | 3 +- .../provider/okta/statestore.go | 8 +- 11 files changed, 231 insertions(+), 243 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index c7bfc6b351bb..f6881a796c09 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -23158,217 +23158,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/urso/sderr -Version: v0.0.0-20210525210834-52b04e8f5c71 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20210525210834-52b04e8f5c71/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/vmware/govmomi Version: v0.0.0-20170802214208-2cad15190b41 @@ -52541,6 +52330,217 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/diag@v0.0.0-202002 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/urso/sderr +Version: v0.0.0-20210525210834-52b04e8f5c71 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20210525210834-52b04e8f5c71/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/vishvananda/netlink Version: v1.1.0 diff --git a/filebeat/input/filestream/copytruncate_prospector.go b/filebeat/input/filestream/copytruncate_prospector.go index 5b1c6bdd4277..50ea7df25c0f 100644 --- a/filebeat/input/filestream/copytruncate_prospector.go +++ b/filebeat/input/filestream/copytruncate_prospector.go @@ -18,14 +18,13 @@ package filestream import ( + "errors" "os" "regexp" "sort" "strconv" "time" - "github.com/urso/sderr" - loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/common/file" @@ -230,7 +229,7 @@ func (p *copyTruncateFileProspector) Run(ctx input.Context, s loginp.StateMetada errs := tg.Wait() if len(errs) > 0 { - log.Error("%s", sderr.WrapAll(errs, "running prospector failed")) + log.Errorf("running prospector failed: %v", errors.Join(errs...)) } } diff --git a/filebeat/input/filestream/internal/input-logfile/manager.go b/filebeat/input/filestream/internal/input-logfile/manager.go index cfa830061175..3eb2f951036d 100644 --- a/filebeat/input/filestream/internal/input-logfile/manager.go +++ b/filebeat/input/filestream/internal/input-logfile/manager.go @@ -25,8 +25,6 @@ import ( "sync" "time" - "github.com/urso/sderr" - "github.com/elastic/go-concert/unison" v2 "github.com/elastic/beats/v7/filebeat/input/v2" @@ -141,7 +139,7 @@ func (cim *InputManager) Init(group unison.Group) error { if err != nil { store.Release() cim.shutdown() - return sderr.Wrap(err, "Can not start registry cleanup process") + return fmt.Errorf("Can not start registry cleanup process: %w", err) } return nil diff --git a/filebeat/input/filestream/prospector.go b/filebeat/input/filestream/prospector.go index 336461fede57..2bf737a86fd9 100644 --- a/filebeat/input/filestream/prospector.go +++ b/filebeat/input/filestream/prospector.go @@ -18,11 +18,10 @@ package filestream import ( + "errors" "fmt" "time" - "github.com/urso/sderr" - loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" @@ -160,7 +159,7 @@ func (p *fileProspector) Run(ctx input.Context, s loginp.StateMetadataUpdater, h errs := tg.Wait() if len(errs) > 0 { - log.Error("%s", sderr.WrapAll(errs, "running prospector failed")) + log.Errorf("running prospector failed: %v", errors.Join(errs...)) } } diff --git a/filebeat/input/journald/input.go b/filebeat/input/journald/input.go index c32d677ffa44..a704962d28c5 100644 --- a/filebeat/input/journald/input.go +++ b/filebeat/input/journald/input.go @@ -20,10 +20,10 @@ package journald import ( + "fmt" "time" "github.com/coreos/go-systemd/v22/sdjournal" - "github.com/urso/sderr" "github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalfield" "github.com/elastic/beats/v7/filebeat/input/journald/pkg/journalread" @@ -181,7 +181,7 @@ func (inp *journald) open(log *logp.Logger, canceler input.Canceler, src cursor. withTransports(inp.Transports), withSyslogIdentifiers(inp.Identifiers)) if err != nil { - return nil, sderr.Wrap(err, "failed to create reader for %{path} journal", src.Name()) + return nil, fmt.Errorf("failed to create reader for %s journal: %w", src.Name(), err) } return reader, nil diff --git a/filebeat/input/journald/pkg/journalread/reader.go b/filebeat/input/journald/pkg/journalread/reader.go index 9994c0aad7c1..6b99036d871d 100644 --- a/filebeat/input/journald/pkg/journalread/reader.go +++ b/filebeat/input/journald/pkg/journalread/reader.go @@ -27,7 +27,6 @@ import ( "time" "github.com/coreos/go-systemd/v22/sdjournal" - "github.com/urso/sderr" "github.com/elastic/beats/v7/libbeat/common/backoff" "github.com/elastic/beats/v7/libbeat/common/cleanup" @@ -96,27 +95,27 @@ func openJournal(path string) (*sdjournal.Journal, error) { if path == localSystemJournalID || path == "" { j, err := sdjournal.NewJournal() if err != nil { - err = sderr.Wrap(err, "failed to open local journal") + err = fmt.Errorf("failed to open local journal: %w", err) } return j, err } stat, err := os.Stat(path) if err != nil { - return nil, sderr.Wrap(err, "failed to read meta data for %{path}", path) + return nil, fmt.Errorf("failed to read meta data for %s: %w", path, err) } if stat.IsDir() { j, err := sdjournal.NewJournalFromDir(path) if err != nil { - err = sderr.Wrap(err, "failed to open journal directory %{path}", path) + err = fmt.Errorf("failed to open journal directory %s: %w", path, err) } return j, err } j, err := sdjournal.NewJournalFromFiles(path) if err != nil { - err = sderr.Wrap(err, "failed to open journal file %{path}", path) + err = fmt.Errorf("failed to open journal file %s: %w", path, err) } return j, err } diff --git a/filebeat/input/v2/input-cursor/input.go b/filebeat/input/v2/input-cursor/input.go index 37036e983c63..c1d4cec07622 100644 --- a/filebeat/input/v2/input-cursor/input.go +++ b/filebeat/input/v2/input-cursor/input.go @@ -19,12 +19,11 @@ package cursor import ( "context" + "errors" "fmt" "runtime/debug" "time" - "github.com/urso/sderr" - "github.com/elastic/go-concert/ctxtool" "github.com/elastic/go-concert/unison" @@ -81,7 +80,7 @@ func (inp *managedInput) Test(ctx input.TestContext) error { errs := grp.Wait() if len(errs) > 0 { - return sderr.WrapAll(errs, "input tests failed") + return fmt.Errorf("input tests failed: %w", errors.Join(errs...)) } return nil } @@ -127,7 +126,7 @@ func (inp *managedInput) Run( } if errs := grp.Wait(); len(errs) > 0 { - return sderr.WrapAll(errs, "input %{id} failed", ctx.ID) + return fmt.Errorf("input %s failed: %w", ctx.ID, errors.Join(errs...)) } return nil } diff --git a/filebeat/input/v2/input-cursor/manager.go b/filebeat/input/v2/input-cursor/manager.go index 0d004acc16a4..1d5578a71223 100644 --- a/filebeat/input/v2/input-cursor/manager.go +++ b/filebeat/input/v2/input-cursor/manager.go @@ -20,11 +20,10 @@ package cursor import ( "context" "errors" + "fmt" "sync" "time" - "github.com/urso/sderr" - "github.com/elastic/go-concert/unison" v2 "github.com/elastic/beats/v7/filebeat/input/v2" @@ -131,7 +130,7 @@ func (cim *InputManager) Init(group unison.Group) error { if err != nil { store.Release() cim.shutdown() - return sderr.Wrap(err, "Can not start registry cleanup process") + return fmt.Errorf("Can not start registry cleanup process: %w", err) } return nil diff --git a/go.mod b/go.mod index bf3ff2ad3124..d77097340676 100644 --- a/go.mod +++ b/go.mod @@ -144,7 +144,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/ugorji/go/codec v1.1.8 - github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 + github.com/urso/sderr v0.0.0-20210525210834-52b04e8f5c71 // indirect github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v1.0.3 go.elastic.co/ecszap v1.0.2 diff --git a/x-pack/filebeat/input/entityanalytics/provider/azuread/statestore.go b/x-pack/filebeat/input/entityanalytics/provider/azuread/statestore.go index 4073b6f0e459..392ce5f04605 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/azuread/statestore.go +++ b/x-pack/filebeat/input/entityanalytics/provider/azuread/statestore.go @@ -11,7 +11,6 @@ import ( "time" "github.com/google/uuid" - "github.com/urso/sderr" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/internal/collections" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/internal/kvstore" @@ -176,7 +175,7 @@ func (s *stateStore) close(commit bool) (err error) { } if err != nil { - err = sderr.WrapAll([]error{err, rollbackErr}, "multiple errors during statestore close") + err = fmt.Errorf("multiple errors during statestore close: %w", errors.Join(err, rollbackErr)) } else { err = rollbackErr } diff --git a/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go b/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go index a54fc3b99287..401b3353d14a 100644 --- a/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go +++ b/x-pack/filebeat/input/entityanalytics/provider/okta/statestore.go @@ -10,8 +10,6 @@ import ( "fmt" "time" - "github.com/urso/sderr" - "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/internal/kvstore" "github.com/elastic/beats/v7/x-pack/filebeat/input/entityanalytics/provider/okta/internal/okta" ) @@ -187,10 +185,8 @@ func (s *stateStore) close(commit bool) (err error) { return } rollbackErr := s.tx.Rollback() - if rollbackErr == nil { - // FIXME: Use fmt.Errorf("multiple errors during statestore close: %w", errors.Join(err, rollbackErr)) - // when go1.20 is supported. - err = sderr.WrapAll([]error{err, rollbackErr}, "multiple errors during statestore close") + if rollbackErr != nil { + err = fmt.Errorf("multiple errors during statestore close: %w", errors.Join(err, rollbackErr)) } }()