From 27864b44ea0ce27c6034788409bf2cba13410fa0 Mon Sep 17 00:00:00 2001 From: ShourieG Date: Fri, 13 Dec 2024 15:04:04 +0530 Subject: [PATCH] [filebeat][streaming] - Added default values for websocket retries & put a cap on waitTime to be <= waitMax (#42012) * added default retry values and ensured a cap on waitTime i.e now waitTime <= wait_max * updated changelog (cherry picked from commit 5ed055bb9dd4c5a4733f00800892d65debd897d8) --- CHANGELOG.next.asciidoc | 4 ++++ x-pack/filebeat/docs/inputs/input-streaming.asciidoc | 6 +++--- x-pack/filebeat/input/streaming/config.go | 5 +++++ x-pack/filebeat/input/streaming/websocket.go | 4 ++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 462066324ac..8223cafc3f9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -222,6 +222,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add support for SSL and Proxy configurations for websoket type in streaming input. {pull}41934[41934] - Refactor & cleanup with updates to default values and documentation. {pull}41834[41834] - Added support for retry configuration in GCS input. {issue}11580[11580] {pull}41862[41862] +- AWS S3 input registry cleanup for untracked s3 objects. {pull}41694[41694] +- The environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true` enables internal logs tracer for the azure-eventhub input. {issue}41931[41931] {pull}41932[41932] +- Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977] +- Added default values in the streaming input for websocket retries and put a cap on retry wait time to be lesser than equal to the maximum defined wait time. {pull}42012[42012] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-streaming.asciidoc b/x-pack/filebeat/docs/inputs/input-streaming.asciidoc index 9a6f67e5bc4..7f07fb4954f 100644 --- a/x-pack/filebeat/docs/inputs/input-streaming.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-streaming.asciidoc @@ -337,17 +337,17 @@ filebeat.inputs: [float] ==== `retry.max_attempts` -The maximum number of times the input should attempt to reconnect to the streaming data source in the event of a connection failure. The default value is `nil` which means no retries will be attempted. +The maximum number of times the input should attempt to reconnect to the streaming data source in the event of a connection failure. The default value is `5` which means a maximum of 5 retries will be attempted. [float] ==== `retry.wait_min` -The minimum time to wait between retries. This ensures that retries are spaced out enough to give the system time to recover or resolve transient issues, rather than bombarding the system with rapid retries. For example, `wait_min` might be set to 1 second, meaning that even if the calculated backoff is less than this, the client will wait at least 1 second before retrying. +The minimum time to wait between retries. This ensures that retries are spaced out enough to give the system time to recover or resolve transient issues, rather than bombarding the system with rapid retries. For example, `wait_min` might be set to 1 second, meaning that even if the calculated backoff is less than this, the client will wait at least 1 second before retrying. The default value is `1` second. [float] ==== `retry.wait_max` -The maximum time to wait between retries. This prevents the retry mechanism from becoming too slow, ensuring that the client does not wait indefinitely between retries. This is crucial in systems where timeouts or user experience are critical. For example, `wait_max` might be set to 10 seconds, meaning that even if the calculated backoff is greater than this, the client will wait at most 10 seconds before retrying. +The maximum time to wait between retries. This prevents the retry mechanism from becoming too slow, ensuring that the client does not wait indefinitely between retries. This is crucial in systems where timeouts or user experience are critical. For example, `wait_max` might be set to 10 seconds, meaning that even if the calculated backoff is greater than this, the client will wait at most 10 seconds before retrying. The default value is `30` seconds. [float] === `timeout` diff --git a/x-pack/filebeat/input/streaming/config.go b/x-pack/filebeat/input/streaming/config.go index 6ccaf0c7349..eea8c2afc70 100644 --- a/x-pack/filebeat/input/streaming/config.go +++ b/x-pack/filebeat/input/streaming/config.go @@ -171,5 +171,10 @@ func defaultConfig() config { Transport: httpcommon.HTTPTransportSettings{ Timeout: 180 * time.Second, }, + Retry: &retry{ + MaxAttempts: 5, + WaitMin: 1 * time.Second, + WaitMax: 30 * time.Second, + }, } } diff --git a/x-pack/filebeat/input/streaming/websocket.go b/x-pack/filebeat/input/streaming/websocket.go index 9e7904260df..a8fbd4c664c 100644 --- a/x-pack/filebeat/input/streaming/websocket.go +++ b/x-pack/filebeat/input/streaming/websocket.go @@ -261,6 +261,10 @@ func calculateWaitTime(waitMin, waitMax time.Duration, attempt int) time.Duratio jitter := rand.Float64() * maxJitter waitTime := time.Duration(backoff + jitter) + // caps the wait time to the maximum wait time + if waitTime > waitMax { + waitTime = waitMax + } return waitTime }