From 33332b5c326dd2e0340dc2809e6e45ec9ebd08df Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Mon, 5 Aug 2024 11:24:52 +0530 Subject: [PATCH 1/5] Added throttling metrics Signed-off-by: Athish Pranav D --- lib/fluent/plugin/in_tail.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index 6a83075cc5..26793885de 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -36,7 +36,7 @@ class TailInput < Fluent::Plugin::Input helpers :timer, :event_loop, :parser, :compat_parameters RESERVED_CHARS = ['/', '*', '%'].freeze - MetricsInfo = Struct.new(:opened, :closed, :rotated) + MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled) class WatcherSetupError < StandardError def initialize(msg) @@ -208,7 +208,8 @@ def configure(conf) opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files") closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files") rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files") - @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics) + throttling_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_throttled_total", help_text: "Total number of times throttling occurs per file when throttling enabled") + @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics) end def configure_tag @@ -793,6 +794,7 @@ def statistics 'opened_file_count' => @metrics.opened.get, 'closed_file_count' => @metrics.closed.get, 'rotated_file_count' => @metrics.rotated.get, + 'throttled_log_count' => @metrics.throttled.get, }) } stats @@ -1192,8 +1194,9 @@ def should_shutdown_now? end def handle_notify - return if limit_bytes_per_second_reached? - return if group_watcher&.limit_lines_reached?(@path) + if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path) + @metrics.throttled.inc + return with_io do |io| begin @@ -1220,6 +1223,7 @@ def handle_notify if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now? # Just get out from tailing loop. + @metrics.throttled.inc read_more = false break end From 23df4e2e1538e9332ac6c462f6878b28e022bc38 Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Mon, 5 Aug 2024 14:19:06 +0530 Subject: [PATCH 2/5] Small bug in the if block Signed-off-by: Athish Pranav D --- lib/fluent/plugin/in_tail.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index 26793885de..f793ae2b85 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -1197,6 +1197,7 @@ def handle_notify if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path) @metrics.throttled.inc return + end with_io do |io| begin From 193ebbafde3645f1c22f092d1959457ab2b8a84c Mon Sep 17 00:00:00 2001 From: Athish Pranav D Date: Tue, 6 Aug 2024 10:10:08 +0530 Subject: [PATCH 3/5] Addressed comments Signed-off-by: Athish Pranav D --- lib/fluent/plugin/in_tail.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index f793ae2b85..852f2723bd 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -1224,7 +1224,7 @@ def handle_notify if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now? # Just get out from tailing loop. - @metrics.throttled.inc + @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached? read_more = false break end From aa02f59ca7a8030cadbaec6a1bd657aab7564cf9 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Tue, 6 Aug 2024 17:11:57 +0900 Subject: [PATCH 4/5] test_in_tail: Remove needless metrics initialization It's introduced at #3504 but it's not needed even in the original pull request because `create_driver` runs `configure` and in_tail's one initializes metrics. Signed-off-by: Takuro Ashie --- test/plugin/test_in_tail.rb | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb index bb16c1ab8e..f9e3b1a0c5 100644 --- a/test/plugin/test_in_tail.rb +++ b/test/plugin/test_in_tail.rb @@ -1878,13 +1878,6 @@ def test_z_refresh_watchers plugin.instance_eval do @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log) @loop = Coolio::Loop.new - opened_file_metrics = Fluent::Plugin::LocalMetrics.new - opened_file_metrics.configure(config_element('metrics', '', {})) - closed_file_metrics = Fluent::Plugin::LocalMetrics.new - closed_file_metrics.configure(config_element('metrics', '', {})) - rotated_file_metrics = Fluent::Plugin::LocalMetrics.new - rotated_file_metrics.configure(config_element('metrics', '', {})) - @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics) end Timecop.freeze(2010, 1, 2, 3, 4, 5) do @@ -2245,15 +2238,6 @@ def test_should_close_watcher_after_rotate_wait config = common_follow_inode_config + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"}) d = create_driver(config, false) - d.instance.instance_eval do - opened_file_metrics = Fluent::Plugin::LocalMetrics.new - opened_file_metrics.configure(config_element('metrics', '', {})) - closed_file_metrics = Fluent::Plugin::LocalMetrics.new - closed_file_metrics.configure(config_element('metrics', '', {})) - rotated_file_metrics = Fluent::Plugin::LocalMetrics.new - rotated_file_metrics.configure(config_element('metrics', '', {})) - @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics) - end Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt", "wb") {|f| f.puts "test1" From 18ef7e696e635c7a44693a0724ce423b2c31eeaa Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Tue, 6 Aug 2024 17:25:33 +0900 Subject: [PATCH 5/5] test_io_handler: Follow a recent change in metrics `throttling_metrics` has been newly added. Signed-off-by: Takuro Ashie --- test/plugin/in_tail/test_io_handler.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/plugin/in_tail/test_io_handler.rb b/test/plugin/in_tail/test_io_handler.rb index f305fdbdc3..5dd37dc4b8 100644 --- a/test/plugin/in_tail/test_io_handler.rb +++ b/test/plugin/in_tail/test_io_handler.rb @@ -15,7 +15,9 @@ def setup closed_file_metrics.configure(config_element('metrics', '', {})) rotated_file_metrics = Fluent::Plugin::LocalMetrics.new rotated_file_metrics.configure(config_element('metrics', '', {})) - @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics) + throttling_metrics = Fluent::Plugin::LocalMetrics.new + throttling_metrics.configure(config_element('metrics', '', {})) + @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics) yield end end