Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a threshold for search if want to emit a metric #165

Merged
merged 1 commit into from
Mar 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
add a threshold for search if want to emit a metric
in case some bad regex search defined,
it would be easier to identify the offender
  • Loading branch information
Aiqin-Aiven committed Mar 28, 2024
commit e06ad2f15a616dd6cfb06d26fe3d72d8f23caacc
5 changes: 4 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ Example configuration for a single reader::
"pattern": "SENSITIVE",
"replacement": "[REDACTED]"
}],
"threshold_for_metric_emit": 10
"tags": {
"type": "container"
}
Expand Down Expand Up @@ -375,7 +376,9 @@ Using backrefs, the message can also be restructured into a new format.
Change this setting to true to emit metrics to the metrics host whenever a secret pattern is matched.
This matching happens before other filtering to help catch secrets being leaked to disk.


``threshold_for_metric_emit`` ( default: ``10``)
For the regex searches in journalpump, if search takes longer than this value, default 10 seconds, a metric will be emitted.
type: int unit: second

Sender Configuration
--------------------
Expand Down
13 changes: 13 additions & 0 deletions journalpump/journalpump.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ def __init__(
self.secret_filter_metrics = self._configure_secret_filter_metrics(config)
self.secret_filter_metric_last_send = time.monotonic()
self._is_ready = True
self.threshold_for_metric_emit = self._configure_threshold_for_metric_emit(config)

def invalidate(self) -> None:
"""
Expand Down Expand Up @@ -551,6 +552,9 @@ def _validate_and_build_secret_filters(self, config):

return secret_filters

def _configure_threshold_for_metric_emit(self, config):
return int(config.get("threshold_for_metric_emit", 10))

def perform_searches(self, jobject):
entry = jobject.entry
results = {}
Expand Down Expand Up @@ -580,7 +584,16 @@ def perform_searches(self, jobject):
break
byte_fields[field] = line

start_time = time.perf_counter()
match = regex.search(line)
regex_search_duration = time.perf_counter() - start_time
if regex_search_duration > self.threshold_for_metric_emit:
self.stats.gauge(
metric="journal.perform_search_regex_duration",
value=regex_search_duration,
tags=self.make_tags({"regex": regex.pattern}),
)
self.log.info("Slow regex search: %s for duration %s seconds", regex, regex_search_duration)
if not match:
all_match = False
break
Expand Down
10 changes: 9 additions & 1 deletion test/test_journalpump.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,9 @@ def test_journal_reader_tagging(tmpdir):
pump = JournalPump(journalpump_path)
reader = pump.readers["system"]

# Mocking stats object
stats_mock = mock.MagicMock()

# matching entry
entry = JournalObject(
entry={
Expand All @@ -323,7 +326,9 @@ def test_journal_reader_tagging(tmpdir):
"SYSLOG_IDENTIFIER": "kernel",
}
)
result = reader.perform_searches(entry)
with mock.patch("time.perf_counter") as mock_perf_counter, mock.patch.object(reader, "stats", stats_mock):
mock_perf_counter.side_effect = [0, 11, 0, 1, 2, 3, 4, 5]
result = reader.perform_searches(entry)
expected = {
"kernel.cpu.temperature": {
"cpu": "CPU0",
Expand All @@ -333,6 +338,9 @@ def test_journal_reader_tagging(tmpdir):
}
}
assert result == expected
stats_mock.gauge.assert_called_once_with(
metric="journal.perform_search_regex_duration", value=11, tags={"reader": "system", "regex": "^(?P<from>.*)$"}
)

# some fields are not matching
entry = JournalObject(
Expand Down
Loading