diff --git a/docs/docs/installation/event-logging.mdx b/docs/docs/installation/event-logging.mdx index e6b0f8b356025..c573a1e953dd6 100644 --- a/docs/docs/installation/event-logging.mdx +++ b/docs/docs/installation/event-logging.mdx @@ -56,5 +56,13 @@ from superset.stats_logger import StatsdStatsLogger STATS_LOGGER = StatsdStatsLogger(host='localhost', port=8125, prefix='superset') ``` +You can also pass metric names that should be ignored via an optional `metric_denylist` parameter: + +``` +STATS_LOGGER = StatsdStatsLogger(host='localhost', port=8125, prefix='superset', metric_denylist={"reports.scheduler"}) +``` + +This would log all metrics, except `reports.scheduler`. + Note that it’s also possible to implement you own logger by deriving `superset.stats_logger.BaseStatsLogger`. diff --git a/superset/models/actor.py b/superset/models/actor.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/superset/stats_logger.py b/superset/stats_logger.py index fc223f752967b..6f6e819703d9c 100644 --- a/superset/stats_logger.py +++ b/superset/stats_logger.py @@ -14,8 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations + import logging -from typing import Optional from colorama import Fore, Style @@ -25,8 +26,19 @@ class BaseStatsLogger: """Base class for logging realtime events""" - def __init__(self, prefix: str = "superset") -> None: + prefix: str + metric_denylist: set[str] + + def should_log(self, key: str) -> bool: + return key not in self.metric_denylist + + def __init__( + self, + prefix: str = "superset", + metric_denylist: set[str] | None = None, + ) -> None: self.prefix = prefix + self.metric_denylist = metric_denylist or set() def key(self, key: str) -> str: if self.prefix: @@ -51,24 +63,30 @@ def gauge(self, key: str, value: float) -> None: class DummyStatsLogger(BaseStatsLogger): def incr(self, key: str) -> None: - logger.debug(Fore.CYAN + "[stats_logger] (incr) " + key + Style.RESET_ALL) + if self.should_log(key): + logger.debug(Fore.CYAN + "[stats_logger] (incr) " + key + Style.RESET_ALL) def decr(self, key: str) -> None: - logger.debug(Fore.CYAN + "[stats_logger] (decr) " + key + Style.RESET_ALL) + if self.should_log(key): + logger.debug(Fore.CYAN + "[stats_logger] (decr) " + key + Style.RESET_ALL) def timing(self, key: str, value: float) -> None: - logger.debug( - Fore.CYAN + f"[stats_logger] (timing) {key} | {value} " + Style.RESET_ALL - ) + if self.should_log(key): + logger.debug( + Fore.CYAN + + f"[stats_logger] (timing) {key} | {value} " + + Style.RESET_ALL + ) def gauge(self, key: str, value: float) -> None: - logger.debug( - Fore.CYAN - + "[stats_logger] (gauge) " - + f"{key}" - + f"{value}" - + Style.RESET_ALL - ) + if self.should_log(key): + logger.debug( + Fore.CYAN + + "[stats_logger] (gauge) " + + f"{key}" + + f"{value}" + + Style.RESET_ALL + ) try: @@ -80,7 +98,8 @@ def __init__( # pylint: disable=super-init-not-called host: str = "localhost", port: int = 8125, prefix: str = "superset", - statsd_client: Optional[StatsClient] = None, + statsd_client: StatsClient | None = None, + metric_denylist: set[str] | None = None, ) -> None: """ Initializes from either params or a supplied, pre-constructed statsd client. @@ -88,22 +107,27 @@ def __init__( # pylint: disable=super-init-not-called If statsd_client argument is given, all other arguments are ignored and the supplied client will be used to emit metrics. """ + super().__init__(metric_denylist=metric_denylist) if statsd_client: self.client = statsd_client else: self.client = StatsClient(host=host, port=port, prefix=prefix) def incr(self, key: str) -> None: - self.client.incr(key) + if self.should_log(key): + self.client.incr(key) def decr(self, key: str) -> None: - self.client.decr(key) + if self.should_log(key): + self.client.decr(key) def timing(self, key: str, value: float) -> None: - self.client.timing(key, value) + if self.should_log(key): + self.client.timing(key, value) def gauge(self, key: str, value: float) -> None: - self.client.gauge(key, value) + if self.should_log(key): + self.client.gauge(key, value) except Exception: # pylint: disable=broad-except pass diff --git a/tests/integration_tests/stats_logger_tests.py b/tests/integration_tests/stats_logger_tests.py index adf7cc1db6f8d..a789bd38ce426 100644 --- a/tests/integration_tests/stats_logger_tests.py +++ b/tests/integration_tests/stats_logger_tests.py @@ -47,3 +47,12 @@ def test_init_with_params(self): stats_logger = StatsdStatsLogger() self.verify_client_calls(stats_logger, mock_client) + + def test_with_metric_deny_list(self): + client = Mock() + stats_logger = StatsdStatsLogger(statsd_client=client, metric_denylist={"foo1"}) + stats_logger.incr("foo1") + client.incr.assert_not_called() + stats_logger.decr("foo2") + client.decr.assert_called_once() + client.decr.assert_called_with("foo2")