From fe5a01b681b1460d9d7f19917947cad0ce26703a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Wed, 17 Jul 2024 20:15:26 -0600 Subject: [PATCH] feat: Developers can now customize the default logging configuration for their taps/targets by adding `default_logging.yml` to their package (#2432) * feat: Developers can now customize the default logging configuration for their taps/targets by overriding the `get_default_logging_config` method Related: * Closes https://github.com/meltano/sdk/issues/1373 * Use a standard location for the default logging config * Small refactor * Tweak docs language * Small refactor * Pass package name to get the logging config * Get module name programmatically * Address broken `importlib.resources` in Python 3.9 --- docs/implementation/logging.md | 18 ++++++++++++++++++ poetry.lock | 2 +- pyproject.toml | 2 +- singer_sdk/helpers/_resources.py | 2 +- singer_sdk/metrics.py | 16 +++++++++++----- singer_sdk/plugin_base.py | 5 ++++- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/implementation/logging.md b/docs/implementation/logging.md index da20a5780..adf6f2182 100644 --- a/docs/implementation/logging.md +++ b/docs/implementation/logging.md @@ -77,3 +77,21 @@ This will send metrics to a `metrics.log`: 2022-09-29 00:48:53,302 INFO METRIC: {"metric_type": "timer", "metric": "sync_duration", "value": 0.5258760452270508, "tags": {"stream": "countries", "context": {}, "status": "succeeded"}} 2022-09-29 00:48:53,303 INFO METRIC: {"metric_type": "counter", "metric": "record_count", "value": 250, "tags": {"stream": "countries", "context": {}}} ``` + +## For package developers + +If you're developing a tap or target package and would like to customize its logging configuration, you can put a `default_loggging.yml` file in the package root to set the default logging configuration for your package. This file will be used if the `SINGER_SDK_LOG_CONFIG` environment variable is not set: + +``` +. +├── README.md +├── poetry.lock +├── pyproject.toml +└── tap_example +    ├── __init__.py +    ├── __main__.py +    ├── default_logging.yml # <-- This file will be used if SINGER_SDK_LOG_CONFIG is not set +    ├── client.py +    ├── streams.py +    └── tap.py +``` diff --git a/poetry.lock b/poetry.lock index 92ea330cd..73a4db60d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2620,4 +2620,4 @@ testing = ["pytest", "pytest-durations"] [metadata] lock-version = "2.0" python-versions = ">=3.8" -content-hash = "91c9f418561bf500a204fa69295a5bb8b66da5b01ea1b77c397d38cb3b749ac4" +content-hash = "6a6acd1298eca878a1a7e0dc47decdb156fdc8eb6622dbba1e6b5422e190cecd" diff --git a/pyproject.toml b/pyproject.toml index dcf67b2e8..eae7ecd64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ backports-datetime-fromisoformat = { version = ">=2.0.1", python = "<3.11" } click = "~=8.0" fs = ">=2.4.16" importlib-metadata = {version = "<9.0.0", python = "<3.12"} -importlib-resources = {version = ">=5.12.0,!=6.2.0,!=6.3.0,!=6.3.1", python = "<3.9"} +importlib-resources = {version = ">=5.12.0,!=6.2.0,!=6.3.0,!=6.3.1", python = "<3.10"} inflection = ">=0.5.1" joblib = ">=1.3.0" jsonpath-ng = ">=1.5.3" diff --git a/singer_sdk/helpers/_resources.py b/singer_sdk/helpers/_resources.py index 89b3ed269..02f7b30ff 100644 --- a/singer_sdk/helpers/_resources.py +++ b/singer_sdk/helpers/_resources.py @@ -8,7 +8,7 @@ from singer_sdk.helpers._compat import Traversable -if sys.version_info < (3, 9): +if sys.version_info < (3, 10): import importlib_resources else: import importlib.resources as importlib_resources diff --git a/singer_sdk/metrics.py b/singer_sdk/metrics.py index 8a7efe51e..50d7d3926 100644 --- a/singer_sdk/metrics.py +++ b/singer_sdk/metrics.py @@ -394,23 +394,29 @@ def _load_yaml_logging_config(path: Traversable | Path) -> t.Any: # noqa: ANN40 return yaml.safe_load(f) -def _get_default_config() -> t.Any: # noqa: ANN401 +def _get_default_config_path(package: str) -> Traversable: """Get a logging configuration. + Args: + package: The package name to get the logging configuration for. + Returns: A logging configuration. """ - log_config_path = get_package_files("singer_sdk").joinpath("default_logging.yml") - return _load_yaml_logging_config(log_config_path) + filename = "default_logging.yml" + path = get_package_files(package) / filename + return path if path.is_file() else get_package_files("singer_sdk") / filename -def _setup_logging(config: t.Mapping[str, t.Any]) -> None: +def _setup_logging(config: t.Mapping[str, t.Any], *, package: str) -> None: """Setup logging. Args: + package: The package name to get the logging configuration for. config: A plugin configuration dictionary. """ - logging.config.dictConfig(_get_default_config()) + path = _get_default_config_path(package) + logging.config.dictConfig(_load_yaml_logging_config(path)) config = config or {} metrics_log_level = config.get(METRICS_LOG_LEVEL_SETTING, "INFO").upper() diff --git a/singer_sdk/plugin_base.py b/singer_sdk/plugin_base.py index 1564558cf..b88559088 100644 --- a/singer_sdk/plugin_base.py +++ b/singer_sdk/plugin_base.py @@ -162,7 +162,10 @@ def __init__( if self._is_secret_config(k): config_dict[k] = SecretString(v) self._config = config_dict - metrics._setup_logging(self.config) # noqa: SLF001 + metrics._setup_logging( # noqa: SLF001 + self.config, + package=self.__module__.split(".", maxsplit=1)[0], + ) self.metrics_logger = metrics.get_metrics_logger() self._validate_config(raise_errors=validate_config)