diff --git a/src/npg/conf.py b/src/npg/conf.py new file mode 100644 index 0000000..03dea3c --- /dev/null +++ b/src/npg/conf.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2024 Genome Research Ltd. All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import configparser +import dataclasses +import os +from dataclasses import dataclass +from os import PathLike +from pathlib import Path +from typing import TypeVar + +from structlog import get_logger + +# The TypeVar isn't bound because dataclasses don't have a common base class. It's +# just here to illustrate that from_file returns an instance of the same dataclass +# that was passed to the constructor. +D = TypeVar("D") + +log = get_logger(__name__) + + +class ParseError(Exception): + """Raised when a configuration file cannot be parsed.""" + + pass + + +class IniData: + """A configuration class that reads values from an INI file to create an instance of + a specified dataclass. Using a dataclass results in configuration that is easier + to understand and more maintainable than using a bare dictionary because all its + fields are explicitly defined. This means that they are readable, navigable and + autocomplete-able in an IDE, can have default values, and can have docstrings. + + E.g. given an INI file 'config.ini': + + [server] + host = localhost + port = 9000 + + and a dataclass: + + @dataclass + class ServerConfig: + host: str + port: str + + the following code will create a new instance obj of the dataclass where + host='localhost' and port='9000': + + parser = IniData(ServerConfig) + obj = parser.from_file("config.ini", "server") + + If the 'use_env' flag is set, the configuration will fall back to an environment + variable if the INI file does not contain a value for that field. The default + environment variable name is the field name in upper case. As the shell environment + lacks the context of an enclosing dataclass, the default name may benefit from a + prefix to make it more descriptive. This can be provided by using the 'env_prefix' + argument. + + E.g. given an INI file 'config.ini': + + [server] + host = localhost + + with an environment variable 'SERVER_PORT' set to '9001' and the same dataclass as + above, the following code will create a new instance obj of the dataclass where + host='localhost' and port='9001': + + parser = IniData(ServerConfig, use_env=True, env_prefix="SERVER_"). + obj = from_file("config.ini", "server") + + All values are treated as strings. + + The class provides INFO level logging of its actions to enable loading of + configurations to be traced. + """ + + def __init__(self, cls: D, use_env: bool = False, env_prefix: str = ""): + """Makes a new configuration instance which can create instances of the + dataclass 'D'. + + Args: + cls: A dataclass to bind to this configuration. + use_env: If True, fall back to environment variables if a value is not found + in an INI file. Defaults to False. The environment variable used to + supply a missing field's value is the field name in upper case + (prefixed with env_prefix, if provided). + env_prefix: A prefix to add to the name of any environment variable used as + a fallback. Defaults to "". The prefix is folded to upper case. + Dataclass field names exist in the context of their class. However, + environment variables exist in a global context and can benefit from a + more descriptive name. The prefix can be used to provide that. + """ + if dataclass is None: + raise ValueError("A dataclass argument is required") + if not dataclasses.is_dataclass(cls): + raise ValueError(f"'{cls}' is not a dataclass") + + self.dataclass = cls + self.use_env = use_env + self.env_prefix = env_prefix + + def from_file( + self, + ini_file: PathLike | str, + section: str, + ) -> D: + """Create a new dataclass instance from an INI file section. + + Args: + ini_file: The INI file path. + section: The section name containing the configuration. + + Returns: + A new dataclass instance with values populated from the INI file. + """ + p = Path(ini_file).resolve().absolute().as_posix() + + log.info( + "Reading configuration from file", + path=p, + section=section, + dataclass=self.dataclass, + ) + + parser = configparser.ConfigParser() + if not parser.read(p): + raise ParseError(f"Could not read '{p}'") + + kwargs = {} + for field in dataclasses.fields(self.dataclass): + if parser.has_option(section, field.name): + val = parser.get(section, field.name) + + if val is None and self.use_env: + env_var = self.env_prefix.upper() + field.name.upper() + log.info( + "Absent field; using an environment variable", + path=p, + section=section, + field=field.name, + env_var=env_var, + ) + + val = os.environ.get(env_var) + + kwargs[field.name] = val + + elif self.use_env: + env_var = self.env_prefix.upper() + field.name.upper() + + log.debug( + "Absent INI section; using an environment variable", + path=p, + section=section, + field=field.name, + env_var=env_var, + ) + + kwargs[field.name] = os.environ.get(env_var) + + log.debug("Reading complete", dataclass=self.dataclass, kwargs=kwargs) + + return self.dataclass(**kwargs) diff --git a/tests/test_conf.py b/tests/test_conf.py new file mode 100644 index 0000000..ccfbc4e --- /dev/null +++ b/tests/test_conf.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2024 Genome Research Ltd. All rights reserved. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from dataclasses import dataclass +from typing import Optional + +import pytest +from pytest import mark as m + +from npg.conf import IniData, ParseError + + +@dataclass +class ExampleConfig: + """An example dataclass for testing.""" + + key1: str + key2: Optional[str] = None + + +class NonDataclass: + """A example non-dataclass for testing.""" + + pass + + +@m.describe("IniData") +@m.context("When the INI file is missing") +@m.it("Raises a ParseError") +def test_missing_ini_file(tmp_path): + ini_file = tmp_path / "missing.ini" + + parser = IniData(ExampleConfig) + with pytest.raises(ParseError): + parser.from_file(ini_file, "section") + + +@m.context("When the dataclass is missing") +@m.it("Raises a ValueError") +def test_invalid_dataclass(): + with pytest.raises(ValueError): + IniData(None) + + +@m.context("When the dataclass is a non-dataclass") +@m.it("Raises a ValueError") +def test_non_dataclass(): + with pytest.raises(ValueError): + IniData(NonDataclass) + + +@m.context("When the INI file is present") +@m.it("Populates a dataclass") +def test_populate_from_ini_file(tmp_path): + ini_file = tmp_path / "config.ini" + section = "test" + val1 = "value1" + val2 = "value2" + ini_file.write_text(f"[{section}]\nkey1={val1}\nkey2={val2}\n") + + parser = IniData(ExampleConfig) + assert parser.from_file(ini_file, section) == ExampleConfig(key1=val1, key2=val2) + + +@m.context("When a field required by the dataclass is absent") +@m.it("Raises a TypeError") +def test_missing_required_value(tmp_path): + ini_file = tmp_path / "config.ini" + section = "test" + val2 = "value2" + ini_file.write_text(f"[{section}]\nkey2={val2}\n") + + parser = IniData(ExampleConfig) + with pytest.raises(TypeError): + parser.from_file(ini_file, section) + + +@m.context("When an optional field is absent") +@m.it("Populates a dataclass") +def test_missing_non_required_value(tmp_path): + ini_file = tmp_path / "config.ini" + section = "test" + val1 = "value1" + ini_file.write_text(f"[{section}]\nkey1={val1}\n") + + parser = IniData(ExampleConfig) + assert parser.from_file(ini_file, section) == ExampleConfig(key1=val1) + + +@m.context("When environment variables are not to be used") +@m.it("Does not fall back to environment variables when a field is absent") +def test_no_env_fallback(monkeypatch, tmp_path): + ini_file = tmp_path / "config.ini" + section = "test" + val1 = "value1" + ini_file.write_text(f"[{section}]\nkey1={val1}\n") + + env_val2 = "environment_value2" + monkeypatch.setenv("KEY2", env_val2) + + parser = IniData(ExampleConfig, use_env=False) + + assert parser.from_file(ini_file, section) == ExampleConfig(key1=val1, key2=None) + + +@m.context("When environment variables are to be used") +@m.it("Falls back to environment variables when a field is absent") +def test_env_fallback(monkeypatch, tmp_path): + ini_file = tmp_path / "config.ini" + section = "test" + val1 = "value1" + ini_file.write_text(f"[{section}]\nkey1={val1}\n") + + env_val2 = "environment_value2" + monkeypatch.setenv("KEY2", env_val2) + + parser = IniData(ExampleConfig, use_env=True) + assert parser.from_file(ini_file, section) == ExampleConfig( + key1=val1, key2=env_val2 + ) + + +@m.context("When environment variables are to be used with a prefix") +@m.it("Falls back to environment variables with a prefix when a field is absent") +def test_env_fallback_with_prefix(monkeypatch, tmp_path): + ini_file = tmp_path / "config.ini" + section = "test" + val1 = "value1" + ini_file.write_text(f"[{section}]\nkey1={val1}\n") + + env_val2 = "environment_value2" + monkeypatch.setenv("EXAMPLE_KEY2", env_val2) + + parser = IniData(ExampleConfig, use_env=True, env_prefix="EXAMPLE_") + assert parser.from_file(ini_file, section) == ExampleConfig( + key1=val1, key2=env_val2 + )