diff --git a/osc_extraction_utils/__init__.py b/osc_extraction_utils/__init__.py index 8b13789..e69de29 100644 --- a/osc_extraction_utils/__init__.py +++ b/osc_extraction_utils/__init__.py @@ -1 +0,0 @@ - diff --git a/osc_extraction_utils/conftest.py b/osc_extraction_utils/conftest.py index 881c0cf..bca70f3 100644 --- a/osc_extraction_utils/conftest.py +++ b/osc_extraction_utils/conftest.py @@ -13,7 +13,6 @@ from osc_extraction_utils.settings import ( MainSettings, S3Settings, - Settings, get_main_settings, get_s3_settings, ) @@ -128,7 +127,7 @@ def s3_settings() -> S3Settings: # TODO add test mode paths? @pytest.fixture(scope="session") -def project_paths(main_settings: Settings) -> ProjectPaths: +def project_paths(main_settings: MainSettings) -> ProjectPaths: return ProjectPaths("test_project", main_settings) diff --git a/osc_extraction_utils/s3_communication.py b/osc_extraction_utils/s3_communication.py index 006f8a9..b6c4028 100644 --- a/osc_extraction_utils/s3_communication.py +++ b/osc_extraction_utils/s3_communication.py @@ -26,7 +26,11 @@ class S3Communication(object): """ def __init__( - self, s3_endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str, s3_bucket: str + self, + s3_endpoint_url: str | None, + aws_access_key_id: str | None, + aws_secret_access_key: str | None, + s3_bucket: str | None, ) -> None: """Initialize communicator.""" self.s3_endpoint_url = s3_endpoint_url diff --git a/osc_extraction_utils/settings_handler.py b/osc_extraction_utils/settings_handler.py index 76be3af..43cc5b6 100644 --- a/osc_extraction_utils/settings_handler.py +++ b/osc_extraction_utils/settings_handler.py @@ -1,8 +1,9 @@ from pathlib import Path +from typing import Type import yaml -from osc_extraction_utils.settings import MainSettings, S3Settings, Settings +from osc_extraction_utils.settings import MainSettings, S3Settings class SettingsHandler: @@ -28,14 +29,14 @@ def read_settings( raise FileNotFoundError @staticmethod - def _read_setting_file(path: Path) -> Settings: + def _read_setting_file(path: Path) -> S3Settings | MainSettings: with open(path, mode="r") as file_settings: loaded_settings: dict = yaml.safe_load(file_settings) - settings: Settings = SettingsHandler._settings_factory(loaded_settings) + settings: Type[S3Settings] | Type[MainSettings] = SettingsHandler._settings_factory(loaded_settings) return settings(**loaded_settings) @staticmethod - def _settings_factory(settings: dict) -> Settings: + def _settings_factory(settings: dict) -> Type[S3Settings] | Type[MainSettings]: if "main_bucket" in settings: return S3Settings else: diff --git a/osc_extraction_utils/tests/test_generate_text.py b/osc_extraction_utils/tests/test_generate_text.py index b5c44f4..42821e9 100644 --- a/osc_extraction_utils/tests/test_generate_text.py +++ b/osc_extraction_utils/tests/test_generate_text.py @@ -4,11 +4,12 @@ from _pytest.capture import CaptureFixture from osc_extraction_utils.merger import generate_text_3434 +from osc_extraction_utils.paths import ProjectPaths from osc_extraction_utils.s3_communication import S3Communication from osc_extraction_utils.settings import S3Settings -def test_generate_text_with_s3(prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path): +def test_generate_text_with_s3(prerequisites_generate_text, path_folder_temporary: Path, project_paths: ProjectPaths): """Tests if the s3 connection objects are created and their methods are called Requesting prerequisites_generate_text automatically (autouse) @@ -55,7 +56,9 @@ def test_generate_text_with_s3(prerequisites_generate_text, path_folder_temporar assert any([call for call in call_list if "upload_file_to_s3" in call]) -def test_generate_text_no_s3(prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path): +def test_generate_text_no_s3( + prerequisites_generate_text, path_folder_temporary: Path, project_paths: ProjectPaths, s3_settings: S3Settings +): """Tests if files are taken from the folder relevance, then read in and putting the content into the file text_3434.csv. Note that the header of text_3434.csv is taken from the first file read in @@ -68,9 +71,8 @@ def test_generate_text_no_s3(prerequisites_generate_text, path_folder_temporary: path_folder_text_3434 = path_folder_temporary / "folder_test_3434" project_name = "test" s3_usage = False - project_settings = None - generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths) + generate_text_3434(project_name, s3_usage, s3_settings, project_paths=project_paths) # ensure that the header and the content form the first file is written to # the file text_3434.csv in folder relevance and the the content of the other @@ -91,7 +93,9 @@ def test_generate_text_no_s3(prerequisites_generate_text, path_folder_temporary: assert line_content.rstrip() in strings_expected -def test_generate_text_successful(prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path): +def test_generate_text_successful( + prerequisites_generate_text, path_folder_temporary: Path, project_paths: ProjectPaths, s3_settings: S3Settings +): """Tests if the function returns true Requesting prerequisites_generate_text automatically (autouse) @@ -100,14 +104,17 @@ def test_generate_text_successful(prerequisites_generate_text, path_folder_tempo """ project_name = "test" s3_usage = False - project_settings = None - return_value = generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths) + return_value = generate_text_3434(project_name, s3_usage, s3_settings, project_paths=project_paths) assert return_value is True def test_generate_text_not_successful_empty_folder( - prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path, capsys: CaptureFixture[str] + prerequisites_generate_text, + path_folder_temporary: Path, + project_paths: ProjectPaths, + capsys: CaptureFixture[str], + s3_settings: S3Settings, ): """Tests if the function returns false Requesting prerequisites_generate_text automatically (autouse) @@ -119,7 +126,6 @@ def test_generate_text_not_successful_empty_folder( """ project_name = "test" s3_usage = False - project_settings = None # clear the relevance folder path_folder_relevance = path_folder_temporary / "relevance" @@ -128,7 +134,7 @@ def test_generate_text_not_successful_empty_folder( file.unlink() # call the function - return_value = generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths) + return_value = generate_text_3434(project_name, s3_usage, s3_settings, project_paths=project_paths) output_cmd, _ = capsys.readouterr() assert "No relevance inference results found." in output_cmd @@ -136,7 +142,10 @@ def test_generate_text_not_successful_empty_folder( def test_generate_text_not_successful_exception( - prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path + prerequisites_generate_text, + path_folder_temporary: Path, + project_paths: ProjectPaths, + s3_settings: S3Settings, ): """Tests if the function returns false Requesting prerequisites_generate_text automatically (autouse) @@ -146,7 +155,6 @@ def test_generate_text_not_successful_exception( """ project_name = "test" s3_usage = False - project_settings = None # clear the relevance folder path_folder_relevance = path_folder_temporary / "relevance" @@ -156,6 +164,11 @@ def test_generate_text_not_successful_exception( # patch glob.iglob to force an exception... with patch("osc_extraction_utils.merger.glob.iglob", side_effect=lambda *args: [None]): - return_value = generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths) + return_value = generate_text_3434( + project_name=project_name, + s3_usage=s3_usage, + s3_settings=s3_settings, + project_paths=project_paths, + ) assert return_value is False diff --git a/osc_extraction_utils/tests/test_merger.py b/osc_extraction_utils/tests/test_merger.py index 065c5c0..346607d 100644 --- a/osc_extraction_utils/tests/test_merger.py +++ b/osc_extraction_utils/tests/test_merger.py @@ -2,8 +2,11 @@ from unittest.mock import patch import pytest -from osc_extraction_utils.conftest import create_multiple_xlsx_files, create_single_xlsx_file +from osc_extraction_utils.conftest import ( + create_multiple_xlsx_files, + create_single_xlsx_file, +) from osc_extraction_utils.merger import Merger from osc_extraction_utils.paths import ProjectPaths from osc_extraction_utils.s3_communication import S3Communication diff --git a/osc_extraction_utils/tests/test_paths.py b/osc_extraction_utils/tests/test_paths.py index ee86f2d..409063f 100644 --- a/osc_extraction_utils/tests/test_paths.py +++ b/osc_extraction_utils/tests/test_paths.py @@ -13,7 +13,7 @@ def path_folder_config_path() -> Path: @pytest.fixture -def paths_project(main_settings: Settings) -> ProjectPaths: +def paths_project(main_settings: MainSettings) -> ProjectPaths: return ProjectPaths(string_project_name="test_project", main_settings=main_settings) @@ -39,7 +39,7 @@ def test_python_executable_set(paths_project: ProjectPaths): assert paths_project.PYTHON_EXECUTABLE == "python" -def test_check_that_all_required_paths_exist_in_project_path_object(main_settings: Settings): +def test_check_that_all_required_paths_exist_in_project_path_object(main_settings: MainSettings): list_paths_expected = [ "input/pdfs/training", "input/annotations", @@ -66,7 +66,7 @@ def test_check_that_all_required_paths_exist_in_project_path_object(main_setting assert str(path_field_attribute) in list_paths_expected -def test_project_paths_update_methods_are_called(main_settings: Settings): +def test_project_paths_update_methods_are_called(main_settings: MainSettings): with ( patch.object(ProjectPaths, "_update_all_paths_depending_on_path_project_data_folder") as mocked_update_data, patch.object(ProjectPaths, "_update_all_paths_depending_on_path_project_model_folder") as mocked_update_model, diff --git a/osc_extraction_utils/tests/test_run_router.py b/osc_extraction_utils/tests/test_run_router.py index 9149183..31056f5 100644 --- a/osc_extraction_utils/tests/test_run_router.py +++ b/osc_extraction_utils/tests/test_run_router.py @@ -1,4 +1,5 @@ import typing +from pathlib import Path from unittest.mock import Mock, patch import pytest @@ -220,7 +221,7 @@ def test_run_router_kpi_training( # force an exception of generate_text_3434 by removing the folder_text_3434 if not project_name: - router._project_paths.path_folder_text_3434 = None + router._project_paths.path_folder_text_3434 = Path() mocked_generate_text = Mock() if project_name: diff --git a/osc_extraction_utils/tests/test_running.py b/osc_extraction_utils/tests/test_running.py index 932de1a..11c1a8f 100644 --- a/osc_extraction_utils/tests/test_running.py +++ b/osc_extraction_utils/tests/test_running.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Generator import pytest @@ -23,7 +24,7 @@ @pytest.fixture -def training_monitor() -> TrainingMonitor: +def training_monitor() -> Generator[TrainingMonitor, None, None]: """The fixture returns a TrainingMonitor object for testing :yield: TrainingMonitor class for monitor training status diff --git a/osc_extraction_utils/tests/test_s3_connection.py b/osc_extraction_utils/tests/test_s3_connection.py index ccdc2bb..d4a0413 100644 --- a/osc_extraction_utils/tests/test_s3_connection.py +++ b/osc_extraction_utils/tests/test_s3_connection.py @@ -8,13 +8,13 @@ upload_data_from_local_folder_to_s3_interim_bucket_if_required, ) from osc_extraction_utils.s3_communication import S3Communication -from osc_extraction_utils.settings import Settings +from osc_extraction_utils.settings import MainSettings # S3Settings = get_s3_settings() @pytest.mark.parametrize("s3_usage", [True, False]) -def test_download_data_from_s3_main_bucket_to_local_folder_if_required(s3_usage: bool, main_settings: Settings): +def test_download_data_from_s3_main_bucket_to_local_folder_if_required(s3_usage: bool, main_settings: MainSettings): mocked_s3_bucket = Mock(spec=S3Communication) mocked_path_local = Mock(spec=Path("path_local")) mocked_path_s3 = Mock(spec=Path("path_s3")) @@ -31,7 +31,7 @@ def test_download_data_from_s3_main_bucket_to_local_folder_if_required(s3_usage: @pytest.mark.parametrize("s3_usage", [True, False]) -def test_upload_data_from_local_folder_to_s3_interim_bucket_if_required(s3_usage: bool, main_settings: Settings): +def test_upload_data_from_local_folder_to_s3_interim_bucket_if_required(s3_usage: bool, main_settings: MainSettings): mocked_s3_bucket = Mock(spec=S3Communication) mocked_path_local = Mock(spec=Path("path_local")) mocked_path_s3 = Mock(spec=Path("path_s3")) diff --git a/osc_extraction_utils/tests/test_settings_handler.py b/osc_extraction_utils/tests/test_settings_handler.py index a8dd78f..0c85f51 100644 --- a/osc_extraction_utils/tests/test_settings_handler.py +++ b/osc_extraction_utils/tests/test_settings_handler.py @@ -1,8 +1,8 @@ from unittest.mock import patch import pytest -from osc_extraction_utils.conftest import project_tests_root +from osc_extraction_utils.conftest import project_tests_root from osc_extraction_utils.settings_handler import SettingsHandler