Minor changes

Signed-off-by: ToWaIDS <[email protected]>

osc_extraction_utils/__init__.py

@@ -1 +0,0 @@
-

osc_extraction_utils/conftest.py

@@ -13,7 +13,6 @@
 from osc_extraction_utils.settings import (
     MainSettings,
     S3Settings,
-    Settings,
     get_main_settings,
     get_s3_settings,
 )
@@ -128,7 +127,7 @@ def s3_settings() -> S3Settings:
 
 # TODO add test mode paths?
 @pytest.fixture(scope="session")
-def project_paths(main_settings: Settings) -> ProjectPaths:
+def project_paths(main_settings: MainSettings) -> ProjectPaths:
     return ProjectPaths("test_project", main_settings)
 
 

osc_extraction_utils/s3_communication.py

@@ -26,7 +26,11 @@ class S3Communication(object):
     """
 
     def __init__(
-        self, s3_endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str, s3_bucket: str
+        self,
+        s3_endpoint_url: str | None,
+        aws_access_key_id: str | None,
+        aws_secret_access_key: str | None,
+        s3_bucket: str | None,
     ) -> None:
         """Initialize communicator."""
         self.s3_endpoint_url = s3_endpoint_url

osc_extraction_utils/settings_handler.py

@@ -1,8 +1,9 @@
 from pathlib import Path
+from typing import Type
 
 import yaml
 
-from osc_extraction_utils.settings import MainSettings, S3Settings, Settings
+from osc_extraction_utils.settings import MainSettings, S3Settings
 
 
 class SettingsHandler:
@@ -28,14 +29,14 @@ def read_settings(
             raise FileNotFoundError
 
     @staticmethod
-    def _read_setting_file(path: Path) -> Settings:
+    def _read_setting_file(path: Path) -> S3Settings | MainSettings:
         with open(path, mode="r") as file_settings:
             loaded_settings: dict = yaml.safe_load(file_settings)
-        settings: Settings = SettingsHandler._settings_factory(loaded_settings)
+        settings: Type[S3Settings] | Type[MainSettings] = SettingsHandler._settings_factory(loaded_settings)
         return settings(**loaded_settings)
 
     @staticmethod
-    def _settings_factory(settings: dict) -> Settings:
+    def _settings_factory(settings: dict) -> Type[S3Settings] | Type[MainSettings]:
         if "main_bucket" in settings:
             return S3Settings
         else:

osc_extraction_utils/tests/test_generate_text.py

@@ -4,11 +4,12 @@
 from _pytest.capture import CaptureFixture
 
 from osc_extraction_utils.merger import generate_text_3434
+from osc_extraction_utils.paths import ProjectPaths
 from osc_extraction_utils.s3_communication import S3Communication
 from osc_extraction_utils.settings import S3Settings
 
 
-def test_generate_text_with_s3(prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path):
+def test_generate_text_with_s3(prerequisites_generate_text, path_folder_temporary: Path, project_paths: ProjectPaths):
     """Tests if the s3 connection objects are created and their methods are called
     Requesting prerequisites_generate_text automatically (autouse)
 
@@ -55,7 +56,9 @@ def test_generate_text_with_s3(prerequisites_generate_text, path_folder_temporar
     assert any([call for call in call_list if "upload_file_to_s3" in call])
 
 
-def test_generate_text_no_s3(prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path):
+def test_generate_text_no_s3(
+    prerequisites_generate_text, path_folder_temporary: Path, project_paths: ProjectPaths, s3_settings: S3Settings
+):
     """Tests if files are taken from the folder relevance,
     then read in and putting the content into the file text_3434.csv. Note that
     the header of text_3434.csv is taken from the first file read in
@@ -68,9 +71,8 @@ def test_generate_text_no_s3(prerequisites_generate_text, path_folder_temporary:
     path_folder_text_3434 = path_folder_temporary / "folder_test_3434"
     project_name = "test"
     s3_usage = False
-    project_settings = None
 
-    generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths)
+    generate_text_3434(project_name, s3_usage, s3_settings, project_paths=project_paths)
 
     # ensure that the header and the content form the first file is written to
     # the file text_3434.csv in folder relevance and the the content of the other
@@ -91,7 +93,9 @@ def test_generate_text_no_s3(prerequisites_generate_text, path_folder_temporary:
                 assert line_content.rstrip() in strings_expected
 
 
-def test_generate_text_successful(prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path):
+def test_generate_text_successful(
+    prerequisites_generate_text, path_folder_temporary: Path, project_paths: ProjectPaths, s3_settings: S3Settings
+):
     """Tests if the function returns true
     Requesting prerequisites_generate_text automatically (autouse)
 
@@ -100,14 +104,17 @@ def test_generate_text_successful(prerequisites_generate_text, path_folder_tempo
     """
     project_name = "test"
     s3_usage = False
-    project_settings = None
 
-    return_value = generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths)
+    return_value = generate_text_3434(project_name, s3_usage, s3_settings, project_paths=project_paths)
     assert return_value is True
 
 
 def test_generate_text_not_successful_empty_folder(
-    prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path, capsys: CaptureFixture[str]
+    prerequisites_generate_text,
+    path_folder_temporary: Path,
+    project_paths: ProjectPaths,
+    capsys: CaptureFixture[str],
+    s3_settings: S3Settings,
 ):
     """Tests if the function returns false
     Requesting prerequisites_generate_text automatically (autouse)
@@ -119,7 +126,6 @@ def test_generate_text_not_successful_empty_folder(
     """
     project_name = "test"
     s3_usage = False
-    project_settings = None
 
     # clear the relevance folder
     path_folder_relevance = path_folder_temporary / "relevance"
@@ -128,15 +134,18 @@ def test_generate_text_not_successful_empty_folder(
             file.unlink()
 
     # call the function
-    return_value = generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths)
+    return_value = generate_text_3434(project_name, s3_usage, s3_settings, project_paths=project_paths)
 
     output_cmd, _ = capsys.readouterr()
     assert "No relevance inference results found." in output_cmd
     assert return_value is False
 
 
 def test_generate_text_not_successful_exception(
-    prerequisites_generate_text, path_folder_temporary: Path, project_paths: Path
+    prerequisites_generate_text,
+    path_folder_temporary: Path,
+    project_paths: ProjectPaths,
+    s3_settings: S3Settings,
 ):
     """Tests if the function returns false
     Requesting prerequisites_generate_text automatically (autouse)
@@ -146,7 +155,6 @@ def test_generate_text_not_successful_exception(
     """
     project_name = "test"
     s3_usage = False
-    project_settings = None
 
     # clear the relevance folder
     path_folder_relevance = path_folder_temporary / "relevance"
@@ -156,6 +164,11 @@ def test_generate_text_not_successful_exception(
 
     # patch glob.iglob to force an exception...
     with patch("osc_extraction_utils.merger.glob.iglob", side_effect=lambda *args: [None]):
-        return_value = generate_text_3434(project_name, s3_usage, project_settings, project_paths=project_paths)
+        return_value = generate_text_3434(
+            project_name=project_name,
+            s3_usage=s3_usage,
+            s3_settings=s3_settings,
+            project_paths=project_paths,
+        )
 
         assert return_value is False

osc_extraction_utils/tests/test_merger.py

@@ -2,8 +2,11 @@
 from unittest.mock import patch
 
 import pytest
-from osc_extraction_utils.conftest import create_multiple_xlsx_files, create_single_xlsx_file
 
+from osc_extraction_utils.conftest import (
+    create_multiple_xlsx_files,
+    create_single_xlsx_file,
+)
 from osc_extraction_utils.merger import Merger
 from osc_extraction_utils.paths import ProjectPaths
 from osc_extraction_utils.s3_communication import S3Communication

osc_extraction_utils/tests/test_paths.py

@@ -13,7 +13,7 @@ def path_folder_config_path() -> Path:
 
 
 @pytest.fixture
-def paths_project(main_settings: Settings) -> ProjectPaths:
+def paths_project(main_settings: MainSettings) -> ProjectPaths:
     return ProjectPaths(string_project_name="test_project", main_settings=main_settings)
 
 
@@ -39,7 +39,7 @@ def test_python_executable_set(paths_project: ProjectPaths):
     assert paths_project.PYTHON_EXECUTABLE == "python"
 
 
-def test_check_that_all_required_paths_exist_in_project_path_object(main_settings: Settings):
+def test_check_that_all_required_paths_exist_in_project_path_object(main_settings: MainSettings):
     list_paths_expected = [
         "input/pdfs/training",
         "input/annotations",
@@ -66,7 +66,7 @@ def test_check_that_all_required_paths_exist_in_project_path_object(main_setting
             assert str(path_field_attribute) in list_paths_expected
 
 
-def test_project_paths_update_methods_are_called(main_settings: Settings):
+def test_project_paths_update_methods_are_called(main_settings: MainSettings):
     with (
         patch.object(ProjectPaths, "_update_all_paths_depending_on_path_project_data_folder") as mocked_update_data,
         patch.object(ProjectPaths, "_update_all_paths_depending_on_path_project_model_folder") as mocked_update_model,

osc_extraction_utils/tests/test_run_router.py

@@ -1,4 +1,5 @@
 import typing
+from pathlib import Path
 from unittest.mock import Mock, patch
 
 import pytest
@@ -220,7 +221,7 @@ def test_run_router_kpi_training(
 
     # force an exception of generate_text_3434 by removing the folder_text_3434
     if not project_name:
-        router._project_paths.path_folder_text_3434 = None
+        router._project_paths.path_folder_text_3434 = Path()
 
     mocked_generate_text = Mock()
     if project_name:

osc_extraction_utils/tests/test_running.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from typing import Generator
 
 import pytest
 
@@ -23,7 +24,7 @@
 
 
 @pytest.fixture
-def training_monitor() -> TrainingMonitor:
+def training_monitor() -> Generator[TrainingMonitor, None, None]:
     """The fixture returns a TrainingMonitor object for testing
 
     :yield: TrainingMonitor class for monitor training status

osc_extraction_utils/tests/test_s3_connection.py

@@ -8,13 +8,13 @@
     upload_data_from_local_folder_to_s3_interim_bucket_if_required,
 )
 from osc_extraction_utils.s3_communication import S3Communication
-from osc_extraction_utils.settings import Settings
+from osc_extraction_utils.settings import MainSettings
 
 # S3Settings = get_s3_settings()
 
 
 @pytest.mark.parametrize("s3_usage", [True, False])
-def test_download_data_from_s3_main_bucket_to_local_folder_if_required(s3_usage: bool, main_settings: Settings):
+def test_download_data_from_s3_main_bucket_to_local_folder_if_required(s3_usage: bool, main_settings: MainSettings):
     mocked_s3_bucket = Mock(spec=S3Communication)
     mocked_path_local = Mock(spec=Path("path_local"))
     mocked_path_s3 = Mock(spec=Path("path_s3"))
@@ -31,7 +31,7 @@ def test_download_data_from_s3_main_bucket_to_local_folder_if_required(s3_usage:
 
 
 @pytest.mark.parametrize("s3_usage", [True, False])
-def test_upload_data_from_local_folder_to_s3_interim_bucket_if_required(s3_usage: bool, main_settings: Settings):
+def test_upload_data_from_local_folder_to_s3_interim_bucket_if_required(s3_usage: bool, main_settings: MainSettings):
     mocked_s3_bucket = Mock(spec=S3Communication)
     mocked_path_local = Mock(spec=Path("path_local"))
     mocked_path_s3 = Mock(spec=Path("path_s3"))

osc_extraction_utils/tests/test_settings_handler.py

@@ -1,8 +1,8 @@
 from unittest.mock import patch
 
 import pytest
-from osc_extraction_utils.conftest import project_tests_root
 
+from osc_extraction_utils.conftest import project_tests_root
 from osc_extraction_utils.settings_handler import SettingsHandler