From 9e9354e611bbbf7e7df340ed81e6925012ecca56 Mon Sep 17 00:00:00 2001 From: Leonard Herold <92177433+LeonardHd@users.noreply.github.com> Date: Mon, 13 Nov 2023 18:26:54 +0100 Subject: [PATCH] style(python): ruff fixes ANN001,ANN201,ANN401 --- .../exceptions/activity_not_found_error.py | 2 +- ...tion_call_invalid_arguments_count_error.py | 2 +- .../pipeline_not_found_exception.py | 2 +- ...le_being_evaluated_does_not_exist_error.py | 2 +- .../variable_does_not_exist_error.py | 2 +- .../expressions/expression_activity.py | 2 +- .../expressions/expression_dataset.py | 2 +- .../expressions/expression_iteration_item.py | 2 +- .../expressions/expression_linked_service.py | 2 +- .../expressions/expression_parameter.py | 4 +-- .../expressions/expression_variable.py | 2 +- .../functions/function_argument.py | 4 +-- .../functions/function_call.py | 4 +-- .../functions/function_parser.py | 2 +- .../functions/functions_repository.py | 12 ++++---- .../models/activities/base/activity.py | 4 +-- .../control_activities/control_activity.py | 2 +- .../control_activities/for_each_activity.py | 4 +-- .../if_condition_activity.py | 4 +-- .../control_activities/until_activity.py | 4 +-- .../activities/set_variable_activity.py | 2 +- .../models/data_factory_element.py | 4 +-- .../models/expression.py | 2 +- .../models/patch_models.py | 4 +-- .../models/pipelines/pipeline_resource.py | 2 +- .../data_factory_repository_factory.py | 4 +-- .../models/state/pipeline_run_state.py | 12 ++++---- .../scripts/utils/ast_utils.py | 6 ++-- .../scripts/utils/docstring_utils.py | 2 +- .../scripts/utils/string_utils.py | 4 +-- src/python/example/batch_job/batchjob_test.py | 2 +- .../tests/functions/test_function_argument.py | 28 +++++++++---------- .../tests/functions/test_function_call.py | 26 ++++++++--------- .../tests/functions/test_function_parser.py | 4 +-- .../models/activities/base/test_activity.py | 8 ++++-- .../test_for_each_activity.py | 2 +- .../test_if_condition_activity.py | 6 ++-- .../control_activities/test_until_activity.py | 2 +- .../activities/test_set_variable_activity.py | 4 +-- .../pipelines/test_pipeline_resource.py | 6 ++-- 40 files changed, 99 insertions(+), 95 deletions(-) diff --git a/src/python/data_factory_testing_framework/exceptions/activity_not_found_error.py b/src/python/data_factory_testing_framework/exceptions/activity_not_found_error.py index 14a7d98c..770e6b23 100644 --- a/src/python/data_factory_testing_framework/exceptions/activity_not_found_error.py +++ b/src/python/data_factory_testing_framework/exceptions/activity_not_found_error.py @@ -1,3 +1,3 @@ class ActivityNotFoundError(Exception): - def __init__(self, activity_name) -> None: + def __init__(self, activity_name: str) -> None: super().__init__(f"Activity with name {activity_name} not found") diff --git a/src/python/data_factory_testing_framework/exceptions/function_call_invalid_arguments_count_error.py b/src/python/data_factory_testing_framework/exceptions/function_call_invalid_arguments_count_error.py index c9665a81..fcd0bda7 100644 --- a/src/python/data_factory_testing_framework/exceptions/function_call_invalid_arguments_count_error.py +++ b/src/python/data_factory_testing_framework/exceptions/function_call_invalid_arguments_count_error.py @@ -1,5 +1,5 @@ class FunctionCallInvalidArgumentsCountError(Exception): - def __init__(self, name, evaluated_arguments, expected_argument_names) -> None: + def __init__(self, name: str, evaluated_arguments: list, expected_argument_names: list) -> None: message = ( f"FunctionCall {name} has invalid arguments count. " f"Evaluated arguments: \"{', '.join(map(str, evaluated_arguments))}\". " diff --git a/src/python/data_factory_testing_framework/exceptions/pipeline_not_found_exception.py b/src/python/data_factory_testing_framework/exceptions/pipeline_not_found_exception.py index e70e4dfd..63e110e8 100644 --- a/src/python/data_factory_testing_framework/exceptions/pipeline_not_found_exception.py +++ b/src/python/data_factory_testing_framework/exceptions/pipeline_not_found_exception.py @@ -1,3 +1,3 @@ class PipelineNotFoundException(Exception): - def __init__(self, pipeline_name) -> None: + def __init__(self, pipeline_name: str) -> None: super().__init__(f"Pipeline with name {pipeline_name} not found") diff --git a/src/python/data_factory_testing_framework/exceptions/variable_being_evaluated_does_not_exist_error.py b/src/python/data_factory_testing_framework/exceptions/variable_being_evaluated_does_not_exist_error.py index 201ab968..5cad0820 100644 --- a/src/python/data_factory_testing_framework/exceptions/variable_being_evaluated_does_not_exist_error.py +++ b/src/python/data_factory_testing_framework/exceptions/variable_being_evaluated_does_not_exist_error.py @@ -1,3 +1,3 @@ class VariableBeingEvaluatedDoesNotExistError(Exception): - def __init__(self, variable_name) -> None: + def __init__(self, variable_name: str) -> None: super().__init__(f"Variable being evaluated does not exist: {variable_name}") diff --git a/src/python/data_factory_testing_framework/exceptions/variable_does_not_exist_error.py b/src/python/data_factory_testing_framework/exceptions/variable_does_not_exist_error.py index 863d2829..5d876a3f 100644 --- a/src/python/data_factory_testing_framework/exceptions/variable_does_not_exist_error.py +++ b/src/python/data_factory_testing_framework/exceptions/variable_does_not_exist_error.py @@ -1,3 +1,3 @@ class VariableDoesNotExistError(Exception): - def __init__(self, variable_name) -> None: + def __init__(self, variable_name: str) -> None: super().__init__(f"Variable does not exist: {variable_name}") diff --git a/src/python/data_factory_testing_framework/functions/expressions/expression_activity.py b/src/python/data_factory_testing_framework/functions/expressions/expression_activity.py index dc8e611e..1072c290 100644 --- a/src/python/data_factory_testing_framework/functions/expressions/expression_activity.py +++ b/src/python/data_factory_testing_framework/functions/expressions/expression_activity.py @@ -4,7 +4,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def find_and_replace_activity(expression: str, state: PipelineRunState): +def find_and_replace_activity(expression: str, state: PipelineRunState) -> str: pattern = r"activity\('(?P.*?)'\)(?:\.(\w+))+$" match = re.match(pattern, expression) if match: diff --git a/src/python/data_factory_testing_framework/functions/expressions/expression_dataset.py b/src/python/data_factory_testing_framework/functions/expressions/expression_dataset.py index 7fdbea47..5287bc58 100644 --- a/src/python/data_factory_testing_framework/functions/expressions/expression_dataset.py +++ b/src/python/data_factory_testing_framework/functions/expressions/expression_dataset.py @@ -5,7 +5,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def find_and_replace_dataset(expression: str, state: PipelineRunState): +def find_and_replace_dataset(expression: str, state: PipelineRunState) -> str: pattern = r"(@?{?dataset\(\'(\w+)\'\)}?)" matches = re.finditer(pattern, expression, re.MULTILINE) for match in matches: diff --git a/src/python/data_factory_testing_framework/functions/expressions/expression_iteration_item.py b/src/python/data_factory_testing_framework/functions/expressions/expression_iteration_item.py index 909824e4..a81c449f 100644 --- a/src/python/data_factory_testing_framework/functions/expressions/expression_iteration_item.py +++ b/src/python/data_factory_testing_framework/functions/expressions/expression_iteration_item.py @@ -4,7 +4,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def find_and_replace_iteration_item(expression: str, state: PipelineRunState): +def find_and_replace_iteration_item(expression: str, state: PipelineRunState) -> str: pattern = r"(@?{?item\(\)\}?)" matches = re.finditer(pattern, expression, re.MULTILINE) for match in matches: diff --git a/src/python/data_factory_testing_framework/functions/expressions/expression_linked_service.py b/src/python/data_factory_testing_framework/functions/expressions/expression_linked_service.py index d1fd4cc9..24e89d69 100644 --- a/src/python/data_factory_testing_framework/functions/expressions/expression_linked_service.py +++ b/src/python/data_factory_testing_framework/functions/expressions/expression_linked_service.py @@ -7,7 +7,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def find_and_replace_linked_services(expression: str, state: PipelineRunState): +def find_and_replace_linked_services(expression: str, state: PipelineRunState) -> str: pattern = r"(@?{?linkedService\(\'(\w+)\'\)}?)" matches = re.finditer(pattern, expression, re.MULTILINE) for match in matches: diff --git a/src/python/data_factory_testing_framework/functions/expressions/expression_parameter.py b/src/python/data_factory_testing_framework/functions/expressions/expression_parameter.py index b6ee6560..e9258a2e 100644 --- a/src/python/data_factory_testing_framework/functions/expressions/expression_parameter.py +++ b/src/python/data_factory_testing_framework/functions/expressions/expression_parameter.py @@ -7,7 +7,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def find_and_replace_parameters(expression: str, parameter_type: RunParameterType, state: PipelineRunState): +def find_and_replace_parameters(expression: str, parameter_type: RunParameterType, state: PipelineRunState) -> str: pattern = rf"(@?{{?pipeline\(\)\.{_get_parameter_string_template(parameter_type)}\.(\w+)}}?)" matches = re.finditer(pattern, expression, re.MULTILINE) for match in matches: @@ -24,7 +24,7 @@ def find_and_replace_parameters(expression: str, parameter_type: RunParameterTyp return expression -def _get_parameter_string_template(parameter_type: RunParameterType): +def _get_parameter_string_template(parameter_type: RunParameterType) -> str: if parameter_type == RunParameterType.Pipeline: return "parameters" elif parameter_type == RunParameterType.Global: diff --git a/src/python/data_factory_testing_framework/functions/expressions/expression_variable.py b/src/python/data_factory_testing_framework/functions/expressions/expression_variable.py index a884a9ea..f78e3659 100644 --- a/src/python/data_factory_testing_framework/functions/expressions/expression_variable.py +++ b/src/python/data_factory_testing_framework/functions/expressions/expression_variable.py @@ -4,7 +4,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def find_and_replace_variables(expression: str, state: PipelineRunState): +def find_and_replace_variables(expression: str, state: PipelineRunState) -> str: pattern = r"(@?{?variables\(\'(\w+)\'\)}?)" matches = re.finditer(pattern, expression, re.MULTILINE) for match in matches: diff --git a/src/python/data_factory_testing_framework/functions/function_argument.py b/src/python/data_factory_testing_framework/functions/function_argument.py index 0c544815..c1bdb1f2 100644 --- a/src/python/data_factory_testing_framework/functions/function_argument.py +++ b/src/python/data_factory_testing_framework/functions/function_argument.py @@ -12,7 +12,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def trim_one_char(text: str, character): +def trim_one_char(text: str, character: str) -> str: if text.startswith(character): text = text[1:] if text.endswith(character): @@ -24,7 +24,7 @@ class FunctionArgument: def __init__(self, expression: str) -> None: self.expression = expression.strip("\n").strip(" ") - def evaluate(self, state: PipelineRunState): + def evaluate(self, state: PipelineRunState) -> str: evaluated_expression = find_and_replace_parameters(self.expression, RunParameterType.Pipeline, state) evaluated_expression = find_and_replace_parameters(evaluated_expression, RunParameterType.Global, state) evaluated_expression = find_and_replace_variables(evaluated_expression, state) diff --git a/src/python/data_factory_testing_framework/functions/function_call.py b/src/python/data_factory_testing_framework/functions/function_call.py index fa06b0cb..8e7a6477 100644 --- a/src/python/data_factory_testing_framework/functions/function_call.py +++ b/src/python/data_factory_testing_framework/functions/function_call.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Any, List from data_factory_testing_framework.exceptions.function_call_invalid_arguments_count_error import ( FunctionCallInvalidArgumentsCountError, @@ -14,7 +14,7 @@ def __init__(self, name: str, arguments: List) -> None: self.name = name self.arguments = arguments - def evaluate(self, state): + def evaluate(self, state: Any) -> str: # noqa: ANN401 - what is the type of state? function = FunctionsRepository.functions.get(self.name) if not function: raise UnsupportedFunctionError(self.name) diff --git a/src/python/data_factory_testing_framework/functions/function_parser.py b/src/python/data_factory_testing_framework/functions/function_parser.py index e4a66115..e3964d43 100644 --- a/src/python/data_factory_testing_framework/functions/function_parser.py +++ b/src/python/data_factory_testing_framework/functions/function_parser.py @@ -6,7 +6,7 @@ extract_func_regex = r"^@?{?([^()]+?)\((.*)\)}?$" -def parse_expression(expression: str): +def parse_expression(expression: str) -> FunctionArgument: match = re.match(extract_func_regex, expression, re.DOTALL) if not match: return FunctionArgument(expression) diff --git a/src/python/data_factory_testing_framework/functions/functions_repository.py b/src/python/data_factory_testing_framework/functions/functions_repository.py index 671b9f24..556c3e8d 100644 --- a/src/python/data_factory_testing_framework/functions/functions_repository.py +++ b/src/python/data_factory_testing_framework/functions/functions_repository.py @@ -1,7 +1,7 @@ import json from collections.abc import Iterable as IterableType from datetime import datetime -from typing import Callable, Dict +from typing import Any, Callable, Dict class FunctionsRepository: @@ -35,25 +35,25 @@ class FunctionsRepository: } @staticmethod - def _register(function_name: str, function: Callable): + def _register(function_name: str, function: Callable) -> None: FunctionsRepository.functions[function_name] = function @staticmethod - def _func_equals(argument0, argument1): + def _func_equals(argument0: Any, argument1: Any) -> bool: # noqa: ANN401 if type(argument0) != type(argument1): raise ValueError("Equals function requires arguments of the same type.") return argument0 == argument1 @staticmethod - def _trim(text, trim_argument): + def _trim(text: str, trim_argument: str) -> str: return text.strip(trim_argument[0]) @staticmethod - def _json(argument): + def _json(argument: Any) -> Any: # noqa: ANN401 return argument @staticmethod - def _contains(obj, value): + def _contains(obj: Any, value: Any) -> bool: # noqa: ANN401 if isinstance(obj, dict): return value in obj elif isinstance(obj, IterableType): diff --git a/src/python/data_factory_testing_framework/models/activities/base/activity.py b/src/python/data_factory_testing_framework/models/activities/base/activity.py index 2ff6765f..44cec30a 100644 --- a/src/python/data_factory_testing_framework/models/activities/base/activity.py +++ b/src/python/data_factory_testing_framework/models/activities/base/activity.py @@ -11,7 +11,7 @@ def evaluate(self, state: PipelineRunState) -> Activity: self.status: DependencyCondition = DependencyCondition.Succeeded return self - def evaluate_expressions(self, obj: Any, state: PipelineRunState, visited: List[Any] = None): + def evaluate_expressions(self, obj: Any, state: PipelineRunState, visited: List[Any] = None) -> None: # noqa: ANN401 if visited is None: visited = [] @@ -46,7 +46,7 @@ def evaluate_expressions(self, obj: Any, state: PipelineRunState, visited: List[ else: self.evaluate_expressions(attribute, state, visited) - def are_dependency_condition_met(self, state: PipelineRunState): + def are_dependency_condition_met(self, state: PipelineRunState) -> bool: if not self.depends_on: return True diff --git a/src/python/data_factory_testing_framework/models/activities/control_activities/control_activity.py b/src/python/data_factory_testing_framework/models/activities/control_activities/control_activity.py index 671d8617..3193907c 100644 --- a/src/python/data_factory_testing_framework/models/activities/control_activities/control_activity.py +++ b/src/python/data_factory_testing_framework/models/activities/control_activities/control_activity.py @@ -9,5 +9,5 @@ def evaluate_control_activity_iterations( self, state: PipelineRunState, evaluate_activities: Callable[[PipelineRunState], Generator[Activity, None, None]], - ): + ) -> Generator[Activity, None, None]: return [] diff --git a/src/python/data_factory_testing_framework/models/activities/control_activities/for_each_activity.py b/src/python/data_factory_testing_framework/models/activities/control_activities/for_each_activity.py index 91daaea4..2719d8af 100644 --- a/src/python/data_factory_testing_framework/models/activities/control_activities/for_each_activity.py +++ b/src/python/data_factory_testing_framework/models/activities/control_activities/for_each_activity.py @@ -5,7 +5,7 @@ class ForEachActivity: - def evaluate(self: ForEachActivity, state: PipelineRunState): + def evaluate(self: ForEachActivity, state: PipelineRunState) -> ForEachActivity: self.items.evaluate(state) return super(ControlActivity, self).evaluate(state) @@ -14,7 +14,7 @@ def evaluate_control_activity_iterations( self: ForEachActivity, state: PipelineRunState, evaluate_activities: Callable[[PipelineRunState], Generator[Activity, None, None]], - ): + ) -> Generator[Activity, None, None]: for item in self.items.evaluated: scoped_state = state.create_iteration_scope(item) for activity in evaluate_activities(self.activities, scoped_state): diff --git a/src/python/data_factory_testing_framework/models/activities/control_activities/if_condition_activity.py b/src/python/data_factory_testing_framework/models/activities/control_activities/if_condition_activity.py index b04d495f..d0c83a2d 100644 --- a/src/python/data_factory_testing_framework/models/activities/control_activities/if_condition_activity.py +++ b/src/python/data_factory_testing_framework/models/activities/control_activities/if_condition_activity.py @@ -9,7 +9,7 @@ class IfConditionActivity: - def evaluate(self: IfConditionActivity, state: PipelineRunState): + def evaluate(self: IfConditionActivity, state: PipelineRunState) -> IfConditionActivity: self.expression.evaluate(state) return super(ControlActivity, self).evaluate(state) @@ -18,7 +18,7 @@ def evaluate_control_activity_iterations( self: IfConditionActivity, state: PipelineRunState, evaluate_activities: Callable[[PipelineRunState], Generator[Activity, None, None]], - ): + ) -> Generator[Activity, None, None]: scoped_state = state.create_iteration_scope(None) activities = self.if_true_activities if self.expression.evaluated else self.if_false_activities for activity in evaluate_activities(activities, scoped_state): diff --git a/src/python/data_factory_testing_framework/models/activities/control_activities/until_activity.py b/src/python/data_factory_testing_framework/models/activities/control_activities/until_activity.py index 3cecf595..b662332a 100644 --- a/src/python/data_factory_testing_framework/models/activities/control_activities/until_activity.py +++ b/src/python/data_factory_testing_framework/models/activities/control_activities/until_activity.py @@ -5,7 +5,7 @@ class UntilActivity: - def evaluate(self: UntilActivity, state: PipelineRunState): + def evaluate(self: UntilActivity, state: PipelineRunState) -> UntilActivity: self.expression.evaluate(state) return super(ControlActivity, self).evaluate(state) @@ -14,7 +14,7 @@ def evaluate_control_activity_iterations( self: UntilActivity, state: PipelineRunState, evaluate_activities: Callable[[PipelineRunState], Generator[Activity, None, None]], - ): + ) -> Generator[Activity, None, None]: while True: scoped_state = state.create_iteration_scope(None) for activity in evaluate_activities(self.activities, scoped_state): diff --git a/src/python/data_factory_testing_framework/models/activities/set_variable_activity.py b/src/python/data_factory_testing_framework/models/activities/set_variable_activity.py index 9e9260a6..8d27f18d 100644 --- a/src/python/data_factory_testing_framework/models/activities/set_variable_activity.py +++ b/src/python/data_factory_testing_framework/models/activities/set_variable_activity.py @@ -3,7 +3,7 @@ class SetVariableActivity: - def evaluate(self: SetVariableActivity, state: PipelineRunState): + def evaluate(self: SetVariableActivity, state: PipelineRunState) -> SetVariableActivity: super(ControlActivity, self).evaluate(state) if self.variable_name == "pipelineReturnValue": diff --git a/src/python/data_factory_testing_framework/models/data_factory_element.py b/src/python/data_factory_testing_framework/models/data_factory_element.py index 6ce2037e..f37daa08 100644 --- a/src/python/data_factory_testing_framework/models/data_factory_element.py +++ b/src/python/data_factory_testing_framework/models/data_factory_element.py @@ -1,4 +1,4 @@ -from typing import Generic, TypeVar +from typing import Any, Generic, TypeVar from data_factory_testing_framework.functions.function_parser import parse_expression from data_factory_testing_framework.generated.models import DataFactoryElement @@ -7,6 +7,6 @@ class DataFactoryElement(Generic[T]): - def evaluate(self: DataFactoryElement, state): + def evaluate(self: DataFactoryElement, state: Any) -> None: # noqa: ANN401 self.value = parse_expression(self.expression).evaluate(state) return self.value diff --git a/src/python/data_factory_testing_framework/models/expression.py b/src/python/data_factory_testing_framework/models/expression.py index 950b537c..0acde239 100644 --- a/src/python/data_factory_testing_framework/models/expression.py +++ b/src/python/data_factory_testing_framework/models/expression.py @@ -11,6 +11,6 @@ class Expression[TResult]: def __init__(self) -> None: self.evaluated: TResult = [] - def evaluate(self: Expression, state: PipelineRunState): + def evaluate(self: Expression, state: PipelineRunState) -> TResult: self.evaluated = parse_expression(self.value).evaluate(state) return self.evaluated diff --git a/src/python/data_factory_testing_framework/models/patch_models.py b/src/python/data_factory_testing_framework/models/patch_models.py index ad1eb768..3bf4df9f 100644 --- a/src/python/data_factory_testing_framework/models/patch_models.py +++ b/src/python/data_factory_testing_framework/models/patch_models.py @@ -16,7 +16,7 @@ # Patch models with our custom classes -def patch_models(): +def patch_models() -> None: patch_model(_models.Activity, Activity) patch_model(_models.ExecutePipelineActivity, ExecutePipelineActivity) patch_model(_models.ControlActivity, ControlActivity) @@ -29,7 +29,7 @@ def patch_models(): patch_model(_models.DataFactoryElement, DataFactoryElement) -def patch_model(main_class, partial_class): +def patch_model(main_class: type, partial_class: type) -> None: partial_class_method_list = [ attribute for attribute in dir(partial_class) diff --git a/src/python/data_factory_testing_framework/models/pipelines/pipeline_resource.py b/src/python/data_factory_testing_framework/models/pipelines/pipeline_resource.py index b5eb279a..4d402b6b 100644 --- a/src/python/data_factory_testing_framework/models/pipelines/pipeline_resource.py +++ b/src/python/data_factory_testing_framework/models/pipelines/pipeline_resource.py @@ -14,7 +14,7 @@ def get_activity_by_name(self: PipelineResource, name: str) -> Activity: raise ActivityNotFoundError(f"Activity with name {name} not found") - def validate_parameters(self: PipelineResource, parameters: List[RunParameter]): + def validate_parameters(self: PipelineResource, parameters: List[RunParameter]) -> None: # Check if all parameters are provided for pipeline_parameter_name, pipeline_parameter_specification in self.parameters.items(): found = False diff --git a/src/python/data_factory_testing_framework/models/repositories/data_factory_repository_factory.py b/src/python/data_factory_testing_framework/models/repositories/data_factory_repository_factory.py index 177ab554..9070e6af 100644 --- a/src/python/data_factory_testing_framework/models/repositories/data_factory_repository_factory.py +++ b/src/python/data_factory_testing_framework/models/repositories/data_factory_repository_factory.py @@ -13,7 +13,7 @@ class DataFactoryRepositoryFactory: @staticmethod - def parse_from_folder(folder_path: str): + def parse_from_folder(folder_path: str) -> DataFactoryRepository: pipelines = DataFactoryRepositoryFactory._get_data_factory_entities_by_folder_path( folder_path, "PipelineResource", @@ -21,7 +21,7 @@ def parse_from_folder(folder_path: str): return DataFactoryRepository(pipelines) @staticmethod - def _get_data_factory_entities_by_folder_path(folder_path: str, target_class: str): + def _get_data_factory_entities_by_folder_path(folder_path: str, target_class: str) -> list: entities = [] files = os.listdir(folder_path) for file in files: diff --git a/src/python/data_factory_testing_framework/models/state/pipeline_run_state.py b/src/python/data_factory_testing_framework/models/state/pipeline_run_state.py index 113e795c..8c12e603 100644 --- a/src/python/data_factory_testing_framework/models/state/pipeline_run_state.py +++ b/src/python/data_factory_testing_framework/models/state/pipeline_run_state.py @@ -36,7 +36,7 @@ def __init__( self.scoped_pipeline_activity_results: Dict[str, Any] = {} self.iteration_item = iteration_item - def add_activity_result(self, activity_name: str, status: DependencyCondition, output: Any = None): + def add_activity_result(self, activity_name: str, status: DependencyCondition, output: Any = None) -> None: # noqa: ANN401 self.pipeline_activity_results[activity_name] = { "status": status, "output": output, @@ -46,7 +46,7 @@ def add_activity_result(self, activity_name: str, status: DependencyCondition, o "output": output, } - def create_iteration_scope(self, iteration_item: str): + def create_iteration_scope(self, iteration_item: str) -> "PipelineRunState": return PipelineRunState( self.parameters, self._variable_specifications, @@ -54,14 +54,14 @@ def create_iteration_scope(self, iteration_item: str): iteration_item, ) - def add_scoped_activity_results_from_scoped_state(self, scoped_state): + def add_scoped_activity_results_from_scoped_state(self, scoped_state: "PipelineRunState") -> None: for result in scoped_state.pipeline_activity_results: self.pipeline_activity_results[result] = scoped_state.pipeline_activity_results[result] - def try_get_scoped_activity_result_by_name(self, name: str): + def try_get_scoped_activity_result_by_name(self, name: str) -> Optional[Dict[str, Any]]: return self.pipeline_activity_results[name] if name in self.pipeline_activity_results else None - def set_variable(self, variable_name: str, value): + def set_variable(self, variable_name: str, value: Any) -> None: # noqa: ANN401 for variable in self.variables: if variable.name == variable_name: variable.value = value @@ -69,7 +69,7 @@ def set_variable(self, variable_name: str, value): raise VariableBeingEvaluatedDoesNotExistError(variable_name) - def get_variable_by_name(self, variable_name): + def get_variable_by_name(self, variable_name: str) -> PipelineRunVariable: for variable in self.variables: if variable.name == variable_name: return variable diff --git a/src/python/data_factory_testing_framework/scripts/utils/ast_utils.py b/src/python/data_factory_testing_framework/scripts/utils/ast_utils.py index 44923cd9..a3ca72aa 100644 --- a/src/python/data_factory_testing_framework/scripts/utils/ast_utils.py +++ b/src/python/data_factory_testing_framework/scripts/utils/ast_utils.py @@ -4,7 +4,7 @@ target_type = "DataFactoryElement" -def get_type_from_description(description): +def get_type_from_description(description: str) -> str: dfe_type = "str" if "Expression with resultType string" in description: dfe_type = "str" @@ -16,7 +16,7 @@ def get_type_from_description(description): return dfe_type -def update_attribute_type(arg, description): +def update_attribute_type(arg: ast.arg, description: str) -> None: dfe_type = get_type_from_description(description) if "Expression" in description: if ( @@ -39,7 +39,7 @@ def update_attribute_type(arg, description): ) -def transform_ast(node): +def transform_ast(node: ast.ClassDef) -> None: if isinstance(node, ast.ClassDef): for body_item in node.body: if isinstance(body_item, ast.FunctionDef): diff --git a/src/python/data_factory_testing_framework/scripts/utils/docstring_utils.py b/src/python/data_factory_testing_framework/scripts/utils/docstring_utils.py index e2e760f2..d0fe7f02 100644 --- a/src/python/data_factory_testing_framework/scripts/utils/docstring_utils.py +++ b/src/python/data_factory_testing_framework/scripts/utils/docstring_utils.py @@ -7,7 +7,7 @@ PARAM_REGEX = re.compile(":ivar (?P[\*\w]+): (?P.*?)" "(?:(?=:ivar)|(?=:return)|(?=:raises)|\Z)", re.S) -def parse_restructured_docstring(docstring): +def parse_restructured_docstring(docstring: str) -> dict: """Parse the docstring into its components. :returns: a dictionary of form diff --git a/src/python/data_factory_testing_framework/scripts/utils/string_utils.py b/src/python/data_factory_testing_framework/scripts/utils/string_utils.py index 62395a9c..e7f47c47 100644 --- a/src/python/data_factory_testing_framework/scripts/utils/string_utils.py +++ b/src/python/data_factory_testing_framework/scripts/utils/string_utils.py @@ -1,7 +1,7 @@ import sys -def trim(docstring): +def trim(docstring: str) -> str: """Trim function from PEP-257.""" if not docstring: return "" @@ -35,5 +35,5 @@ def trim(docstring): return "\n".join(trimmed) -def reindent(string): +def reindent(string: str) -> str: return "\n".join(l.strip() for l in string.strip().split("\n")) diff --git a/src/python/example/batch_job/batchjob_test.py b/src/python/example/batch_job/batchjob_test.py index b50ad368..d26dfc4b 100644 --- a/src/python/example/batch_job/batchjob_test.py +++ b/src/python/example/batch_job/batchjob_test.py @@ -6,7 +6,7 @@ from data_factory_testing_framework.models.test_framework import TestFramework -def test_batch_job_pipeline(): +def test_batch_job_pipeline() -> None: # Arrange test_framework = TestFramework("pipelines") pipeline = test_framework.repository.get_pipeline_by_name("batch_job") diff --git a/src/python/tests/functions/test_function_argument.py b/src/python/tests/functions/test_function_argument.py index 515ee1ed..682c4d68 100644 --- a/src/python/tests/functions/test_function_argument.py +++ b/src/python/tests/functions/test_function_argument.py @@ -15,7 +15,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def test_evaluate_parameter_expression(): +def test_evaluate_parameter_expression() -> None: # Arrange expression = "pipeline().parameters.parameterName" argument = FunctionArgument(expression) @@ -29,7 +29,7 @@ def test_evaluate_parameter_expression(): assert evaluated == "parameterValue" -def test_evaluate_global_parameter_expression(): +def test_evaluate_global_parameter_expression() -> None: # Arrange expression = "pipeline().globalParameters.parameterName" argument = FunctionArgument(expression) @@ -43,7 +43,7 @@ def test_evaluate_global_parameter_expression(): assert evaluated == "parameterValue" -def test_evaluate_variable_string_expression(): +def test_evaluate_variable_string_expression() -> None: # Arrange expression = "variables('variableName')" argument = FunctionArgument(expression) @@ -60,7 +60,7 @@ def test_evaluate_variable_string_expression(): assert evaluated == "variableValue" -def test_evaluate_linked_service_string_expression(): +def test_evaluate_linked_service_string_expression() -> None: # Arrange expression = "@linkedService('linkedServiceName')" argument = FunctionArgument(expression) @@ -74,7 +74,7 @@ def test_evaluate_linked_service_string_expression(): assert evaluated == "linkedServiceNameValue" -def test_evaluate_dataset_string_expression(): +def test_evaluate_dataset_string_expression() -> None: # Arrange expression = "dataset('datasetName')" argument = FunctionArgument(expression) @@ -88,7 +88,7 @@ def test_evaluate_dataset_string_expression(): assert evaluated == "datasetNameValue" -def test_evaluate_iteration_item_string_expression(): +def test_evaluate_iteration_item_string_expression() -> None: # Arrange expression = "item()" argument = FunctionArgument(expression) @@ -102,7 +102,7 @@ def test_evaluate_iteration_item_string_expression(): assert evaluated == "iterationItemValue" -def test_evaluate_unknown_pipeline_parameter(): +def test_evaluate_unknown_pipeline_parameter() -> None: # Arrange expression = "pipeline().parameters.parameterName" argument = FunctionArgument(expression) @@ -114,7 +114,7 @@ def test_evaluate_unknown_pipeline_parameter(): print("hi") -def test_evaluate_unknown_global_pipeline_parameter(): +def test_evaluate_unknown_global_pipeline_parameter() -> None: # Arrange expression = "pipeline().globalParameters.parameterName" argument = FunctionArgument(expression) @@ -125,7 +125,7 @@ def test_evaluate_unknown_global_pipeline_parameter(): argument.evaluate(state) -def test_evaluate_unknown_variable(): +def test_evaluate_unknown_variable() -> None: # Arrange expression = "variables('variableName')" argument = FunctionArgument(expression) @@ -136,7 +136,7 @@ def test_evaluate_unknown_variable(): argument.evaluate(state) -def test_evaluate_unknown_dataset(): +def test_evaluate_unknown_dataset() -> None: # Arrange expression = "dataset('datasetName')" argument = FunctionArgument(expression) @@ -147,7 +147,7 @@ def test_evaluate_unknown_dataset(): argument.evaluate(state) -def test_evaluate_unknown_linked_service(): +def test_evaluate_unknown_linked_service() -> None: # Arrange expression = "linkedService('linkedServiceName')" argument = FunctionArgument(expression) @@ -158,7 +158,7 @@ def test_evaluate_unknown_linked_service(): argument.evaluate(state) -def test_evaluate_activity_output_expression(): +def test_evaluate_activity_output_expression() -> None: # Arrange expression = "activity('activityName').output.outputName" argument = FunctionArgument(expression) @@ -172,7 +172,7 @@ def test_evaluate_activity_output_expression(): assert evaluated == "outputValue" -def test_evaluate_activity_output_nested_expression(): +def test_evaluate_activity_output_nested_expression() -> None: # Arrange expression = "activity('activityName').output.nestedOutput.nestedField" argument = FunctionArgument(expression) @@ -190,7 +190,7 @@ def test_evaluate_activity_output_nested_expression(): assert evaluated == "outputValue" -def test_evaluate_complex_json_expression(): +def test_evaluate_complex_json_expression() -> None: # Arrange expression = ( '" { "command": "@pipeline().globalParameters.command", "argument": @pipeline().parameters.argument } "' diff --git a/src/python/tests/functions/test_function_call.py b/src/python/tests/functions/test_function_call.py index 66a7a514..8ee2b12d 100644 --- a/src/python/tests/functions/test_function_call.py +++ b/src/python/tests/functions/test_function_call.py @@ -10,7 +10,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def test_evaluate_expression_with_nested_function(): +def test_evaluate_expression_with_nested_function() -> None: # Arrange raw_expression = "concat('https://example.com/jobs/,', '123''', concat('&', 'abc,'))" expression = parse_expression(raw_expression) @@ -22,7 +22,7 @@ def test_evaluate_expression_with_nested_function(): assert evaluated == "https://example.com/jobs/,123'&abc," -def test_evaluate_with_parameter(): +def test_evaluate_with_parameter() -> None: # Arrange raw_expression = "concat('https://example.com/jobs/', pipeline().parameters.abc)" expression = parse_expression(raw_expression) @@ -39,7 +39,7 @@ def test_evaluate_with_parameter(): assert evaluated == "https://example.com/jobs/123" -def test_evaluate_with_global_parameter(): +def test_evaluate_with_global_parameter() -> None: # Arrange raw_expression = "concat('https://example.com/jobs/', pipeline().globalParameters.abc)" expression = parse_expression(raw_expression) @@ -56,7 +56,7 @@ def test_evaluate_with_global_parameter(): assert evaluated == "https://example.com/jobs/123" -def test_evaluate_with_variable(): +def test_evaluate_with_variable() -> None: # Arrange raw_expression = "concat('https://example.com/jobs/', variables('abc'))" expression = parse_expression(raw_expression) @@ -73,7 +73,7 @@ def test_evaluate_with_variable(): assert evaluated == "https://example.com/jobs/123" -def test_evaluate_with_activity_output(): +def test_evaluate_with_activity_output() -> None: # Arrange raw_expression = "concat('https://example.com/jobs/', activity('abc').output.abc)" expression = parse_expression(raw_expression) @@ -87,7 +87,7 @@ def test_evaluate_with_activity_output(): assert evaluated == "https://example.com/jobs/123" -def test_evaluate_with_activity_output_and_variable(): +def test_evaluate_with_activity_output_and_variable() -> None: # Arrange raw_expression = "concat('https://example.com/jobs/', activity('abc').output.abc, '/', variables('abc'))" expression = parse_expression(raw_expression) @@ -105,7 +105,7 @@ def test_evaluate_with_activity_output_and_variable(): assert evaluated == "https://example.com/jobs/123/456" -def test_evaluate_with_activity_output_and_variable_and_parameters(): +def test_evaluate_with_activity_output_and_variable_and_parameters() -> None: # Arrange raw_expression = ( "concat('https://example.com/jobs/', activity('abc').output.abc, '/', " @@ -139,7 +139,7 @@ def test_evaluate_with_activity_output_and_variable_and_parameters(): ("1", "2", False), ], ) -def test_evaluate_equals_expression(left, right, expected): +def test_evaluate_equals_expression(left: str, right: str, expected: bool) -> None: # Arrange raw_expression = f"equals({left}, {right})" expression = parse_expression(raw_expression) @@ -160,7 +160,7 @@ def test_evaluate_equals_expression(left, right, expected): (0, -1, False), ], ) -def test_evaluate_equals_int_expression(left, right, expected): +def test_evaluate_equals_int_expression(left: int, right: int, expected: bool) -> None: # Arrange raw_expression = f"equals({left}, {right})" expression = parse_expression(raw_expression) @@ -180,7 +180,7 @@ def test_evaluate_equals_int_expression(left, right, expected): ("SomeKey3", False), ], ) -def test_contains_dictionary_key_expression(key, expected): +def test_contains_dictionary_key_expression(key: str, expected: str) -> None: # Arrange state = PipelineRunState() state.add_activity_result( @@ -206,7 +206,7 @@ def test_contains_dictionary_key_expression(key, expected): ("SomeItem3", False), ], ) -def test_contains_list_item_expression(key, expected): +def test_contains_list_item_expression(key: str, expected: bool) -> None: # Arrange state = PipelineRunState() state.add_activity_result( @@ -231,7 +231,7 @@ def test_contains_list_item_expression(key, expected): ("NotPartOfString", False), ], ) -def test_contains_string_expression(substring, expected): +def test_contains_string_expression(substring: str, expected: bool) -> None: # Arrange state = PipelineRunState() state.add_activity_result( @@ -249,7 +249,7 @@ def test_contains_string_expression(substring, expected): assert evaluated == expected -def test_function_call_wrong_arguments_error(): +def test_function_call_wrong_arguments_error() -> None: # Arrange raw_expression = "trim('abc', 'a', 'b')" expression = parse_expression(raw_expression) diff --git a/src/python/tests/functions/test_function_parser.py b/src/python/tests/functions/test_function_parser.py index dbf75878..105ffdbd 100644 --- a/src/python/tests/functions/test_function_parser.py +++ b/src/python/tests/functions/test_function_parser.py @@ -3,7 +3,7 @@ from data_factory_testing_framework.models.state.pipeline_run_state import PipelineRunState -def test_parse_expression_with_nested_function_and_single_quote(): +def test_parse_expression_with_nested_function_and_single_quote() -> None: # Arrange PipelineRunState() raw_expression = "concat('https://example.com/jobs/', '123''', concat('&', 'abc,'))" @@ -28,7 +28,7 @@ def test_parse_expression_with_nested_function_and_single_quote(): assert inner_function.arguments[1].expression == "'abc,'" -def test_parse_expression_with_adf_native_functions(): +def test_parse_expression_with_adf_native_functions() -> None: # Arrange PipelineRunState() raw_expression = "concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)" diff --git a/src/python/tests/models/activities/base/test_activity.py b/src/python/tests/models/activities/base/test_activity.py index 5bbff89f..7b2393c7 100644 --- a/src/python/tests/models/activities/base/test_activity.py +++ b/src/python/tests/models/activities/base/test_activity.py @@ -32,7 +32,9 @@ ("Completed", "Completed", True), ], ) -def test_dependency_conditions_when_called_returns_expected(required_condition, actual_condition, expected): +def test_dependency_conditions_when_called_returns_expected( + required_condition: str, actual_condition: str, expected: bool +) -> None: # Arrange pipeline_activity = Activity( name="activity", @@ -51,7 +53,7 @@ def test_dependency_conditions_when_called_returns_expected(required_condition, assert result == expected -def test_evaluate_when_no_status_is_set_should_set_status_to_succeeded(): +def test_evaluate_when_no_status_is_set_should_set_status_to_succeeded() -> None: # Arrange pipeline_activity = Activity(name="activity", depends_on=[]) state = PipelineRunState() @@ -63,7 +65,7 @@ def test_evaluate_when_no_status_is_set_should_set_status_to_succeeded(): assert pipeline_activity.status == DependencyCondition.Succeeded -def test_evaluate_is_evaluating_expressions_inside_dict(): +def test_evaluate_is_evaluating_expressions_inside_dict() -> None: # Arrange pipeline_activity = ExecutePipelineActivity( name="activity", diff --git a/src/python/tests/models/activities/control_activities/test_for_each_activity.py b/src/python/tests/models/activities/control_activities/test_for_each_activity.py index 02f6295a..87e1c3a2 100644 --- a/src/python/tests/models/activities/control_activities/test_for_each_activity.py +++ b/src/python/tests/models/activities/control_activities/test_for_each_activity.py @@ -12,7 +12,7 @@ from data_factory_testing_framework.models.test_framework import TestFramework -def test_when_evaluate_child_activities_then_should_return_the_activity_with_item_expression_evaluated(): +def test_when_evaluate_child_activities_then_should_return_the_activity_with_item_expression_evaluated() -> None: # Arrange test_framework = TestFramework() for_each_activity = ForEachActivity( diff --git a/src/python/tests/models/activities/control_activities/test_if_condition_activity.py b/src/python/tests/models/activities/control_activities/test_if_condition_activity.py index ecdeb5a8..2596bdf8 100644 --- a/src/python/tests/models/activities/control_activities/test_if_condition_activity.py +++ b/src/python/tests/models/activities/control_activities/test_if_condition_activity.py @@ -14,7 +14,7 @@ TestFramework() -def test_when_evaluated_should_evaluate_expression(): +def test_when_evaluated_should_evaluate_expression() -> None: # Arrange activity = IfConditionActivity( name="IfConditionActivity", @@ -32,7 +32,9 @@ def test_when_evaluated_should_evaluate_expression(): "expression_outcome,expected_activity_name", [(True, "setVariableActivity1"), (False, "setVariableActivity2")], ) -def test_when_evaluated_should_evaluate_correct_child_activities(expression_outcome, expected_activity_name): +def test_when_evaluated_should_evaluate_correct_child_activities( + expression_outcome: bool, expected_activity_name: str +) -> None: # Arrange test_framework = TestFramework() expression = "@equals(1, 1)" if expression_outcome else "@equals(1, 2)" diff --git a/src/python/tests/models/activities/control_activities/test_until_activity.py b/src/python/tests/models/activities/control_activities/test_until_activity.py index 544dbc64..d5327fa5 100644 --- a/src/python/tests/models/activities/control_activities/test_until_activity.py +++ b/src/python/tests/models/activities/control_activities/test_until_activity.py @@ -12,7 +12,7 @@ from data_factory_testing_framework.models.test_framework import TestFramework -def test_when_evaluate_until_activity_should_repeat_until_expression_is_true(): +def test_when_evaluate_until_activity_should_repeat_until_expression_is_true() -> None: # Arrange test_framework = TestFramework() until_activity = UntilActivity( diff --git a/src/python/tests/models/activities/test_set_variable_activity.py b/src/python/tests/models/activities/test_set_variable_activity.py index 13035ea0..abe7d800 100644 --- a/src/python/tests/models/activities/test_set_variable_activity.py +++ b/src/python/tests/models/activities/test_set_variable_activity.py @@ -12,7 +12,7 @@ from data_factory_testing_framework.models.test_framework import TestFramework -def test_when_string_variable_evaluated_then_state_variable_should_be_set(): +def test_when_string_variable_evaluated_then_state_variable_should_be_set() -> None: # Arrange TestFramework() variable_name = "TestVariable" @@ -35,7 +35,7 @@ def test_when_string_variable_evaluated_then_state_variable_should_be_set(): assert variable.value == "TestValue" -def test_when_unknown_variable_evaluated_then_should_raise_exception(): +def test_when_unknown_variable_evaluated_then_should_raise_exception() -> None: # Arrange TestFramework() variable_name = "TestVariable" diff --git a/src/python/tests/models/pipelines/test_pipeline_resource.py b/src/python/tests/models/pipelines/test_pipeline_resource.py index b541e7ba..fee758f5 100644 --- a/src/python/tests/models/pipelines/test_pipeline_resource.py +++ b/src/python/tests/models/pipelines/test_pipeline_resource.py @@ -8,7 +8,7 @@ TestFramework() -def test_when_validate_parameters_is_accurate_should_pass(): +def test_when_validate_parameters_is_accurate_should_pass() -> None: # Arrange pipeline = PipelineResource( name="pipeline", @@ -27,7 +27,7 @@ def test_when_validate_parameters_is_accurate_should_pass(): ) -def test_when_validate_parameters_is_missing_run_parameter_should_throw_error(): +def test_when_validate_parameters_is_missing_run_parameter_should_throw_error() -> None: # Arrange pipeline = PipelineResource( parameters={ @@ -52,7 +52,7 @@ def test_when_validate_parameters_is_missing_run_parameter_should_throw_error(): ) -def test_when_duplicate_parameters_supplied_should_throw_error(): +def test_when_duplicate_parameters_supplied_should_throw_error() -> None: # Arrange pipeline = PipelineResource( parameters={