From a2351fac1bb71c3fe8f503eda40e99f4d397f166 Mon Sep 17 00:00:00 2001 From: Jerry Sun Date: Tue, 9 Jan 2024 00:43:31 -0800 Subject: [PATCH 1/4] completed draft for function level env var --- docs/design.md | 7 +++++++ .../deployment/client/cli/template/app.py | 6 ++++++ multi_x_serverless/deployment/client/config.py | 6 ++++-- .../client/deploy/workflow_builder.py | 17 ++++++++++++++++- .../client/multi_x_serverless_function.py | 2 ++ .../client/multi_x_serverless_workflow.py | 9 +++++++-- 6 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/design.md b/docs/design.md index bcac7a5c..6a149347 100644 --- a/docs/design.md +++ b/docs/design.md @@ -82,6 +82,12 @@ workflow = MultiXServerlessWorkflow("workflow_name") "only_regions": [["aws", "us-east-1"], ["aws", "us-east-2"], ["aws", "us-west-1"], ["aws", "us-west-2"]], "forbidden_regions": None, }, + func_environment_variables=[ + { + "key": "example_key", + "value": "example_value" + } + ] providers=[ { "name": "aws", @@ -101,6 +107,7 @@ The meaning of the different parameters is as follows: - `regions_and_providers`: A dictionary that contains the regions and providers that the function can be deployed to. This can be used to override the global settings in the `config.yml`. If none or an empty dictionary is provided, the global config takes precedence. The dictionary has two keys: - `only_regions`: A list of regions that the function can be deployed to. If this list is empty, the function can be deployed to any region. - `forbidden_regions`: A list of regions that the function cannot be deployed to. If this list is empty, the function can be deployed to any region. +- `func_environment_variables`: A list of dictionaries that allows users to provide environment variables on the function level. - `providers`: A list of providers that the function can be deployed to. This can be used to override the global settings in the `config.yml`. If a list of providers is specified at the function level this takes precedence over the global configurations. If none or an empty list is provided, the global config takes precedence. Each provider is a dictionary with two keys: - `name`: The name of the provider. This is the name that is used directly in the physical representation of the workflow. - `config`: A dictionary that contains the configuration for the specific provider. diff --git a/multi_x_serverless/deployment/client/cli/template/app.py b/multi_x_serverless/deployment/client/cli/template/app.py index 750b05a0..6769e3f1 100644 --- a/multi_x_serverless/deployment/client/cli/template/app.py +++ b/multi_x_serverless/deployment/client/cli/template/app.py @@ -21,6 +21,12 @@ } ], }, + func_environment_variables=[ + { + "key": "example_key", + "value": "example_value" + } + ] ) def first_function(event: dict[str, Any]) -> dict[str, Any]: payload = { diff --git a/multi_x_serverless/deployment/client/config.py b/multi_x_serverless/deployment/client/config.py index f3fd3834..1c1933ef 100644 --- a/multi_x_serverless/deployment/client/config.py +++ b/multi_x_serverless/deployment/client/config.py @@ -42,12 +42,14 @@ def python_version(self) -> str: return "python3.11" @property - def environment_variables(self) -> dict[str, Any]: + def environment_variables(self) -> dict[str, str]: list_of_env_variables: list[dict] = self._lookup("environment_variables") if list_of_env_variables is None: return {} - env_variables: dict[str, Any] = {} + env_variables: dict[str, str] = {} for env_variable in list_of_env_variables: + if not isinstance(env_variable["value"], str): + raise RuntimeError("Environment variable value need to be a str") env_variables[env_variable["name"]] = env_variable["value"] return env_variables diff --git a/multi_x_serverless/deployment/client/deploy/workflow_builder.py b/multi_x_serverless/deployment/client/deploy/workflow_builder.py index efbecc80..2c3bca72 100644 --- a/multi_x_serverless/deployment/client/deploy/workflow_builder.py +++ b/multi_x_serverless/deployment/client/deploy/workflow_builder.py @@ -38,11 +38,14 @@ def build_workflow(self, config: Config) -> Workflow: # pylint: disable=too-man else: providers = config.regions_and_providers["providers"] self._verify_providers(providers) + + # TODO (#22): merge list here (create a function) get env var from function.func_environment_variables, merge to config.environment_variables + merged_env_vars = self.merge_environment_variables(function.func_environment_variables, config.environment_variables) resources.append( Function( name=function_deployment_name, # TODO (#22): Add function specific environment variables - environment_variables=config.environment_variables, + environment_variables=merged_env_vars, runtime=config.python_version, handler=function.handler, role=function_role, @@ -134,3 +137,15 @@ def get_function_role(self, config: Config, function_name: str) -> IAMRole: filename = os.path.join(config.project_dir, ".multi-x-serverless", "iam_policy.yml") return IAMRole(role_name=role_name, policy=filename) + + def merge_environment_variables(self, function_env_vars: list[dict[str, str]], config_env_vars: dict[str, str]) -> dict[str, str]: + if not function_env_vars: + return config_env_vars + + merged_env_vars: dict[str, str] = config_env_vars + # overwrite config env vars with function env vars if duplicate + for env_var in function_env_vars: + merged_env_vars[env_var["key"]] = env_var["value"] + + return merged_env_vars + diff --git a/multi_x_serverless/deployment/client/multi_x_serverless_function.py b/multi_x_serverless/deployment/client/multi_x_serverless_function.py index 7ab7ea33..9116b72a 100644 --- a/multi_x_serverless/deployment/client/multi_x_serverless_function.py +++ b/multi_x_serverless/deployment/client/multi_x_serverless_function.py @@ -13,12 +13,14 @@ def __init__( name: str, entry_point: bool, regions_and_providers: dict, + func_environment_variables: list[dict[str, str]], ): self.function_callable = function_callable self.name = name self.entry_point = entry_point self.handler = function_callable.__name__ self.regions_and_providers = regions_and_providers if len(regions_and_providers) > 0 else None + self.func_environment_variables = func_environment_variables if len(func_environment_variables) > 0 else None self.validate_function_name() def validate_function_name(self) -> None: diff --git a/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py b/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py index bb91bebe..3f6900d6 100644 --- a/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py +++ b/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py @@ -296,6 +296,7 @@ def register_function( name: str, entry_point: bool, regions_and_providers: dict, + func_environment_variables: list[dict[str, str]], ) -> None: """ Register a function as a serverless function. @@ -305,7 +306,7 @@ def register_function( At this point we only need to register the function with the wrapper, the actual deployment will be done later by the deployment manager. """ - wrapper = MultiXServerlessFunction(function, name, entry_point, regions_and_providers) + wrapper = MultiXServerlessFunction(function, name, entry_point, regions_and_providers, func_environment_variables) self.functions[function.__name__] = wrapper # TODO (#22): Add function specific environment variables @@ -314,6 +315,7 @@ def serverless_function( name: Optional[str] = None, entry_point: bool = False, regions_and_providers: Optional[dict] = None, + func_environment_variables: Optional[list[dict[str, str]]] = None, ) -> Callable[..., Any]: """ Decorator to register a function as a Lambda function. @@ -351,6 +353,9 @@ def serverless_function( if regions_and_providers is None: regions_and_providers = {} + if func_environment_variables is None: + func_environment_variables = [] + def _register_handler(func: Callable[..., Any]) -> Callable[..., Any]: handler_name = name if name is not None else func.__name__ @@ -377,7 +382,7 @@ def wrapper(*args, **kwargs): # type: ignore # pylint: disable=unused-argument wrapper.routing_decision = {} # type: ignore wrapper.entry_point = entry_point # type: ignore wrapper.original_function = func # type: ignore - self.register_function(func, handler_name, entry_point, regions_and_providers) + self.register_function(func, handler_name, entry_point, regions_and_providers, func_environment_variables) return wrapper return _register_handler From 2379be0fd85c6e1b2bd5c3729e975101e3aebe40 Mon Sep 17 00:00:00 2001 From: Jerry Sun Date: Wed, 10 Jan 2024 11:06:03 -0800 Subject: [PATCH 2/4] completed func level env variable + added new tests --- docs/design.md | 2 +- .../deployment/client/cli/config_schema.py | 2 +- .../template/.multi-x-serverless/config.yml | 2 +- .../deployment/client/cli/template/app.py | 7 +- .../deployment/client/config.py | 2 +- .../client/deploy/workflow_builder.py | 19 ++--- .../client/multi_x_serverless_workflow.py | 16 +++- .../client/deploy/test_workflow_builder.py | 2 + .../test_workflow_builder_func_env_var.py | 81 +++++++++++++++++++ .../tests/deployment/client/test_config.py | 2 +- .../test_multi_x_serverless_function.py | 24 +++++- .../test_multi_x_serverless_workflow.py | 75 ++++++++++++++++- 12 files changed, 205 insertions(+), 29 deletions(-) create mode 100644 multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py diff --git a/docs/design.md b/docs/design.md index 6a149347..8285c3af 100644 --- a/docs/design.md +++ b/docs/design.md @@ -107,7 +107,7 @@ The meaning of the different parameters is as follows: - `regions_and_providers`: A dictionary that contains the regions and providers that the function can be deployed to. This can be used to override the global settings in the `config.yml`. If none or an empty dictionary is provided, the global config takes precedence. The dictionary has two keys: - `only_regions`: A list of regions that the function can be deployed to. If this list is empty, the function can be deployed to any region. - `forbidden_regions`: A list of regions that the function cannot be deployed to. If this list is empty, the function can be deployed to any region. -- `func_environment_variables`: A list of dictionaries that allows users to provide environment variables on the function level. +- `func_environment_variables`: This parameter represents a list of dictionaries, each designed for setting environment variables specifically for a function. Users must adhere to a structured format within each dictionary. This format requires two entries: "key" and "value". The "key" entry should contain the name of the environment variable, serving as an identifier. The "value" entry holds the corresponding value assigned to that variable. - `providers`: A list of providers that the function can be deployed to. This can be used to override the global settings in the `config.yml`. If a list of providers is specified at the function level this takes precedence over the global configurations. If none or an empty list is provided, the global config takes precedence. Each provider is a dictionary with two keys: - `name`: The name of the provider. This is the name that is used directly in the physical representation of the workflow. - `config`: A dictionary that contains the configuration for the specific provider. diff --git a/multi_x_serverless/deployment/client/cli/config_schema.py b/multi_x_serverless/deployment/client/cli/config_schema.py index ad77bc39..fc48a3cf 100644 --- a/multi_x_serverless/deployment/client/cli/config_schema.py +++ b/multi_x_serverless/deployment/client/cli/config_schema.py @@ -4,7 +4,7 @@ class EnvironmentVariable(BaseModel): - name: str = Field(..., title="The name of the environment variable") + key: str = Field(..., title="The name of the environment variable") value: str = Field(..., title="The value of the environment variable") diff --git a/multi_x_serverless/deployment/client/cli/template/.multi-x-serverless/config.yml b/multi_x_serverless/deployment/client/cli/template/.multi-x-serverless/config.yml index e1448edd..64a67f71 100644 --- a/multi_x_serverless/deployment/client/cli/template/.multi-x-serverless/config.yml +++ b/multi_x_serverless/deployment/client/cli/template/.multi-x-serverless/config.yml @@ -1,6 +1,6 @@ workflow_name: "{{ workflow_name }}" environment_variables: - - name: "ENV_VAR_1" + - key: "ENV_VAR_1" value: "value_1" iam_policy_file: "iam_policy.json" home_regions: [["aws", us-west-2"]] # Regions are defined as "provider:region" (e.g. aws:us-west-2) diff --git a/multi_x_serverless/deployment/client/cli/template/app.py b/multi_x_serverless/deployment/client/cli/template/app.py index 6769e3f1..4e4e3033 100644 --- a/multi_x_serverless/deployment/client/cli/template/app.py +++ b/multi_x_serverless/deployment/client/cli/template/app.py @@ -21,12 +21,7 @@ } ], }, - func_environment_variables=[ - { - "key": "example_key", - "value": "example_value" - } - ] + func_environment_variables=[{"key": "example_key", "value": "example_value"}], ) def first_function(event: dict[str, Any]) -> dict[str, Any]: payload = { diff --git a/multi_x_serverless/deployment/client/config.py b/multi_x_serverless/deployment/client/config.py index 1c1933ef..15cd0d01 100644 --- a/multi_x_serverless/deployment/client/config.py +++ b/multi_x_serverless/deployment/client/config.py @@ -50,7 +50,7 @@ def environment_variables(self) -> dict[str, str]: for env_variable in list_of_env_variables: if not isinstance(env_variable["value"], str): raise RuntimeError("Environment variable value need to be a str") - env_variables[env_variable["name"]] = env_variable["value"] + env_variables[env_variable["key"]] = env_variable["value"] return env_variables @property diff --git a/multi_x_serverless/deployment/client/deploy/workflow_builder.py b/multi_x_serverless/deployment/client/deploy/workflow_builder.py index 2c3bca72..5113ca2a 100644 --- a/multi_x_serverless/deployment/client/deploy/workflow_builder.py +++ b/multi_x_serverless/deployment/client/deploy/workflow_builder.py @@ -39,12 +39,12 @@ def build_workflow(self, config: Config) -> Workflow: # pylint: disable=too-man providers = config.regions_and_providers["providers"] self._verify_providers(providers) - # TODO (#22): merge list here (create a function) get env var from function.func_environment_variables, merge to config.environment_variables - merged_env_vars = self.merge_environment_variables(function.func_environment_variables, config.environment_variables) + merged_env_vars = self.merge_environment_variables( + function.func_environment_variables, config.environment_variables + ) resources.append( Function( name=function_deployment_name, - # TODO (#22): Add function specific environment variables environment_variables=merged_env_vars, runtime=config.python_version, handler=function.handler, @@ -137,15 +137,16 @@ def get_function_role(self, config: Config, function_name: str) -> IAMRole: filename = os.path.join(config.project_dir, ".multi-x-serverless", "iam_policy.yml") return IAMRole(role_name=role_name, policy=filename) - - def merge_environment_variables(self, function_env_vars: list[dict[str, str]], config_env_vars: dict[str, str]) -> dict[str, str]: + + def merge_environment_variables( + self, function_env_vars: list[dict[str, str]] | None, config_env_vars: dict[str, str] + ) -> dict[str, str]: if not function_env_vars: return config_env_vars - - merged_env_vars: dict[str, str] = config_env_vars + + merged_env_vars: dict[str, str] = dict(config_env_vars) # overwrite config env vars with function env vars if duplicate for env_var in function_env_vars: merged_env_vars[env_var["key"]] = env_var["value"] - + return merged_env_vars - diff --git a/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py b/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py index 3f6900d6..85015acd 100644 --- a/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py +++ b/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py @@ -306,7 +306,9 @@ def register_function( At this point we only need to register the function with the wrapper, the actual deployment will be done later by the deployment manager. """ - wrapper = MultiXServerlessFunction(function, name, entry_point, regions_and_providers, func_environment_variables) + wrapper = MultiXServerlessFunction( + function, name, entry_point, regions_and_providers, func_environment_variables + ) self.functions[function.__name__] = wrapper # TODO (#22): Add function specific environment variables @@ -355,6 +357,18 @@ def serverless_function( if func_environment_variables is None: func_environment_variables = [] + else: + if not isinstance(func_environment_variables, list): + raise RuntimeError("func_environment_variables must be a list of dicts") + for env_variable in func_environment_variables: + if not isinstance(env_variable, dict): + raise RuntimeError("func_environment_variables must be a list of dicts") + if "key" not in env_variable or "value" not in env_variable: + raise RuntimeError("func_environment_variables must be a list of dicts with keys 'key' and 'value'") + if not isinstance(env_variable["key"], str): + raise RuntimeError("func_environment_variables must be a list of dicts with 'key' as a string") + if not isinstance(env_variable["value"], str): + raise RuntimeError("func_environment_variables must be a list of dicts with 'value' as a string") def _register_handler(func: Callable[..., Any]) -> Callable[..., Any]: handler_name = name if name is not None else func.__name__ diff --git a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py index 514a5834..37a0d738 100644 --- a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py +++ b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py @@ -29,11 +29,13 @@ def test_build_workflow_multiple_entry_points(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.func_environment_variables = {} function2 = Mock(spec=MultiXServerlessFunction) function2.entry_point = True function2.name = "function2" function2.handler = "function1" function2.regions_and_providers = {"providers": []} + function2.func_environment_variables = {} self.config.workflow_app.functions = {"function1": function1, "function2": function2} with self.assertRaises(RuntimeError): self.builder.build_workflow(self.config) diff --git a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py new file mode 100644 index 00000000..bea9dc75 --- /dev/null +++ b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py @@ -0,0 +1,81 @@ +import os +import unittest +from unittest.mock import Mock, patch +from multi_x_serverless.deployment.client.config import Config +from multi_x_serverless.deployment.client.multi_x_serverless_workflow import MultiXServerlessFunction +from multi_x_serverless.deployment.client.deploy.workflow_builder import WorkflowBuilder + + +class TestWorkflowBuilderFuncEnvVar(unittest.TestCase): + def test_build_func_environment_variables(self): + # function 1 (empty function level environment variables) + function1 = Mock(spec=MultiXServerlessFunction) + function1.entry_point = True + function1.name = "function1" + function1.handler = "function1" + function1.regions_and_providers = {} + function1.func_environment_variables = [] + + # function 2 (no overlap with global environment variables) + function2 = Mock(spec=MultiXServerlessFunction) + function2.entry_point = False + function2.name = "function2" + function2.handler = "function1" + function2.regions_and_providers = {"providers": []} + function2.func_environment_variables = [{"key": "ENV_3", "value": "function2_env_3"}] + + # function 3 (overlap with global environment variables) + function3 = Mock(spec=MultiXServerlessFunction) + function3.entry_point = False + function3.name = "function2" + function3.handler = "function1" + function3.regions_and_providers = {"providers": []} + function3.func_environment_variables = [{"key": "ENV_1", "value": "function3_env_1"}] + + self.builder = WorkflowBuilder() + self.config = Mock(spec=Config) + self.config.workflow_name = "test_workflow" + self.config.workflow_app.functions = {"function1": function1, "function2": function2, "function3": function3} + self.config.environment_variables = { + "ENV_1": "global_env_1", + "ENV_2": "global_env_2", + } + self.config.python_version = "3.8" + self.config.home_regions = [] + self.config.project_dir = "/path/to/project" + self.config.iam_policy_file = None + self.config.regions_and_providers = {"providers": []} + self.config.workflow_app.get_successors.return_value = [] + + workflow = self.builder.build_workflow(self.config) + + self.assertEqual(len(workflow._resources), 3) + built_func1 = workflow._resources[0] + built_func2 = workflow._resources[1] + built_func3 = workflow._resources[2] + self.assertEqual( + built_func1.environment_variables, + { + "ENV_1": "global_env_1", + "ENV_2": "global_env_2", + }, + ) + self.assertEqual( + built_func2.environment_variables, + { + "ENV_1": "global_env_1", + "ENV_2": "global_env_2", + "ENV_3": "function2_env_3", + }, + ) + self.assertEqual( + built_func3.environment_variables, + { + "ENV_1": "function3_env_1", + "ENV_2": "global_env_2", + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/multi_x_serverless/tests/deployment/client/test_config.py b/multi_x_serverless/tests/deployment/client/test_config.py index 99831109..edb23005 100644 --- a/multi_x_serverless/tests/deployment/client/test_config.py +++ b/multi_x_serverless/tests/deployment/client/test_config.py @@ -21,7 +21,7 @@ def test_python_version(self): self.assertTrue(self.config.python_version.startswith("python")) def test_environment_variables(self): - self.config.project_config["environment_variables"] = [{"name": "ENV", "value": "test"}] + self.config.project_config["environment_variables"] = [{"key": "ENV", "value": "test"}] self.assertEqual(self.config.environment_variables, {"ENV": "test"}) def test_home_regions(self): diff --git a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py index 7e898705..671761e9 100644 --- a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py +++ b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py @@ -20,14 +20,18 @@ def function(x): "forbidden_regions": [["aws", "us-east-2"]], "providers": providers, } + func_environment_variables = [{"key": "example_key", "value": "example_value"}] - function_obj = MultiXServerlessFunction(function, name, entry_point, regions_and_providers) + function_obj = MultiXServerlessFunction( + function, name, entry_point, regions_and_providers, func_environment_variables + ) self.assertEqual(function_obj.function_callable, function) self.assertEqual(function_obj.name, name) self.assertEqual(function_obj.entry_point, entry_point) self.assertEqual(function_obj.handler, function.__name__) self.assertEqual(function_obj.regions_and_providers, regions_and_providers) + self.assertEqual(function_obj.func_environment_variables, func_environment_variables) def test_is_waiting_for_predecessors(self): def function(x): @@ -36,15 +40,20 @@ def function(x): name = "test_function" entry_point = True regions_and_providers = {} + func_environment_variables = [] - function_obj = MultiXServerlessFunction(function, name, entry_point, regions_and_providers) + function_obj = MultiXServerlessFunction( + function, name, entry_point, regions_and_providers, func_environment_variables + ) self.assertFalse(function_obj.is_waiting_for_predecessors()) def function(x): return get_predecessor_data() - function_obj = MultiXServerlessFunction(function, name, entry_point, regions_and_providers) + function_obj = MultiXServerlessFunction( + function, name, entry_point, regions_and_providers, func_environment_variables + ) self.assertTrue(function_obj.is_waiting_for_predecessors()) @@ -55,8 +64,11 @@ def function(x): name = "test_function" entry_point = True regions_and_providers = {} + func_environment_variables = [] - function_obj = MultiXServerlessFunction(function, name, entry_point, regions_and_providers) + function_obj = MultiXServerlessFunction( + function, name, entry_point, regions_and_providers, func_environment_variables + ) function_obj.validate_function_name() @@ -64,3 +76,7 @@ def function(x): with self.assertRaises(ValueError): function_obj.validate_function_name() + + +if __name__ == "__main__": + unittest.main() diff --git a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py index cead008a..46e51bcc 100644 --- a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py +++ b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py @@ -64,6 +64,7 @@ def test_func(payload): } ], }, + [], ), ) @@ -74,6 +75,67 @@ def test_func(payload): # Check if the routing_decision attribute was set correctly self.assertEqual(test_func.routing_decision["decision"], 1) + def test_serverless_function_with_environment_variables(self): + workflow = MultiXServerlessWorkflow( + name="test-workflow" + ) # Assuming Workflow is the class containing serverless_function + workflow.register_function = Mock() + workflow.get_routing_decision_from_platform = Mock(return_value={"decision": 1}) + + @workflow.serverless_function( + name="test_func", + entry_point=True, + regions_and_providers={ + "only_regions": [["aws", "us-east-1"]], + "forbidden_regions": [["aws", "us-east-2"]], + "providers": [ + { + "name": "aws", + "config": { + "timeout": 60, + "memory": 128, + }, + } + ], + }, + func_environment_variables=[ + {"key": "example_key", "value": "example_value"}, + {"key": "example_key_2", "value": "example_value_2"}, + {"key": "example_key_3", "value": "example_value_3"}, + ], + ) + def test_func(payload): + return payload * 2 + + args, _ = workflow.register_function.call_args + + # Test with multiple environment variables + self.assertEqual( + args[1:], + ( + "test_func", + True, + { + "only_regions": [["aws", "us-east-1"]], + "forbidden_regions": [["aws", "us-east-2"]], + "providers": [ + { + "name": "aws", + "config": { + "timeout": 60, + "memory": 128, + }, + } + ], + }, + [ + {"key": "example_key", "value": "example_value"}, + {"key": "example_key_2", "value": "example_value_2"}, + {"key": "example_key_3", "value": "example_value_3"}, + ], + ), + ) + def test_invoke_serverless_function(self): workflow = MultiXServerlessWorkflow(name="test-workflow") workflow.register_function = Mock() @@ -130,7 +192,7 @@ def test_func(payload: dict[str, Any]) -> dict[str, Any]: registered_func = args[0] registered_func.name = "test_func" self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {})) + self.assertEqual(args[1:], ("test_func", False, {}, [])) workflow.functions["test_func"] = registered_func # Call test_func with a payload @@ -159,8 +221,11 @@ def test_function(x): entry_point = True regions_and_providers = {} providers = [] + func_environment_variables = [] - function_obj_1 = MultiXServerlessFunction(test_function, name, entry_point, regions_and_providers) + function_obj_1 = MultiXServerlessFunction( + test_function, name, entry_point, regions_and_providers, func_environment_variables + ) workflow = MultiXServerlessWorkflow(name="test-workflow") workflow.functions = [function_obj_1] @@ -170,7 +235,9 @@ def test_function(x): def function(x): return invoke_serverless_function("test_function", x) - function_obj_2 = MultiXServerlessFunction(function, name, entry_point, regions_and_providers) + function_obj_2 = MultiXServerlessFunction( + function, name, entry_point, regions_and_providers, func_environment_variables + ) workflow.functions = {"test_function": function_obj_1, "test_function_2": function_obj_2} @@ -197,7 +264,7 @@ def test_func(payload: str) -> dict[str, Any]: registered_func = args[0] registered_func.name = "test_func" self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {})) + self.assertEqual(args[1:], ("test_func", False, {}, [])) workflow.functions["test_func"] = registered_func # Call test_func with a payload From 3e22bbedb9c5a82672f836aab01a4be95e98ed2c Mon Sep 17 00:00:00 2001 From: Jerry Sun Date: Wed, 10 Jan 2024 12:31:42 -0800 Subject: [PATCH 3/4] fix issues --- docs/design.md | 4 +-- .../deployment/client/cli/template/app.py | 2 +- .../deployment/client/config.py | 2 ++ .../client/deploy/workflow_builder.py | 4 +-- .../client/multi_x_serverless_function.py | 4 +-- .../client/multi_x_serverless_workflow.py | 29 +++++++++---------- .../client/deploy/test_workflow_builder.py | 4 +-- .../test_workflow_builder_func_env_var.py | 6 ++-- .../test_multi_x_serverless_function.py | 16 +++++----- .../test_multi_x_serverless_workflow.py | 8 ++--- 10 files changed, 39 insertions(+), 40 deletions(-) diff --git a/docs/design.md b/docs/design.md index 8285c3af..c496a209 100644 --- a/docs/design.md +++ b/docs/design.md @@ -82,7 +82,7 @@ workflow = MultiXServerlessWorkflow("workflow_name") "only_regions": [["aws", "us-east-1"], ["aws", "us-east-2"], ["aws", "us-west-1"], ["aws", "us-west-2"]], "forbidden_regions": None, }, - func_environment_variables=[ + environment_variables=[ { "key": "example_key", "value": "example_value" @@ -107,7 +107,7 @@ The meaning of the different parameters is as follows: - `regions_and_providers`: A dictionary that contains the regions and providers that the function can be deployed to. This can be used to override the global settings in the `config.yml`. If none or an empty dictionary is provided, the global config takes precedence. The dictionary has two keys: - `only_regions`: A list of regions that the function can be deployed to. If this list is empty, the function can be deployed to any region. - `forbidden_regions`: A list of regions that the function cannot be deployed to. If this list is empty, the function can be deployed to any region. -- `func_environment_variables`: This parameter represents a list of dictionaries, each designed for setting environment variables specifically for a function. Users must adhere to a structured format within each dictionary. This format requires two entries: "key" and "value". The "key" entry should contain the name of the environment variable, serving as an identifier. The "value" entry holds the corresponding value assigned to that variable. +- `environment_variables`: This parameter represents a list of dictionaries, each designed for setting environment variables specifically for a function. Users must adhere to a structured format within each dictionary. This format requires two entries: "key" and "value". The "key" entry should contain the name of the environment variable, serving as an identifier. The "value" entry holds the corresponding value assigned to that variable. - `providers`: A list of providers that the function can be deployed to. This can be used to override the global settings in the `config.yml`. If a list of providers is specified at the function level this takes precedence over the global configurations. If none or an empty list is provided, the global config takes precedence. Each provider is a dictionary with two keys: - `name`: The name of the provider. This is the name that is used directly in the physical representation of the workflow. - `config`: A dictionary that contains the configuration for the specific provider. diff --git a/multi_x_serverless/deployment/client/cli/template/app.py b/multi_x_serverless/deployment/client/cli/template/app.py index 4e4e3033..559e6bb5 100644 --- a/multi_x_serverless/deployment/client/cli/template/app.py +++ b/multi_x_serverless/deployment/client/cli/template/app.py @@ -21,7 +21,7 @@ } ], }, - func_environment_variables=[{"key": "example_key", "value": "example_value"}], + environment_variables=[{"key": "example_key", "value": "example_value"}], ) def first_function(event: dict[str, Any]) -> dict[str, Any]: payload = { diff --git a/multi_x_serverless/deployment/client/config.py b/multi_x_serverless/deployment/client/config.py index 15cd0d01..3af19af1 100644 --- a/multi_x_serverless/deployment/client/config.py +++ b/multi_x_serverless/deployment/client/config.py @@ -50,6 +50,8 @@ def environment_variables(self) -> dict[str, str]: for env_variable in list_of_env_variables: if not isinstance(env_variable["value"], str): raise RuntimeError("Environment variable value need to be a str") + if not isinstance(env_variable["key"], str): + raise RuntimeError("Environment variable key need to be a str") env_variables[env_variable["key"]] = env_variable["value"] return env_variables diff --git a/multi_x_serverless/deployment/client/deploy/workflow_builder.py b/multi_x_serverless/deployment/client/deploy/workflow_builder.py index 5113ca2a..2f6dbb70 100644 --- a/multi_x_serverless/deployment/client/deploy/workflow_builder.py +++ b/multi_x_serverless/deployment/client/deploy/workflow_builder.py @@ -40,7 +40,7 @@ def build_workflow(self, config: Config) -> Workflow: # pylint: disable=too-man self._verify_providers(providers) merged_env_vars = self.merge_environment_variables( - function.func_environment_variables, config.environment_variables + function.environment_variables, config.environment_variables ) resources.append( Function( @@ -139,7 +139,7 @@ def get_function_role(self, config: Config, function_name: str) -> IAMRole: return IAMRole(role_name=role_name, policy=filename) def merge_environment_variables( - self, function_env_vars: list[dict[str, str]] | None, config_env_vars: dict[str, str] + self, function_env_vars: Optional[list[dict[str, str]]], config_env_vars: dict[str, str] ) -> dict[str, str]: if not function_env_vars: return config_env_vars diff --git a/multi_x_serverless/deployment/client/multi_x_serverless_function.py b/multi_x_serverless/deployment/client/multi_x_serverless_function.py index 9116b72a..ae03950c 100644 --- a/multi_x_serverless/deployment/client/multi_x_serverless_function.py +++ b/multi_x_serverless/deployment/client/multi_x_serverless_function.py @@ -13,14 +13,14 @@ def __init__( name: str, entry_point: bool, regions_and_providers: dict, - func_environment_variables: list[dict[str, str]], + environment_variables: list[dict[str, str]], ): self.function_callable = function_callable self.name = name self.entry_point = entry_point self.handler = function_callable.__name__ self.regions_and_providers = regions_and_providers if len(regions_and_providers) > 0 else None - self.func_environment_variables = func_environment_variables if len(func_environment_variables) > 0 else None + self.environment_variables = environment_variables if len(environment_variables) > 0 else None self.validate_function_name() def validate_function_name(self) -> None: diff --git a/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py b/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py index 85015acd..1e0d9fb8 100644 --- a/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py +++ b/multi_x_serverless/deployment/client/multi_x_serverless_workflow.py @@ -296,7 +296,7 @@ def register_function( name: str, entry_point: bool, regions_and_providers: dict, - func_environment_variables: list[dict[str, str]], + environment_variables: list[dict[str, str]], ) -> None: """ Register a function as a serverless function. @@ -306,18 +306,15 @@ def register_function( At this point we only need to register the function with the wrapper, the actual deployment will be done later by the deployment manager. """ - wrapper = MultiXServerlessFunction( - function, name, entry_point, regions_and_providers, func_environment_variables - ) + wrapper = MultiXServerlessFunction(function, name, entry_point, regions_and_providers, environment_variables) self.functions[function.__name__] = wrapper - # TODO (#22): Add function specific environment variables def serverless_function( self, name: Optional[str] = None, entry_point: bool = False, regions_and_providers: Optional[dict] = None, - func_environment_variables: Optional[list[dict[str, str]]] = None, + environment_variables: Optional[list[dict[str, str]]] = None, ) -> Callable[..., Any]: """ Decorator to register a function as a Lambda function. @@ -355,20 +352,20 @@ def serverless_function( if regions_and_providers is None: regions_and_providers = {} - if func_environment_variables is None: - func_environment_variables = [] + if environment_variables is None: + environment_variables = [] else: - if not isinstance(func_environment_variables, list): - raise RuntimeError("func_environment_variables must be a list of dicts") - for env_variable in func_environment_variables: + if not isinstance(environment_variables, list): + raise RuntimeError("environment_variables must be a list of dicts") + for env_variable in environment_variables: if not isinstance(env_variable, dict): - raise RuntimeError("func_environment_variables must be a list of dicts") + raise RuntimeError("environment_variables must be a list of dicts") if "key" not in env_variable or "value" not in env_variable: - raise RuntimeError("func_environment_variables must be a list of dicts with keys 'key' and 'value'") + raise RuntimeError("environment_variables must be a list of dicts with keys 'key' and 'value'") if not isinstance(env_variable["key"], str): - raise RuntimeError("func_environment_variables must be a list of dicts with 'key' as a string") + raise RuntimeError("environment_variables must be a list of dicts with 'key' as a string") if not isinstance(env_variable["value"], str): - raise RuntimeError("func_environment_variables must be a list of dicts with 'value' as a string") + raise RuntimeError("environment_variables must be a list of dicts with 'value' as a string") def _register_handler(func: Callable[..., Any]) -> Callable[..., Any]: handler_name = name if name is not None else func.__name__ @@ -396,7 +393,7 @@ def wrapper(*args, **kwargs): # type: ignore # pylint: disable=unused-argument wrapper.routing_decision = {} # type: ignore wrapper.entry_point = entry_point # type: ignore wrapper.original_function = func # type: ignore - self.register_function(func, handler_name, entry_point, regions_and_providers, func_environment_variables) + self.register_function(func, handler_name, entry_point, regions_and_providers, environment_variables) return wrapper return _register_handler diff --git a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py index 37a0d738..586d0815 100644 --- a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py +++ b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py @@ -29,13 +29,13 @@ def test_build_workflow_multiple_entry_points(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} - function1.func_environment_variables = {} + function1.environment_variables = {} function2 = Mock(spec=MultiXServerlessFunction) function2.entry_point = True function2.name = "function2" function2.handler = "function1" function2.regions_and_providers = {"providers": []} - function2.func_environment_variables = {} + function2.environment_variables = {} self.config.workflow_app.functions = {"function1": function1, "function2": function2} with self.assertRaises(RuntimeError): self.builder.build_workflow(self.config) diff --git a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py index bea9dc75..ce85a7a7 100644 --- a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py +++ b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py @@ -14,7 +14,7 @@ def test_build_func_environment_variables(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} - function1.func_environment_variables = [] + function1.environment_variables = [] # function 2 (no overlap with global environment variables) function2 = Mock(spec=MultiXServerlessFunction) @@ -22,7 +22,7 @@ def test_build_func_environment_variables(self): function2.name = "function2" function2.handler = "function1" function2.regions_and_providers = {"providers": []} - function2.func_environment_variables = [{"key": "ENV_3", "value": "function2_env_3"}] + function2.environment_variables = [{"key": "ENV_3", "value": "function2_env_3"}] # function 3 (overlap with global environment variables) function3 = Mock(spec=MultiXServerlessFunction) @@ -30,7 +30,7 @@ def test_build_func_environment_variables(self): function3.name = "function2" function3.handler = "function1" function3.regions_and_providers = {"providers": []} - function3.func_environment_variables = [{"key": "ENV_1", "value": "function3_env_1"}] + function3.environment_variables = [{"key": "ENV_1", "value": "function3_env_1"}] self.builder = WorkflowBuilder() self.config = Mock(spec=Config) diff --git a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py index 671761e9..592f0de7 100644 --- a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py +++ b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_function.py @@ -20,10 +20,10 @@ def function(x): "forbidden_regions": [["aws", "us-east-2"]], "providers": providers, } - func_environment_variables = [{"key": "example_key", "value": "example_value"}] + environment_variables = [{"key": "example_key", "value": "example_value"}] function_obj = MultiXServerlessFunction( - function, name, entry_point, regions_and_providers, func_environment_variables + function, name, entry_point, regions_and_providers, environment_variables ) self.assertEqual(function_obj.function_callable, function) @@ -31,7 +31,7 @@ def function(x): self.assertEqual(function_obj.entry_point, entry_point) self.assertEqual(function_obj.handler, function.__name__) self.assertEqual(function_obj.regions_and_providers, regions_and_providers) - self.assertEqual(function_obj.func_environment_variables, func_environment_variables) + self.assertEqual(function_obj.environment_variables, environment_variables) def test_is_waiting_for_predecessors(self): def function(x): @@ -40,10 +40,10 @@ def function(x): name = "test_function" entry_point = True regions_and_providers = {} - func_environment_variables = [] + environment_variables = [] function_obj = MultiXServerlessFunction( - function, name, entry_point, regions_and_providers, func_environment_variables + function, name, entry_point, regions_and_providers, environment_variables ) self.assertFalse(function_obj.is_waiting_for_predecessors()) @@ -52,7 +52,7 @@ def function(x): return get_predecessor_data() function_obj = MultiXServerlessFunction( - function, name, entry_point, regions_and_providers, func_environment_variables + function, name, entry_point, regions_and_providers, environment_variables ) self.assertTrue(function_obj.is_waiting_for_predecessors()) @@ -64,10 +64,10 @@ def function(x): name = "test_function" entry_point = True regions_and_providers = {} - func_environment_variables = [] + environment_variables = [] function_obj = MultiXServerlessFunction( - function, name, entry_point, regions_and_providers, func_environment_variables + function, name, entry_point, regions_and_providers, environment_variables ) function_obj.validate_function_name() diff --git a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py index 46e51bcc..e77585a2 100644 --- a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py +++ b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py @@ -98,7 +98,7 @@ def test_serverless_function_with_environment_variables(self): } ], }, - func_environment_variables=[ + environment_variables=[ {"key": "example_key", "value": "example_value"}, {"key": "example_key_2", "value": "example_value_2"}, {"key": "example_key_3", "value": "example_value_3"}, @@ -221,10 +221,10 @@ def test_function(x): entry_point = True regions_and_providers = {} providers = [] - func_environment_variables = [] + environment_variables = [] function_obj_1 = MultiXServerlessFunction( - test_function, name, entry_point, regions_and_providers, func_environment_variables + test_function, name, entry_point, regions_and_providers, environment_variables ) workflow = MultiXServerlessWorkflow(name="test-workflow") @@ -236,7 +236,7 @@ def function(x): return invoke_serverless_function("test_function", x) function_obj_2 = MultiXServerlessFunction( - function, name, entry_point, regions_and_providers, func_environment_variables + function, name, entry_point, regions_and_providers, environment_variables ) workflow.functions = {"test_function": function_obj_1, "test_function_2": function_obj_2} From 2667a054a1c6b9283b6e72b53291866fd74a1f1f Mon Sep 17 00:00:00 2001 From: vgsteiger Date: Wed, 10 Jan 2024 16:58:38 -0800 Subject: [PATCH 4/4] Remove test_workflow_builder_func_env_var.py and fix tests --- .../client/deploy/test_workflow_builder.py | 80 ++++++++++++++++++ .../test_workflow_builder_func_env_var.py | 81 ------------------- .../test_multi_x_serverless_workflow.py | 56 ++----------- 3 files changed, 87 insertions(+), 130 deletions(-) delete mode 100644 multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py diff --git a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py index 8b012e1f..6de7c4d6 100644 --- a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py +++ b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder.py @@ -47,6 +47,7 @@ def test_build_workflow_merge_case_self_cycle(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.environment_variables = [] function1.is_waiting_for_predecessors = Mock(return_value=False) function2 = Mock(spec=MultiXServerlessFunction) @@ -54,6 +55,7 @@ def test_build_workflow_merge_case_self_cycle(self): function2.name = "function2" function2.handler = "function2" function2.regions_and_providers = {"providers": []} + function2.environment_variables = [] function2.is_waiting_for_predecessors = Mock(return_value=True) # This is a merge function # Mock the workflow app to return the successors of function1 @@ -76,6 +78,7 @@ def test_build_workflow_merge_case_multiple_incoming(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.environment_variables = [] function1.is_waiting_for_predecessors = Mock(return_value=False) function2 = Mock(spec=MultiXServerlessFunction) @@ -83,6 +86,7 @@ def test_build_workflow_merge_case_multiple_incoming(self): function2.name = "function2" function2.handler = "function2" function2.regions_and_providers = {"providers": []} + function2.environment_variables = [] function2.is_waiting_for_predecessors = Mock(return_value=True) # This is a merge function # Mock the workflow app to return the successors of function1 @@ -142,6 +146,7 @@ def test_build_workflow_self_call(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.environment_variables = [] function1.is_waiting_for_predecessors = Mock(return_value=False) # Mock the workflow app to return the successors of function1 @@ -164,6 +169,7 @@ def test_build_workflow_merge_working(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.environment_variables = [] function1.is_waiting_for_predecessors = Mock(return_value=False) function2 = Mock(spec=MultiXServerlessFunction) @@ -171,6 +177,7 @@ def test_build_workflow_merge_working(self): function2.name = "function2" function2.handler = "function2" function2.regions_and_providers = {"providers": []} + function2.environment_variables = [] function2.is_waiting_for_predecessors = Mock(return_value=True) # This is a merge function # Mock the workflow app to return the successors of function1 @@ -191,6 +198,7 @@ def test_build_workflow_cycle_in_function_calls(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.environment_variables = [] function1.is_waiting_for_predecessors = Mock(return_value=False) function2 = Mock(spec=MultiXServerlessFunction) @@ -198,6 +206,7 @@ def test_build_workflow_cycle_in_function_calls(self): function2.name = "function2" function2.handler = "function2" function2.regions_and_providers = {"providers": []} + function2.environment_variables = [] function2.is_waiting_for_predecessors = Mock(return_value=False) # This is a merge function # Mock the workflow app to return the successors of function1 @@ -222,6 +231,7 @@ def test_build_workflow_merge_cycle(self): function1.name = "function1" function1.handler = "function1" function1.regions_and_providers = {} + function1.environment_variables = [] function1.is_waiting_for_predecessors = Mock(return_value=False) function2 = Mock(spec=MultiXServerlessFunction) @@ -229,6 +239,7 @@ def test_build_workflow_merge_cycle(self): function2.name = "function2" function2.handler = "function2" function2.regions_and_providers = {"providers": []} + function2.environment_variables = [] function2.is_waiting_for_predecessors = Mock(return_value=True) self.config.workflow_app.get_successors = Mock(side_effect=[[function2], [function2], []]) @@ -253,6 +264,75 @@ def test_get_function_role_without_policy_file(self, mock_join): self.assertEqual(role.name, "test_function-role") self.assertEqual(role.policy, "/path/to/default_policy") + def test_build_func_environment_variables(self): + # function 1 (empty function level environment variables) + function1 = Mock(spec=MultiXServerlessFunction) + function1.entry_point = True + function1.name = "function1" + function1.handler = "function1" + function1.regions_and_providers = {} + function1.environment_variables = [] + + # function 2 (no overlap with global environment variables) + function2 = Mock(spec=MultiXServerlessFunction) + function2.entry_point = False + function2.name = "function2" + function2.handler = "function1" + function2.regions_and_providers = {"providers": []} + function2.environment_variables = [{"key": "ENV_3", "value": "function2_env_3"}] + + # function 3 (overlap with global environment variables) + function3 = Mock(spec=MultiXServerlessFunction) + function3.entry_point = False + function3.name = "function2" + function3.handler = "function1" + function3.regions_and_providers = {"providers": []} + function3.environment_variables = [{"key": "ENV_1", "value": "function3_env_1"}] + + self.builder = WorkflowBuilder() + self.config = Mock(spec=Config) + self.config.workflow_name = "test_workflow" + self.config.workflow_app.functions = {"function1": function1, "function2": function2, "function3": function3} + self.config.environment_variables = { + "ENV_1": "global_env_1", + "ENV_2": "global_env_2", + } + self.config.python_version = "3.8" + self.config.home_regions = [] + self.config.project_dir = "/path/to/project" + self.config.iam_policy_file = None + self.config.regions_and_providers = {"providers": []} + self.config.workflow_app.get_successors.return_value = [] + + workflow = self.builder.build_workflow(self.config) + + self.assertEqual(len(workflow._resources), 3) + built_func1 = workflow._resources[0] + built_func2 = workflow._resources[1] + built_func3 = workflow._resources[2] + self.assertEqual( + built_func1.environment_variables, + { + "ENV_1": "global_env_1", + "ENV_2": "global_env_2", + }, + ) + self.assertEqual( + built_func2.environment_variables, + { + "ENV_1": "global_env_1", + "ENV_2": "global_env_2", + "ENV_3": "function2_env_3", + }, + ) + self.assertEqual( + built_func3.environment_variables, + { + "ENV_1": "function3_env_1", + "ENV_2": "global_env_2", + }, + ) + if __name__ == "__main__": unittest.main() diff --git a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py b/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py deleted file mode 100644 index ce85a7a7..00000000 --- a/multi_x_serverless/tests/deployment/client/deploy/test_workflow_builder_func_env_var.py +++ /dev/null @@ -1,81 +0,0 @@ -import os -import unittest -from unittest.mock import Mock, patch -from multi_x_serverless.deployment.client.config import Config -from multi_x_serverless.deployment.client.multi_x_serverless_workflow import MultiXServerlessFunction -from multi_x_serverless.deployment.client.deploy.workflow_builder import WorkflowBuilder - - -class TestWorkflowBuilderFuncEnvVar(unittest.TestCase): - def test_build_func_environment_variables(self): - # function 1 (empty function level environment variables) - function1 = Mock(spec=MultiXServerlessFunction) - function1.entry_point = True - function1.name = "function1" - function1.handler = "function1" - function1.regions_and_providers = {} - function1.environment_variables = [] - - # function 2 (no overlap with global environment variables) - function2 = Mock(spec=MultiXServerlessFunction) - function2.entry_point = False - function2.name = "function2" - function2.handler = "function1" - function2.regions_and_providers = {"providers": []} - function2.environment_variables = [{"key": "ENV_3", "value": "function2_env_3"}] - - # function 3 (overlap with global environment variables) - function3 = Mock(spec=MultiXServerlessFunction) - function3.entry_point = False - function3.name = "function2" - function3.handler = "function1" - function3.regions_and_providers = {"providers": []} - function3.environment_variables = [{"key": "ENV_1", "value": "function3_env_1"}] - - self.builder = WorkflowBuilder() - self.config = Mock(spec=Config) - self.config.workflow_name = "test_workflow" - self.config.workflow_app.functions = {"function1": function1, "function2": function2, "function3": function3} - self.config.environment_variables = { - "ENV_1": "global_env_1", - "ENV_2": "global_env_2", - } - self.config.python_version = "3.8" - self.config.home_regions = [] - self.config.project_dir = "/path/to/project" - self.config.iam_policy_file = None - self.config.regions_and_providers = {"providers": []} - self.config.workflow_app.get_successors.return_value = [] - - workflow = self.builder.build_workflow(self.config) - - self.assertEqual(len(workflow._resources), 3) - built_func1 = workflow._resources[0] - built_func2 = workflow._resources[1] - built_func3 = workflow._resources[2] - self.assertEqual( - built_func1.environment_variables, - { - "ENV_1": "global_env_1", - "ENV_2": "global_env_2", - }, - ) - self.assertEqual( - built_func2.environment_variables, - { - "ENV_1": "global_env_1", - "ENV_2": "global_env_2", - "ENV_3": "function2_env_3", - }, - ) - self.assertEqual( - built_func3.environment_variables, - { - "ENV_1": "function3_env_1", - "ENV_2": "global_env_2", - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py index 4761b498..222e72f5 100644 --- a/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py +++ b/multi_x_serverless/tests/deployment/client/test_multi_x_serverless_workflow.py @@ -168,7 +168,7 @@ def test_func(payload: dict[str, Any]) -> dict[str, Any]: args, _ = workflow.register_function.call_args registered_func = args[0] self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {})) + self.assertEqual(args[1:], ("test_func", False, {}, [])) self.assertEqual(test_func.routing_decision, {}) @@ -256,7 +256,7 @@ def test_func(payload: dict[str, Any]) -> dict[str, Any]: registered_func = args[0] registered_func.name = "test_func" self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {})) + self.assertEqual(args[1:], ("test_func", False, {}, [])) workflow.functions["test_func"] = registered_func @workflow.serverless_function(name="merge_func") @@ -268,7 +268,7 @@ def merge_func(payload: dict[str, Any]) -> dict[str, Any]: registered_func = args[0] registered_func.name = "merge_func" self.assertEqual(registered_func.__name__, "merge_func") - self.assertEqual(args[1:], ("merge_func", False, {})) + self.assertEqual(args[1:], ("merge_func", False, {}, [])) workflow.functions["merge_func"] = registered_func # Call test_func with a payload @@ -312,7 +312,7 @@ def test_func(payload: dict[str, Any]) -> dict[str, Any]: registered_func = args[0] registered_func.name = "test_func" self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {})) + self.assertEqual(args[1:], ("test_func", False, {}, [])) workflow.functions["test_func"] = registered_func @workflow.serverless_function(name="test_func2") @@ -327,7 +327,7 @@ def test_func2(payload: dict[str, Any]) -> dict[str, Any]: registered_func = args[0] registered_func.name = "test_func2" self.assertEqual(registered_func.__name__, "test_func2") - self.assertEqual(args[1:], ("test_func2", False, {})) + self.assertEqual(args[1:], ("test_func2", False, {}, [])) workflow.functions["test_func2"] = registered_func @workflow.serverless_function(name="merge_func") @@ -339,7 +339,7 @@ def merge_func(payload: dict[str, Any]) -> dict[str, Any]: registered_func = args[0] registered_func.name = "merge_func" self.assertEqual(registered_func.__name__, "merge_func") - self.assertEqual(args[1:], ("merge_func", False, {})) + self.assertEqual(args[1:], ("merge_func", False, {}, [])) workflow.functions["merge_func"] = registered_func # Call test_func with a payload @@ -383,7 +383,7 @@ def test_func(payload: str) -> dict[str, Any]: registered_func = args[0] registered_func.name = "test_func" self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {})) + self.assertEqual(args[1:], ("test_func", False, {}, [])) workflow.functions["test_func"] = registered_func # Call test_func with a payload @@ -436,48 +436,6 @@ def function(x): self.assertEqual(workflow.get_successors(function_obj_2), [function_obj_1]) - def test_invoke_serverless_function_json_argument(self): - workflow = MultiXServerlessWorkflow(name="test-workflow") - workflow.register_function = Mock() - mock_remote_client = Mock() - mock_remote_client.invoke_function = Mock(return_value={"statusCode": 200, "body": "Some response"}) - mock_remote_client_factory = Mock() - mock_remote_client_factory.get_remote_client = Mock(return_value=mock_remote_client) - workflow._remote_client_factory = mock_remote_client_factory - - @workflow.serverless_function(name="test_func") - def test_func(payload: str) -> dict[str, Any]: - # Call invoke_serverless_function from within test_func - workflow.invoke_serverless_function(test_func, payload) - - return "Some response" - - # Check if the function was registered correctly - args, _ = workflow.register_function.call_args - registered_func = args[0] - registered_func.name = "test_func" - self.assertEqual(registered_func.__name__, "test_func") - self.assertEqual(args[1:], ("test_func", False, {}, [])) - workflow.functions["test_func"] = registered_func - - # Call test_func with a payload - response = test_func( - r'{"payload": "{\"key\": \"value\"}", "routing_decision": {"routing_placement": {"test_func": {"provider_region": "aws:region", "identifier": "test_identifier"}, "test_func_1::": {"provider_region": "aws:region", "identifier": "test_identifier"}}, "current_instance_name": "test_func", "instances": [{"instance_name": "test_func", "succeeding_instances": ["test_func_1::"]}]}}' - ) - - mock_remote_client_factory.get_remote_client.assert_called_once_with("aws", "region") - - # Check if invoke_serverless_function was called with the correct arguments - mock_remote_client.invoke_function.assert_called_once_with( - message=r'{"payload": "{\"key\": \"value\"}", "routing_decision": {"routing_placement": {"test_func": {"provider_region": "aws:region", "identifier": "test_identifier"}, "test_func_1::": {"provider_region": "aws:region", "identifier": "test_identifier"}}, "current_instance_name": "test_func_1::", "instances": [{"instance_name": "test_func", "succeeding_instances": ["test_func_1::"]}]}}', - region="region", - identifier="test_identifier", - merge=False, - ) - - # Check if the response from invoke_serverless_function is correct - self.assertEqual(response, "Some response") - def test_get_routing_decision(self): workflow = MultiXServerlessWorkflow(name="test-workflow") # Test when 'wrapper' is in frame.f_locals and wrapper has 'routing_decision' attribute