Merge branch 'main' into features/vGsteiger/merge_functionality_retri…

…eval

.github/workflows/workflow.yaml

@@ -42,7 +42,7 @@ jobs:
         uses: ./.github/actions/poetry
       - name: Run tests
         run: |
-          poetry run pytest --cache-clear --cov=multi_x_serverless multi_x_serverless/ > pytest-coverage.txt
+          poetry run pytest --cov-reset --cache-clear > pytest-coverage.txt
       - name: Comment coverage
         uses: coroo/[email protected]
 

README.md

@@ -8,6 +8,19 @@ We are working with poetry, so you need to install it first.
 pip install poetry
 ```
 
+Alternatively if this doesn't work you will need to install poetry on Linux with:
+
+```bash
+apt install python3-poetry
+```
+
+Or you need to reinstall poetry:
+
+```bash
+curl -sSL https://install.python-poetry.org | python3 -
+poetry self update
+```
+
 Then, install the dependencies:
 
 ```bash

benchmarks/image_processing/app.py

@@ -15,8 +15,18 @@
     name="GetInput",
     entry_point=True,
     regions_and_providers={
-        "only_regions": [["aws", "us-east-1"], ["aws", "us-east-2"], ["aws", "us-west-1"], ["aws", "us-west-2"]],
-        "forbidden_regions": None,
+        "only_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-1",
+            }
+        ],
+        "forbidden_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-2",
+            }
+        ],
         "providers": [
             {
                 "name": "aws",

benchmarks/regression_tuning/app.py

@@ -21,8 +21,18 @@
     name="GetInput",
     entry_point=True,
     regions_and_providers={
-        "only_regions": [["aws", "us-east-1"], ["aws", "us-east-2"], ["aws", "us-west-1"], ["aws", "us-west-2"]],
-        "forbidden_regions": None,
+        "only_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-1",
+            }
+        ],
+        "forbidden_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-2",
+            }
+        ],
         "providers": [
             {
                 "name": "aws",

benchmarks/text_2_speech_censoring/app.py

@@ -20,8 +20,18 @@
     name="GetInput",
     entry_point=True,
     regions_and_providers={
-        "only_regions": [["aws", "us-east-1"], ["aws", "us-east-2"], ["aws", "us-west-1"], ["aws", "us-west-2"]],
-        "forbidden_regions": None,
+        "only_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-1",
+            }
+        ],
+        "forbidden_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-2",
+            }
+        ],
         "providers": [
             {
                 "name": "aws",

docs/design.md

@@ -120,8 +120,13 @@ workflow = MultiXServerlessWorkflow("workflow_name")
     name="First-Function",
     entry_point=True,
     regions_and_providers={
-        "only_regions": [["aws", "us-east-1"]],
-        "forbidden_regions": [["aws", "us-east-2"]],
+        "only_regions": [
+          {
+            "provider": "aws",
+            "region": "us-east-1",
+          }
+        ],
+        "forbidden_regions": None,
         "providers": [
             {
                 "name": "aws",

multi_x_serverless/deployment/client/cli/config_schema.py

@@ -23,15 +23,20 @@ def validate_config(cls: Any, values: Any) -> Any:  # pylint: disable=no-self-ar
         return values
 
 
+class ProviderRegion(BaseModel):
+    provider: str = Field(..., title="The name of the provider")
+    region: str = Field(..., title="The name of the region")
+
+
 class RegionAndProviders(BaseModel):
-    only_regions: Optional[List[List[str]]] = Field(None, title="List of regions to deploy to")
-    forbidden_regions: Optional[List[List[str]]] = Field(None, title="List of regions to not deploy to")
+    only_regions: Optional[List[ProviderRegion]] = Field(None, title="List of regions to deploy to")
+    forbidden_regions: Optional[List[ProviderRegion]] = Field(None, title="List of regions to not deploy to")
     providers: List[Provider] = Field(..., title="List of possible providers with their configurations")
 
 
 class ConfigSchema(BaseModel):
     workflow_name: str = Field(..., title="The name of the workflow")
     environment_variables: List[EnvironmentVariable] = Field(..., title="List of environment variables")
     iam_policy_file: str = Field(..., title="The IAM policy file")
-    home_regions: List[List[str]] = Field(..., title="List of home regions")
+    home_regions: List[ProviderRegion] = Field(..., title="List of home regions")
     regions_and_providers: RegionAndProviders = Field(..., title="List of regions and providers")

multi_x_serverless/deployment/client/cli/template/.multi-x-serverless/config.yml

@@ -3,7 +3,9 @@ environment_variables:
   - name: "ENV_VAR_1"
     value: "value_1"
 iam_policy_file: "iam_policy.json"
-home_regions: [["aws", us-west-2"]] # Regions are defined as "provider:region" (e.g. aws:us-west-2)
+home_regions:
+  - provider: "aws"
+    region: "us-east-1"
 estimated_invocations_per_month: 1000000
 constraints:
   hard_resource_constraints: # None for none
@@ -25,8 +27,12 @@ constraints:
     - runtime
     - carbon
 regions_and_providers: # Either the user specify only allowed regions (which will override everything else)
-  only_regions: null # Only check other conditions if this is None, else will restrict to ONLY those regions
-  forbidden_regions: null
+  only_regions:
+    - provider: "aws"
+      region: "us-east-1"
+  forbidden_regions:
+    - provider: "aws"
+      region: "us-east-2"
   providers:
     - name: "aws"
       config:

multi_x_serverless/deployment/client/cli/template/app.py

@@ -9,8 +9,18 @@
     name="First-Function",
     entry_point=True,
     regions_and_providers={
-        "only_regions": [["aws", "us-east-1"]],
-        "forbidden_regions": [["aws", "us-east-2"]],
+        "only_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-1",
+            }
+        ],
+        "forbidden_regions": [
+            {
+                "provider": "aws",
+                "region": "us-east-2",
+            }
+        ],
         "providers": [
             {
                 "name": "aws",

multi_x_serverless/deployment/client/config.py

@@ -52,19 +52,7 @@ def environment_variables(self) -> dict[str, Any]:
         return env_variables
 
     @property
-    def home_regions(self) -> list[tuple[str, str]]:
-        if "home_regions_tuple" in self.project_config:
-            return self.project_config["home_regions_tuple"]
-        home_regions: list[list[str]] = self._lookup("home_regions")
-        if home_regions is None:
-            return []
-        home_regions_tuple = [tuple(home_region[0:2]) for home_region in home_regions]
-        self.project_config["home_regions_tuple"] = home_regions_tuple
-        return home_regions_tuple  # type: ignore
-        # somehow mypy thinks this is a list of tuples with one or more elements
-
-    @property
-    def home_regions_json(self) -> list[list[str]]:
+    def home_regions(self) -> list[dict[str, str]]:
         return self._lookup("home_regions")
 
     @property

multi_x_serverless/deployment/client/deploy/executor.py

@@ -18,8 +18,8 @@ def __init__(self, config: Config) -> None:
 
     def execute(self, deployment_plan: DeploymentPlan) -> None:
         for home_region, instructions in deployment_plan.instructions.items():
-            endpoint, region = home_region.split(":")
-            client = self._remote_client_factory.get_remote_client(endpoint, region)
+            provider, region = home_region.split(":")
+            client = self._remote_client_factory.get_remote_client(provider, region)
             for instruction in instructions:
                 getattr(self, f"_do_{instruction.__class__.__name__.lower()}", self._default_handler)(
                     instruction,

multi_x_serverless/deployment/client/deploy/models/function.py

@@ -9,7 +9,7 @@
 from multi_x_serverless.deployment.client.deploy.models.remote_state import RemoteState
 from multi_x_serverless.deployment.client.deploy.models.resource import Resource
 from multi_x_serverless.deployment.client.deploy.models.variable import Variable
-from multi_x_serverless.deployment.client.enums import Endpoint
+from multi_x_serverless.deployment.client.enums import Provider
 
 
 class Function(Resource):  # pylint: disable=too-many-instance-attributes
@@ -22,7 +22,7 @@ def __init__(
         environment_variables: dict[str, str],
         handler: str,
         runtime: str,
-        home_regions: list[tuple[str, str]],
+        home_regions: list[dict[str, str]],
         providers: list[dict],
     ) -> None:
         super().__init__(name, "function")
@@ -50,12 +50,12 @@ def __repr__(self) -> str:
                     Providers: {self.providers}
                 """
 
-    def initialise_remote_states(self, home_regions: list[tuple[str, str]]) -> None:
+    def initialise_remote_states(self, home_regions: list[dict[str, str]]) -> None:
         for home_region in home_regions:
-            endpoint, region = home_region
-            if endpoint not in self._remote_states:
-                self._remote_states[endpoint] = {}
-            self._remote_states[endpoint][region] = RemoteState(endpoint=endpoint, region=region)
+            provider, region = home_region["provider"], home_region["region"]
+            if provider not in self._remote_states:
+                self._remote_states[provider] = {}
+            self._remote_states[provider][region] = RemoteState(provider=provider, region=region)
 
     def dependencies(self) -> Sequence[Resource]:
         resources: list[Resource] = [self.role, self.deployment_package]
@@ -64,21 +64,21 @@ def dependencies(self) -> Sequence[Resource]:
     def get_deployment_instructions(self) -> dict[str, list[Instruction]]:
         instructions: dict[str, list[Instruction]] = {}
         for home_region in self.home_regions:
-            endpoint, region = home_region
-            if endpoint == Endpoint.AWS.value:
+            provider, region = home_region["provider"], home_region["region"]
+            if provider == Provider.AWS.value:
                 instruction = self.get_deployment_instructions_aws(region)
-            elif endpoint == Endpoint.GCP.value:
+            elif provider == Provider.GCP.value:
                 instruction = self.get_deployment_instructions_gcp(region)
             else:
-                raise RuntimeError(f"Unknown endpoint {endpoint}")
-            instructions[f"{endpoint}:{region}"] = instruction
+                raise RuntimeError(f"Unknown provider {provider}")
+            instructions[f"{provider}:{region}"] = instruction
         return instructions
 
     def _get_memory_and_timeout(self) -> tuple[int, int]:
         memory = 128
         timeout = 3
         for provider in self.providers:
-            if provider["name"] == Endpoint.AWS.value:
+            if provider["name"] == Provider.AWS.value:
                 if "memory" in provider:
                     memory = provider["memory"]
                 if "timeout" in provider:
@@ -117,7 +117,7 @@ def get_deployment_instructions_aws(self, region: str) -> list[Instruction]:
         if policy is None:
             raise RuntimeError(f"Lambda policy could not be read, check the path ({self.role.policy})")
         policy = json.dumps(json.loads(policy))
-        if not self._remote_states[Endpoint.AWS.value][region].resource_exists(self.role):
+        if not self._remote_states[Provider.AWS.value][region].resource_exists(self.role):
             instructions.extend(
                 [
                     APICall(
@@ -164,7 +164,7 @@ def get_deployment_instructions_aws(self, region: str) -> list[Instruction]:
         with open(self.deployment_package.filename, "rb") as f:
             zip_contents = f.read()
         function_varname = f"{self.name}_lambda_arn_{region}"
-        if not self._remote_states[Endpoint.AWS.value][region].resource_exists(self):
+        if not self._remote_states[Provider.AWS.value][region].resource_exists(self):
             instructions.extend(
                 [
                     APICall(
@@ -253,4 +253,4 @@ def get_sns_topic_instruction_for_region(self, region: str, output_var: str) ->
         )
 
     def get_deployment_instructions_gcp(self, region: str) -> list[Instruction]:  # pylint: disable=unused-argument
-        return []
+        raise NotImplementedError()

multi_x_serverless/deployment/client/deploy/models/remote_state.py

@@ -3,8 +3,8 @@
 
 
 class RemoteState:
-    def __init__(self, endpoint: str, region: str) -> None:
-        self._client = RemoteClientFactory().get_remote_client(endpoint, region)
+    def __init__(self, provider: str, region: str) -> None:
+        self._client = RemoteClientFactory().get_remote_client(provider, region)
 
     def resource_exists(self, resource: Resource) -> bool:
         return self._client.resource_exists(resource)

multi_x_serverless/deployment/client/deploy/models/workflow.py

@@ -63,7 +63,7 @@ def get_description(self) -> WorkflowConfig:
             raise RuntimeError("Error in workflow config creation, given config is None, this should not happen")
         workflow_description = {
             "instances": [function_instance.to_json() for function_instance in self._functions],
-            "start_hops": self._config.home_regions_json,
+            "start_hops": self._config.home_regions,
             # TODO (#27): Implement and incorporate Free Tier considerations into data_sources
             "estimated_invocations_per_month": self._config.estimated_invocations_per_month,
             "constraints": self._config.constraints,

multi_x_serverless/deployment/client/enums.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
 
-class Endpoint(Enum):
+class Provider(Enum):
     AWS = "aws"
     GCP = "gcp"

multi_x_serverless/deployment/client/factories/cli_factory.py

@@ -14,7 +14,7 @@
     create_default_deployer,
     create_deletion_deployer,
 )
-from multi_x_serverless.deployment.client.enums import Endpoint
+from multi_x_serverless.deployment.client.enums import Provider
 from multi_x_serverless.deployment.client.multi_x_serverless_workflow import MultiXServerlessWorkflow
 
 
@@ -72,24 +72,24 @@ def __validate_only_regions_and_providers(self, project_config: dict) -> None:
         if "providers" not in project_config["regions_and_providers"]:
             raise RuntimeError("at least one provider must be defined in regions_and_providers")
         if "only_regions" in project_config["regions_and_providers"]:
-            possible_endpoints = [endpoint.value for endpoint in Endpoint]
-            defined_endpoints = [
+            possible_providers = [provider.value for provider in Provider]
+            defined_providers = [
                 provider["name"]
                 for provider in project_config["regions_and_providers"]["providers"]
-                if provider["name"] in possible_endpoints
+                if provider["name"] in possible_providers
             ]
             only_regions = project_config["regions_and_providers"]["only_regions"]
             if not only_regions:
                 only_regions = []
             if only_regions and not isinstance(only_regions, list):
                 raise RuntimeError("only_regions must be a list")
-            for region in only_regions:
-                if not isinstance(region, str):
+            for provider_region in only_regions:
+                if not isinstance(provider_region, dict):
                     raise RuntimeError("only_regions must be a list of strings")
-                provider = region[0]
-                if provider not in Endpoint.__members__:
+                provider = provider_region["provider"]
+                if provider not in Provider.__members__:
                     raise RuntimeError(f"Provider {provider} is not supported")
-                if provider not in defined_endpoints:
+                if provider not in defined_providers:
                     raise RuntimeError(f"Provider {provider} is not defined in providers")
 
     def load_workflow_app(self) -> MultiXServerlessWorkflow:

multi_x_serverless/deployment/client/factories/remote_client_factory.py

@@ -1,13 +1,13 @@
-from multi_x_serverless.deployment.client.enums import Endpoint
+from multi_x_serverless.deployment.client.enums import Provider
 from multi_x_serverless.deployment.client.remote_client.aws_remote_client import AWSRemoteClient
 from multi_x_serverless.deployment.client.remote_client.remote_client import RemoteClient
 
 
 class RemoteClientFactory:
-    def get_remote_client(self, endpoint: str, region: str) -> RemoteClient:
-        endpoint_enum = Endpoint(endpoint)
-        if endpoint_enum == Endpoint.AWS:
+    def get_remote_client(self, provider: str, region: str) -> RemoteClient:
+        provider_enum = Provider(provider)
+        if provider_enum == Provider.AWS:
             return AWSRemoteClient(region)
-        if endpoint_enum == Endpoint.GCP:
+        if provider_enum == Provider.GCP:
             raise NotImplementedError()
-        raise RuntimeError(f"Unknown endpoint {endpoint}")
+        raise RuntimeError(f"Unknown provider {provider}")