From 91154df1324f70068c2042b5be1acec37514a1eb Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Sun, 12 Nov 2023 01:54:20 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A5feat:=20Add=20hclv2=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sld-api-backend/requirements.txt | 2 + .../activityLogs/domain/entities/activity.py | 2 +- .../src/aws/domain/entities/aws.py | 2 +- .../src/azure/domain/entities/azure.py | 2 +- .../domain/entities/custom_providers.py | 2 +- .../src/deploy/domain/entities/deploy.py | 2 +- .../src/gcp/domain/entities/gcp.py | 2 +- .../src/stacks/domain/entities/stacks.py | 6 +-- .../src/tasks/domain/entities/tasks.py | 4 +- .../src/users/domain/entities/users.py | 4 +- .../src/worker/helpers/hcl2_to_json.py | 29 +++++++++++ .../worker/providers/hashicorp/templates.py | 52 ++++++++++++++++--- 12 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 sld-api-backend/src/worker/helpers/hcl2_to_json.py diff --git a/sld-api-backend/requirements.txt b/sld-api-backend/requirements.txt index 248c120c..e5d187da 100644 --- a/sld-api-backend/requirements.txt +++ b/sld-api-backend/requirements.txt @@ -32,6 +32,7 @@ iniconfig==2.0.0 Jinja2==3.1.2 jmespath==1.0.1 kombu==5.3.2 +lark==1.1.8 MarkupSafe==2.1.3 mysqlclient==2.2.0 packaging==23.2 @@ -52,6 +53,7 @@ pyparsing==3.1.1 pytest==7.4.2 python-dateutil==2.8.2 python-dotenv==1.0.0 +python-hcl2==4.3.2 python-jose==3.3.0 python-multipart==0.0.6 python-usernames==1.0.0 diff --git a/sld-api-backend/src/activityLogs/domain/entities/activity.py b/sld-api-backend/src/activityLogs/domain/entities/activity.py index ae6634d4..9d991e3e 100644 --- a/sld-api-backend/src/activityLogs/domain/entities/activity.py +++ b/sld-api-backend/src/activityLogs/domain/entities/activity.py @@ -8,4 +8,4 @@ class ActivityLogs(BaseModel): action: constr(strip_whitespace=True) class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/aws/domain/entities/aws.py b/sld-api-backend/src/aws/domain/entities/aws.py index ab2112c5..52129be3 100644 --- a/sld-api-backend/src/aws/domain/entities/aws.py +++ b/sld-api-backend/src/aws/domain/entities/aws.py @@ -23,4 +23,4 @@ class Aws(AwsBase): id: int class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/azure/domain/entities/azure.py b/sld-api-backend/src/azure/domain/entities/azure.py index 2d4ef892..01ef8cee 100644 --- a/sld-api-backend/src/azure/domain/entities/azure.py +++ b/sld-api-backend/src/azure/domain/entities/azure.py @@ -14,4 +14,4 @@ class Azure(AzureBase): id: int class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/custom_providers/domain/entities/custom_providers.py b/sld-api-backend/src/custom_providers/domain/entities/custom_providers.py index 71ea8f99..87743253 100644 --- a/sld-api-backend/src/custom_providers/domain/entities/custom_providers.py +++ b/sld-api-backend/src/custom_providers/domain/entities/custom_providers.py @@ -11,4 +11,4 @@ class CustomProvider(CustomProviderBase): id: int class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/deploy/domain/entities/deploy.py b/sld-api-backend/src/deploy/domain/entities/deploy.py index e99247bb..514a2de9 100644 --- a/sld-api-backend/src/deploy/domain/entities/deploy.py +++ b/sld-api-backend/src/deploy/domain/entities/deploy.py @@ -57,4 +57,4 @@ class Deploy(DeployBase): user_id: int class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/gcp/domain/entities/gcp.py b/sld-api-backend/src/gcp/domain/entities/gcp.py index 11a154b9..2805d0a0 100644 --- a/sld-api-backend/src/gcp/domain/entities/gcp.py +++ b/sld-api-backend/src/gcp/domain/entities/gcp.py @@ -11,4 +11,4 @@ class Gcloud(GcloudBase): id: int class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/stacks/domain/entities/stacks.py b/sld-api-backend/src/stacks/domain/entities/stacks.py index 1745a8de..fb768dfd 100644 --- a/sld-api-backend/src/stacks/domain/entities/stacks.py +++ b/sld-api-backend/src/stacks/domain/entities/stacks.py @@ -15,7 +15,7 @@ class StackBase(BaseModel): class Config: """Extra configuration options""" - anystr_strip_whitespace = True # remove trailing whitespace + str_strip_whitespace = True # remove trailing whitespace class StackCreate(StackBase): @@ -24,7 +24,7 @@ class StackCreate(StackBase): class Config: """Extra configuration options""" - anystr_strip_whitespace = True # remove trailing whitespace + str_strip_whitespace = True # remove trailing whitespace class Stack(StackBase): @@ -33,4 +33,4 @@ class Stack(StackBase): user_id: int class Config: - orm_mode = True + from_attributes = True diff --git a/sld-api-backend/src/tasks/domain/entities/tasks.py b/sld-api-backend/src/tasks/domain/entities/tasks.py index 7dd10765..7910a757 100644 --- a/sld-api-backend/src/tasks/domain/entities/tasks.py +++ b/sld-api-backend/src/tasks/domain/entities/tasks.py @@ -36,14 +36,14 @@ class PasswordReset(BaseModel): passwd: str class Config: - orm_mode = True + from_attributes = True class User(UserBase): id: int class Config: - orm_mode = True + from_attributes = True class Token(BaseModel): diff --git a/sld-api-backend/src/users/domain/entities/users.py b/sld-api-backend/src/users/domain/entities/users.py index a761adda..8445755e 100644 --- a/sld-api-backend/src/users/domain/entities/users.py +++ b/sld-api-backend/src/users/domain/entities/users.py @@ -36,14 +36,14 @@ class PasswordReset(BaseModel): passwd: str class Config: - orm_mode = True + from_attributes = True class User(UserBase): id: int class Config: - orm_mode = True + from_attributes = True class Token(BaseModel): diff --git a/sld-api-backend/src/worker/helpers/hcl2_to_json.py b/sld-api-backend/src/worker/helpers/hcl2_to_json.py new file mode 100644 index 00000000..21de38b7 --- /dev/null +++ b/sld-api-backend/src/worker/helpers/hcl2_to_json.py @@ -0,0 +1,29 @@ +import hcl2 + + +def convert_to_json(file_path): + with open(file_path, "r") as f: + hcl_data = hcl2.load(f) + clean_data = remove_interpolations(hcl_data) + json_data = list_to_dict(clean_data["variable"]) + return json_data + + +def remove_interpolations(data): + if isinstance(data, dict): + return {k: remove_interpolations(v) for k, v in data.items()} + elif isinstance(data, list): + return [remove_interpolations(elem) for elem in data] + elif isinstance(data, str): + return data.replace("${", "").replace("}", "") + else: + return data + + +def is_interpolation(value): + return isinstance(value, str) and "${" in value + + +def list_to_dict(variables_list: list) -> dict: + variables_dict = {k: v for d in variables_list for k, v in d.items()} + return variables_dict \ No newline at end of file diff --git a/sld-api-backend/src/worker/providers/hashicorp/templates.py b/sld-api-backend/src/worker/providers/hashicorp/templates.py index 52c59e48..76c8ea8c 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/templates.py +++ b/sld-api-backend/src/worker/providers/hashicorp/templates.py @@ -1,8 +1,10 @@ import json +import logging from dataclasses import dataclass import hcl from jinja2 import Template +from src.worker.helpers.hcl2_to_json import convert_to_json @dataclass @@ -13,6 +15,11 @@ class StructBase: squad: str +@dataclass +class StructProject(StructBase): + project_path: str + + @dataclass class Backend(StructBase): project_path: str @@ -62,8 +69,7 @@ def save(self) -> dict: @dataclass -class GetVars(StructBase): - project_path: str +class GetVars(StructProject): """ In this class are the methods to obtain information from the terraform variables """ @@ -73,18 +79,52 @@ def __set_path(self): return f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/variables.tf" return f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}/variables.tf" + def process_dict(self, d): + return { + key: ( + value.replace("${", "").replace("}", "") + if isinstance(value, str) + else ( + self.process_dict(value) + if isinstance(value, dict) + else [self.process_dict(item) for item in value] + if isinstance(value, list) + else value + ) + ) + for key, value in d.items() + } + def get_vars_json(self) -> dict: try: file_hcl = self.__set_path() with open(file_hcl, "r") as fp: - obj = hcl.load(fp) - if obj.get("variable"): - return {"command": "get_vars_json", "rc": 0, "stdout": json.dumps(obj)} + hcl_data = hcl.load(fp) + if hcl_data.get("variable"): + return { + "command": "get_vars_json", + "rc": 0, + "stdout": json.dumps(hcl_data), + } else: error_msg = "Variable file is empty, not iterable" return {"command": "get_vars_json", "rc": 1, "stdout": error_msg} except IOError: error_msg = "Variable file not accessible" return {"command": "get_vars_json", "rc": 1, "stdout": error_msg} + except ValueError as err: + logging.warn(err) + logging.warn("hclv1 cannot read variables.tf trying to read with hcl2") + try: + result = {"variable": convert_to_json(file_hcl)} + return { + "command": "get_vars_json", + "rc": 0, + "stdout": json.dumps(result), + } + except Exception as err: + error_msg = f"Syntax error in variable file(check with terraform validate): {err}" + logging.error(error_msg) + return {"command": "get_vars_json", "rc": 1, "stdout": error_msg} except Exception as err: - return {"command": "get_vars_json", "rc": 1, "stdout": err} + return {"command": "get_vars_json", "rc": 1, "stdout": err} \ No newline at end of file