From 52a35782f796d104c874f5081fb190228950a337 Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Sun, 26 Nov 2023 16:31:07 +0100 Subject: [PATCH 1/9] =?UTF-8?q?=F0=9F=94=A7refactor:=20terraform=20command?= =?UTF-8?q?s=20action=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/worker/domain/services/provider.py | 6 +- .../src/worker/providers/hashicorp/actions.py | 243 +++--------------- 2 files changed, 37 insertions(+), 212 deletions(-) diff --git a/sld-api-backend/src/worker/domain/services/provider.py b/sld-api-backend/src/worker/domain/services/provider.py index 1e307766..dcd41db2 100644 --- a/sld-api-backend/src/worker/domain/services/provider.py +++ b/sld-api-backend/src/worker/domain/services/provider.py @@ -98,7 +98,7 @@ def plan( variables_file, project_path, ) - return config_action.plan_execute() + return config_action.execute_terraform_command("plan") def apply( name: str, @@ -123,7 +123,7 @@ def apply( variables_file, project_path, ) - return config_action.apply_execute() + return config_action.execute_terraform_command("apply") def destroy( name: str, @@ -148,7 +148,7 @@ def destroy( variables_file, project_path, ) - return config_action.destroy_execute() + return config_action.execute_terraform_command("destroy") def output( stack_name: str, squad: str, environment: str, name: str, action=SimpleActions diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py index 1dc4ac94..b635d973 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/actions.py +++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py @@ -24,66 +24,42 @@ class Actions(StructBase): secreto: dict variables_file: str project_path: str - """ - In this class are all the methods equivalent to the terraform commands - """ - def plan_execute(self) -> dict: + def execute_terraform_command(self, command: str) -> dict: try: - secret( - self.stack_name, self.environment, self.squad, self.name, self.secreto - ) - deploy_state = ( - f"{self.environment}_{self.stack_name}_{self.squad}_{self.name}" - ) - # Execute task + secret(self.stack_name, self.environment, self.squad, self.name, self.secreto) + deploy_state = f"{self.environment}_{self.stack_name}_{self.squad}_{self.name}" + variables_files = ( f"{self.stack_name}.tfvars.json" - if self.variables_file == "" or self.variables_file == None + if not self.variables_file else self.variables_file ) if not self.project_path: - os.chdir( - f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}" - ) - os.chdir( - f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}" - ) - result = subprocess.run( - f"/tmp/{self.version}/terraform init -input=false --upgrade", - shell=True, - capture_output=True, - encoding="utf8", - ) - result = subprocess.run( - f"/tmp/{self.version}/terraform plan -input=false -refresh -no-color -var-file={variables_files} -out={self.stack_name}.tfplan", - shell=True, - capture_output=True, - encoding="utf8", - ) - unsecret( - self.stack_name, self.environment, self.squad, self.name, self.secreto - ) + os.chdir(f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}") + else: + os.chdir(f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}") + + init_command = f"/tmp/{self.version}/terraform init -input=false --upgrade" + plan_command = f"/tmp/{self.version}/terraform {command} -input=false -refresh -no-color -var-file={variables_files} -out={self.stack_name}.tfplan" + apply_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color {self.stack_name}.tfplan" + destroy_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color -var-file={variables_files}" + + result = subprocess.run(init_command, shell=True, capture_output=True, encoding="utf8") + result = subprocess.run(plan_command, shell=True, capture_output=True, encoding="utf8") + + if command == "apply": + result = subprocess.run(apply_command, shell=True, capture_output=True, encoding="utf8") + elif command == "destroy": + result = subprocess.run(destroy_command, shell=True, capture_output=True, encoding="utf8") + + unsecret(self.stack_name, self.environment, self.squad, self.name, self.secreto) - # Capture events rc = result.returncode - # check result - if rc != 0: - return { - "command": "plan", - "deploy": self.name, - "squad": self.squad, - "stack_name": self.stack_name, - "environment": self.environment, - "rc": rc, - "tfvars_files": self.variables_file, - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "stdout": result.stderr.split("\n"), - } - return { - "command": "plan", + + output_data = { + "command": command, "deploy": self.name, "squad": self.squad, "stack_name": self.stack_name, @@ -92,175 +68,24 @@ def plan_execute(self) -> dict: "tfvars_files": self.variables_file, "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "stdout": result.stdout.split("\n"), + "stdout": result.stderr.split("\n") if rc != 0 else result.stdout.split("\n"), } - except Exception: + + return output_data + + except Exception as e: return { - "command": "plan", + "command": command, "deploy": self.name, "squad": self.squad, "stack_name": self.stack_name, "environment": self.environment, - "rc": rc, - "tfvars_files": self.variables_file, - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "stdout": result.stderr.split("\n"), - } - - def apply_execute(self) -> dict: - try: - secret( - self.stack_name, self.environment, self.squad, self.name, self.secreto - ) - deploy_state = ( - f"{self.environment}_{self.stack_name}_{self.squad}_{self.name}" - ) - # Execute task - - if not self.project_path: - os.chdir( - f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}" - ) - os.chdir( - f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}" - ) - - result = subprocess.run( - f"/tmp/{self.version}/terraform init -input=false --upgrade", - shell=True, - capture_output=True, - encoding="utf8", - ) - result = subprocess.run( - f"/tmp/{self.version}/terraform apply -input=false -auto-approve -no-color {self.stack_name}.tfplan", - shell=True, - capture_output=True, - encoding="utf8", - ) - unsecret( - self.stack_name, self.environment, self.squad, self.name, self.secreto - ) - - # Capture events - rc = result.returncode - # check result - if rc != 0: - return { - "command": "apply", - "deploy": self.name, - "self.squad": self.squad, - "self.stack_name": self.stack_name, - "self.environment": self.environment, - "rc": rc, - "tfvars_files": self.variables_file, - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "stdout": result.stderr.split("\n"), - } - return { - "command": "apply", - "deploy": self.name, - "self.squad": self.squad, - "self.stack_name": self.stack_name, - "self.environment": self.environment, - "rc": rc, - "tfvars_files": self.variables_file, - "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "stdout": result.stdout.split("\n"), - } - except Exception: - return { - "command": "apply", - "deploy": self.name, - "self.squad": self.squad, - "self.stack_name": self.stack_name, - "self.environment": self.environment, "rc": 1, "tfvars_files": self.variables_file, - "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "stdout": "ko", - } - - def destroy_execute(self) -> dict: - try: - secret( - self.stack_name, self.environment, self.squad, self.name, self.secreto - ) - deploy_state = ( - f"{self.environment}_{self.stack_name}_{self.squad}_{self.name}" - ) - # Execute task - variables_files = ( - f"{self.stack_name}.tfvars.json" - if self.variables_file == "" or self.variables_file == None - else self.variables_file - ) - if not self.project_path: - os.chdir( - f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}" - ) - os.chdir( - f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}" - ) - - result = subprocess.run( - f"/tmp/{self.version}/terraform init -input=false --upgrade", - shell=True, - capture_output=True, - encoding="utf8", - ) - result = subprocess.run( - f"/tmp/{self.version}/terraform destroy -input=false -auto-approve -no-color -var-file={variables_files}", - shell=True, - capture_output=True, - encoding="utf8", - ) - unsecret( - self.stack_name, self.environment, self.squad, self.name, self.secreto - ) - # Capture events - rc = result.returncode - # check result - if rc != 0: - return { - "command": "destroy", - "deploy": self.name, - "self.squad": self.squad, - "self.stack_name": self.stack_name, - "self.environment": self.environment, - "rc": rc, - "tfvars_files": self.variables_file, - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "stdout": result.stderr.split("\n"), - } - return { - "command": "destroy", - "deploy": self.name, - "self.squad": self.squad, - "self.stack_name": self.stack_name, - "self.environment": self.environment, - "rc": rc, - "tfvars_files": self.variables_file, - "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "stdout": result.stdout.split("\n"), - } - except Exception: - return { - "command": "destroy", - "deploy": self.name, - "self.squad": self.squad, - "self.stack_name": self.stack_name, - "self.environment": self.environment, - "rc": 1, - "tfvars_files": self.variables_file, - "self.project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", + "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", "stdout": "ko", + "error_message": str(e), } From 782c00a98bc625af0045a71bc20f01261d6ff6c6 Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Sun, 26 Nov 2023 18:43:49 +0100 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=94=A7refactor:=20pydantic=20provider?= =?UTF-8?q?=20validate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/worker/domain/entities/worker.py | 64 ++++++++++++- .../src/worker/domain/services/pipeline.py | 94 +++++++++---------- .../src/worker/domain/services/provider.py | 94 ++++++------------- .../src/worker/providers/hashicorp/actions.py | 50 ++++++++-- .../src/worker/tasks/terraform_worker.py | 7 +- sld-dashboard/app/home/routes.py | 1 + 6 files changed, 179 insertions(+), 131 deletions(-) diff --git a/sld-api-backend/src/worker/domain/entities/worker.py b/sld-api-backend/src/worker/domain/entities/worker.py index 74c1d0a1..b2a1634e 100644 --- a/sld-api-backend/src/worker/domain/entities/worker.py +++ b/sld-api-backend/src/worker/domain/entities/worker.py @@ -1,13 +1,48 @@ from pydantic import BaseModel -from typing import Optional, Any +from typing import Optional, Any, Dict + + +import logging +from typing import List, Tuple +from subprocess import Popen, PIPE + + +class SubprocessHandler: + def run_command(self, command: str) -> Tuple[int, List[str]]: + try: + process = Popen( + command, + shell=True, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True + ) + + # Read stdout and stderr in real-time + output_lines = [] + while True: + line = process.stdout.readline() + logging.info(line.rstrip('\n')) + if not line: + break + output_lines.append(line.strip()) + + # Wait for the process to finish + returncode = process.wait() + return returncode, output_lines + + except Exception as e: + return 1, [str(e)] + class DownloadBinaryParams(BaseModel): version: str - #url: Optional[str] - + + # url: Optional[str] class Config: frozenset = True + class DeployParamsBase(BaseModel): name: str stack_name: str @@ -30,6 +65,7 @@ class TfvarsParams(DeployParamsBase): class config: frozenset = True + class DeployParams(BaseModel): git_repo: str name: str @@ -38,11 +74,12 @@ class DeployParams(BaseModel): squad: str branch: str version: str - variables: Any + variables: Optional[Dict[str, Any]] = {} secreto: Any variables_file: Optional[str] = "" project_path: Optional[str] = "" user: Optional[str] = "" + task_id: Optional[str] = "" class Config: frozenset = True @@ -60,6 +97,7 @@ class DownloadGitRepoParams(BaseModel): class Config: frozenset = True + class ApplyParams(DeployParamsBase, DownloadBinaryParams): branch: str secreto: Any @@ -69,8 +107,24 @@ class ApplyParams(DeployParamsBase, DownloadBinaryParams): class Config: frozenset = True + class PlanParams(ApplyParams): pass + class DestroyParams(ApplyParams): - pass \ No newline at end of file + pass + + +class ActionBase(BaseModel): + name: str + stack_name: str + branch: str + environment: str + squad: str + version: str + secreto: dict + project_path: Optional[str] = "" + variables_file: Optional[str] = "" + task_id: Optional[str] = "" + subprocess_handler: Optional[Any] = SubprocessHandler() diff --git a/sld-api-backend/src/worker/domain/services/pipeline.py b/sld-api-backend/src/worker/domain/services/pipeline.py index 90560021..4f5a3eb0 100644 --- a/sld-api-backend/src/worker/domain/services/pipeline.py +++ b/sld-api-backend/src/worker/domain/services/pipeline.py @@ -14,6 +14,7 @@ decode_responses=True, ) + class DownloadGitRepo: def __init__(self, params: DownloadGitRepoParams, provider: ProviderRequirements): self.params = params @@ -29,6 +30,8 @@ def __call__(self): self.params.branch, self.params.project_path ) + + class DownloadBinary: def __init__(self, params: DownloadBinaryParams, provider: ProviderRequirements): self.params = params @@ -39,6 +42,7 @@ def __call__(self): self.params.version, ) + class RemoteState: def __init__(self, params: RemoteStateParams, provider: ProviderRequirements): self.params = params @@ -53,6 +57,7 @@ def __call__(self): self.params.project_path ) + class Tfvars: def __init__(self, params: TfvarsParams, provider: ProviderRequirements): self.params = params @@ -68,6 +73,7 @@ def __call__(self): self.params.variables ) + class GetVariables: def __init__(self, params: GetVariablesParams, provider: ProviderGetVars): self.params = params @@ -75,30 +81,22 @@ def __init__(self, params: GetVariablesParams, provider: ProviderGetVars): def __call__(self): return self.provider.json_vars( - self.params.name, - self.params.stack_name, - self.params.environment, + self.params.name, + self.params.stack_name, + self.params.environment, self.params.squad, self.params.project_path ) + class plan: def __init__(self, params: PlanParams, provider: ProviderActions): self.params = params self.provider = provider def __call__(self): - return self.provider.plan( - self.params.name, - self.params.stack_name, - self.params.branch, - self.params.environment, - self.params.squad, - self.params.version, - self.params.secreto, - self.params.variables_file, - self.params.project_path, - ) + return self.provider.plan(self.params) + class apply: def __init__(self, params: ApplyParams, provider: ProviderActions): @@ -106,40 +104,23 @@ def __init__(self, params: ApplyParams, provider: ProviderActions): self.provider = provider def __call__(self): - return self.provider.apply( - self.params.name, - self.params.stack_name, - self.params.branch, - self.params.environment, - self.params.squad, - self.params.version, - self.params.secreto, - self.params.variables_file, - self.params.project_path, - ) + return self.provider.apply(self.params) + class destroy: - def __init__(self, params: DestroyParams, provider: ProviderActions): + def __init__(self, params: DeployParams, provider: ProviderActions): self.params = params self.provider = provider def __call__(self): - return self.provider.destroy( - self.params.name, - self.params.stack_name, - self.params.branch, - self.params.environment, - self.params.squad, - self.params.version, - self.params.secreto, - self.params.variables_file, - self.params.project_path, - ) - + return self.provider.destroy(self.params) + + # PIPELINE class Pipeline: def __init__(self, params: DeployParams): self.params = params + def locked_task(self): logging.info(f"Checking if task {self.params.name}-{self.params.squad}-{self.params.environment} is locked in cache server {settings.CACHE_SERVER}") logging.info(f"Locking task {self.params.name}-{self.params.squad}-{self.params.environment}") @@ -148,8 +129,7 @@ def locked_task(self): def unlock_task(self): r.delete(f"{self.params.name}-{self.params.squad}-{self.params.environment}") - - + # Git clone repo def download_git_repo(self): git_params = DownloadGitRepoParams( @@ -166,8 +146,7 @@ def download_git_repo(self): if download_git_repo_result["rc"] != 0: raise Exception(download_git_repo_result) return download_git_repo_result - - + # Download terrafom def download_binary(self): binari_params = DownloadBinaryParams( @@ -209,7 +188,6 @@ def set_tfvars(self): if tfvars_result["rc"] != 0: raise Exception(tfvars_result) return tfvars_result - def get_variables(self): get_variables_params = GetVariablesParams( @@ -227,16 +205,20 @@ def get_variables(self): # Plan execute def execute_plan(self): - plan_params = PlanParams( + plan_params = DeployParams( + git_repo=self.params.git_repo, name=self.params.name, stack_name=self.params.stack_name, - branch=self.params.branch, environment=self.params.environment, squad=self.params.squad, + branch=self.params.branch, version=self.params.version, + variables=self.params.variables, + project_path=self.params.project_path, secreto=self.params.secreto, variables_file=self.params.variables_file, - project_path=self.params.project_path, + user=self.params.user, + task_id=self.params.task_id ) plan_execute = plan(params=plan_params, provider=ProviderActions) plan_result = plan_execute() @@ -246,16 +228,20 @@ def execute_plan(self): # Apply execute def execute_apply(self): - apply_params = ApplyParams( + apply_params = DeployParams( + git_repo=self.params.git_repo, name=self.params.name, stack_name=self.params.stack_name, - branch=self.params.branch, environment=self.params.environment, squad=self.params.squad, + branch=self.params.branch, version=self.params.version, + variables=self.params.variables, + project_path=self.params.project_path, secreto=self.params.secreto, variables_file=self.params.variables_file, - project_path=self.params.project_path, + user=self.params.user, + task_id=self.params.task_id ) apply_execute = apply(params=apply_params, provider=ProviderActions) apply_result = apply_execute() @@ -265,16 +251,20 @@ def execute_apply(self): # Destroy execute def execute_destroy(self): - destroy_params = DestroyParams( + destroy_params = DeployParams( + git_repo=self.params.git_repo, name=self.params.name, stack_name=self.params.stack_name, - branch=self.params.branch, environment=self.params.environment, squad=self.params.squad, + branch=self.params.branch, version=self.params.version, + variables=self.params.variables, + project_path=self.params.project_path, secreto=self.params.secreto, variables_file=self.params.variables_file, - project_path=self.params.project_path, + user=self.params.user, + task_id=self.params.task_id ) destroy_execute = destroy(params=destroy_params, provider=ProviderActions) destroy_result = destroy_execute() diff --git a/sld-api-backend/src/worker/domain/services/provider.py b/sld-api-backend/src/worker/domain/services/provider.py index dcd41db2..bd008bbf 100644 --- a/sld-api-backend/src/worker/domain/services/provider.py +++ b/sld-api-backend/src/worker/domain/services/provider.py @@ -3,6 +3,7 @@ from src.worker.providers.hashicorp.artifact import Artifact from src.worker.providers.hashicorp.download import BinaryDownload from src.worker.providers.hashicorp.templates import Backend, GetVars, Tfvars +from src.worker.domain.entities.worker import DeployParams class ProviderRequirements: @@ -75,78 +76,45 @@ class ProviderActions: This class contains the typical methods of a deployment """ - def plan( - name: str, - stack_name: str, - branch: str, - environment: str, - squad: str, - version: str, - secreto: dict, - variables_file: str = "", - project_path: str = "", - action=Actions, - ) -> dict: + def plan(params: DeployParams, action: Actions = Actions) -> dict: config_action = action( - name, - stack_name, - branch, - environment, - squad, - version, - secreto, - variables_file, - project_path, + params.name, + params.stack_name, + params.branch, + params.environment, + params.squad, + params.version, + params.secreto, + params.variables_file, + params.project_path, ) return config_action.execute_terraform_command("plan") - def apply( - name: str, - stack_name: str, - branch: str, - environment: str, - squad: str, - version: str, - secreto: dict, - variables_file: str = "", - project_path: str = "", - action=Actions, - ) -> dict: + def apply(params: DeployParams, action: Actions = Actions) -> dict: config_action = action( - name, - stack_name, - branch, - environment, - squad, - version, - secreto, - variables_file, - project_path, + params.name, + params.stack_name, + params.branch, + params.environment, + params.squad, + params.version, + params.secreto, + params.variables_file, + params.project_path, ) return config_action.execute_terraform_command("apply") - def destroy( - name: str, - stack_name: str, - branch: str, - environment: str, - squad: str, - version: str, - secreto: dict, - variables_file: str = "", - project_path: str = "", - action=Actions, - ) -> dict: + def destroy(params: DeployParams, action: Actions = Actions) -> dict: config_action = action( - name, - stack_name, - branch, - environment, - squad, - version, - secreto, - variables_file, - project_path, + params.name, + params.stack_name, + params.branch, + params.environment, + params.squad, + params.version, + params.secreto, + params.variables_file, + params.project_path, ) return config_action.execute_terraform_command("destroy") diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py index b635d973..ddd1900e 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/actions.py +++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py @@ -1,7 +1,9 @@ import os +import logging +from typing import List, Tuple import subprocess from dataclasses import dataclass - +from subprocess import Popen, PIPE, STDOUT import jmespath import requests from config.api import settings @@ -9,6 +11,34 @@ from src.worker.security.providers_credentials import secret, unsecret +class SubprocessHandler: + def run_command(self, command: str) -> Tuple[int, List[str]]: + try: + process = Popen( + command, + shell=True, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True + ) + + # Read stdout and stderr in real-time + output_lines = [] + while True: + line = process.stdout.readline() + logging.info(line.rstrip('\n')) + if not line: + break + output_lines.append(line.strip()) + + # Wait for the process to finish + returncode = process.wait() + return returncode, output_lines + + except Exception as e: + return 1, [str(e)] + + @dataclass class StructBase: name: str @@ -24,6 +54,7 @@ class Actions(StructBase): secreto: dict variables_file: str project_path: str + subprocess_handler = SubprocessHandler() def execute_terraform_command(self, command: str) -> dict: try: @@ -41,22 +72,21 @@ def execute_terraform_command(self, command: str) -> dict: else: os.chdir(f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}") - init_command = f"/tmp/{self.version}/terraform init -input=false --upgrade" + init_command = f"/tmp/{self.version}/terraform init -no-color -input=false --upgrade" plan_command = f"/tmp/{self.version}/terraform {command} -input=false -refresh -no-color -var-file={variables_files} -out={self.stack_name}.tfplan" apply_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color {self.stack_name}.tfplan" destroy_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color -var-file={variables_files}" - result = subprocess.run(init_command, shell=True, capture_output=True, encoding="utf8") - result = subprocess.run(plan_command, shell=True, capture_output=True, encoding="utf8") + result, output = self.subprocess_handler.run_command(init_command) + result, output = self.subprocess_handler.run_command(plan_command) if command == "apply": - result = subprocess.run(apply_command, shell=True, capture_output=True, encoding="utf8") + result, output = self.subprocess_handler.run_command(apply_command) elif command == "destroy": - result = subprocess.run(destroy_command, shell=True, capture_output=True, encoding="utf8") + result, output = self.subprocess_handler.run_command(destroy_command) unsecret(self.stack_name, self.environment, self.squad, self.name, self.secreto) - - rc = result.returncode + rc = result output_data = { "command": command, @@ -68,7 +98,9 @@ def execute_terraform_command(self, command: str) -> dict: "tfvars_files": self.variables_file, "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", - "stdout": result.stderr.split("\n") if rc != 0 else result.stdout.split("\n"), + "stdout": output, + + } return output_data diff --git a/sld-api-backend/src/worker/tasks/terraform_worker.py b/sld-api-backend/src/worker/tasks/terraform_worker.py index 8ea130d6..daa566de 100644 --- a/sld-api-backend/src/worker/tasks/terraform_worker.py +++ b/sld-api-backend/src/worker/tasks/terraform_worker.py @@ -7,10 +7,10 @@ from celery.utils.log import get_task_logger from config.api import settings from config.celery_config import celery_app -from src.worker.domain.entities.worker import DeployParams, DestroyParams, PlanParams, DownloadGitRepoParams, GetVariablesParams +from src.worker.domain.entities.worker import DeployParams, DownloadGitRepoParams from src.worker.domain.services.pipeline import Pipeline -from src.worker.domain.services.provider import ProviderActions, ProviderGetVars, ProviderRequirements +from src.worker.domain.services.provider import ProviderActions from src.worker.tasks.helpers.folders import Utils from src.worker.tasks.helpers.metrics import push_metric from src.worker.tasks.helpers.schedule import request_url @@ -36,6 +36,7 @@ def pipeline_deploy( ): try: + params["task_id"] = self.request.id params = DeployParams(**params) pipeline = Pipeline(params=params) pipeline.locked_task() @@ -74,6 +75,7 @@ def pipeline_destroy( params: DeployParams, ): try: + params["task_id"] = self.request.id params = DeployParams(**params) pipeline = Pipeline(params=params) pipeline.locked_task() @@ -107,6 +109,7 @@ def pipeline_plan( params: DeployParams, ): try: + params["task_id"] = self.request.id params = DeployParams(**params) pipeline = Pipeline(params=params) pipeline.locked_task() diff --git a/sld-dashboard/app/home/routes.py b/sld-dashboard/app/home/routes.py index 66e7d4a3..eefe5095 100644 --- a/sld-dashboard/app/home/routes.py +++ b/sld-dashboard/app/home/routes.py @@ -546,6 +546,7 @@ def clone_deploy(deploy_id): "branch", "tfvar_file", "project_path", + "deploy_name", "squad", "environment", ] From 6c96c557f7adbbe462933da9af631ef415db262b Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Mon, 27 Nov 2023 02:07:16 +0100 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=90=9Bfix:=20when=20terraform=20error?= =?UTF-8?q?=20not=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sld-api-backend/config/api.py | 2 +- sld-api-backend/config/celery_config.py | 3 +- .../src/worker/domain/services/command.py | 35 ++++++++++++++ .../src/worker/providers/hashicorp/actions.py | 46 ++++--------------- .../worker/providers/hashicorp/templates.py | 4 +- .../app/home/templates/deploy-edit.html | 2 +- 6 files changed, 50 insertions(+), 42 deletions(-) create mode 100644 sld-api-backend/src/worker/domain/services/command.py diff --git a/sld-api-backend/config/api.py b/sld-api-backend/config/api.py index 36164151..482ed62c 100644 --- a/sld-api-backend/config/api.py +++ b/sld-api-backend/config/api.py @@ -50,7 +50,7 @@ class Settings(BaseSettings): AWS_CONGIG_DEFAULT_FOLDER: str = f"{os.environ['HOME']}/.aws" AWS_SHARED_CREDENTIALS_FILE: str = f"{AWS_CONGIG_DEFAULT_FOLDER}/credentials" AWS_SHARED_CONFIG_FILE: str = f"{AWS_CONGIG_DEFAULT_FOLDER}/config" - TASK_MAX_RETRY: int = os.getenv("SLD_TASK_MAX_RETRY", 1) + TASK_MAX_RETRY: int = os.getenv("SLD_TASK_MAX_RETRY", 0) TASK_RETRY_INTERVAL: int = os.getenv("SLD_TASK_RETRY_INTERVAL", 20) TASK_LOCKED_EXPIRED: int = os.getenv("SLD_TASK_LOCKED_EXPIRED", 300) TASK_ROUTE: bool = os.getenv("SLD_TASK_ROUTE", False) diff --git a/sld-api-backend/config/celery_config.py b/sld-api-backend/config/celery_config.py index f8f3f0cf..22e01b8c 100644 --- a/sld-api-backend/config/celery_config.py +++ b/sld-api-backend/config/celery_config.py @@ -38,4 +38,5 @@ celery_app.conf.update(task_track_started=True) celery_app.conf.update(result_extended=True) celery_app.conf.broker_transport_options = {"visibility_timeout": 28800} # 8 hours. -celery_app.conf.result_expires = os.getenv("SLD_RESULT_EXPIRE", "259200") \ No newline at end of file +celery_app.conf.result_expires = os.getenv("SLD_RESULT_EXPIRE", "259200") +celery_app.conf.broker_connection_retry_on_startup = True \ No newline at end of file diff --git a/sld-api-backend/src/worker/domain/services/command.py b/sld-api-backend/src/worker/domain/services/command.py new file mode 100644 index 00000000..f65df2c2 --- /dev/null +++ b/sld-api-backend/src/worker/domain/services/command.py @@ -0,0 +1,35 @@ +import logging +import subprocess +from subprocess import Popen, PIPE +from typing import Tuple, List + + +class SubprocessHandler: + def run_command(self, command: str) -> Tuple[int, List[str], List[str]]: + try: + process = Popen( + command, + shell=True, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True + ) + + stdout_lines, stderr_lines = process.communicate() + + output_lines = stdout_lines.strip().split('\n') if stdout_lines else [] + error_lines = stderr_lines.strip().split('\n') if stderr_lines else [] + + if process.returncode == 0: + return process.returncode, output_lines + return process.returncode, error_lines + + except subprocess.CalledProcessError as err: + logging.error( + f"Error executing command, code: {err.returncode}. Error:\n{err.output}" + ) + return err.returncode, [], [err.output] + + except Exception as e: + logging.error(f"An error occurred: {e}") + return 1, [], [str(e)] diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py index ddd1900e..27a14af5 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/actions.py +++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py @@ -1,42 +1,13 @@ import os -import logging -from typing import List, Tuple -import subprocess from dataclasses import dataclass -from subprocess import Popen, PIPE, STDOUT import jmespath import requests from config.api import settings from src.worker.security.providers_credentials import secret, unsecret +from src.worker.domain.services.command import SubprocessHandler -class SubprocessHandler: - def run_command(self, command: str) -> Tuple[int, List[str]]: - try: - process = Popen( - command, - shell=True, - stdout=PIPE, - stderr=PIPE, - universal_newlines=True - ) - - # Read stdout and stderr in real-time - output_lines = [] - while True: - line = process.stdout.readline() - logging.info(line.rstrip('\n')) - if not line: - break - output_lines.append(line.strip()) - - # Wait for the process to finish - returncode = process.wait() - return returncode, output_lines - - except Exception as e: - return 1, [str(e)] @dataclass @@ -54,7 +25,7 @@ class Actions(StructBase): secreto: dict variables_file: str project_path: str - subprocess_handler = SubprocessHandler() + subprocess_handler: SubprocessHandler = SubprocessHandler() def execute_terraform_command(self, command: str) -> dict: try: @@ -62,7 +33,7 @@ def execute_terraform_command(self, command: str) -> dict: deploy_state = f"{self.environment}_{self.stack_name}_{self.squad}_{self.name}" variables_files = ( - f"{self.stack_name}.tfvars.json" + f"{self.name}.tfvars.json" if not self.variables_file else self.variables_file ) @@ -73,8 +44,8 @@ def execute_terraform_command(self, command: str) -> dict: os.chdir(f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}") init_command = f"/tmp/{self.version}/terraform init -no-color -input=false --upgrade" - plan_command = f"/tmp/{self.version}/terraform {command} -input=false -refresh -no-color -var-file={variables_files} -out={self.stack_name}.tfplan" - apply_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color {self.stack_name}.tfplan" + plan_command = f"/tmp/{self.version}/terraform {command} -input=false -refresh -no-color -var-file={variables_files} -out={self.name}.tfplan" + apply_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color {self.name}.tfplan" destroy_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color -var-file={variables_files}" result, output = self.subprocess_handler.run_command(init_command) @@ -86,6 +57,7 @@ def execute_terraform_command(self, command: str) -> dict: result, output = self.subprocess_handler.run_command(destroy_command) unsecret(self.stack_name, self.environment, self.squad, self.name, self.secreto) + rc = result output_data = { @@ -105,7 +77,7 @@ def execute_terraform_command(self, command: str) -> dict: return output_data - except Exception as e: + except Exception as err: return { "command": command, "deploy": self.name, @@ -116,8 +88,8 @@ def execute_terraform_command(self, command: str) -> dict: "tfvars_files": self.variables_file, "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", - "stdout": "ko", - "error_message": str(e), + "stdout": output, + "error_message": err, } diff --git a/sld-api-backend/src/worker/providers/hashicorp/templates.py b/sld-api-backend/src/worker/providers/hashicorp/templates.py index 8af4273b..cbdbd2f9 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/templates.py +++ b/sld-api-backend/src/worker/providers/hashicorp/templates.py @@ -58,9 +58,9 @@ class Tfvars(StructBase): def save(self) -> dict: try: - file_path = f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.stack_name}.tfvars.json" + file_path = f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.name}.tfvars.json" if self.project_path: - file_path = f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}/{self.stack_name}.tfvars.json" + file_path = f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}/{self.name}.tfvars.json" with open(file_path, "w") as tfvars_json: json.dump(self.variables, tfvars_json) return {"command": "tfvars", "rc": 0, "stdout": self.variables} diff --git a/sld-dashboard/app/home/templates/deploy-edit.html b/sld-dashboard/app/home/templates/deploy-edit.html index fce3d407..0ae08b27 100644 --- a/sld-dashboard/app/home/templates/deploy-edit.html +++ b/sld-dashboard/app/home/templates/deploy-edit.html @@ -108,7 +108,7 @@
{{key}}
{% endif %} - From 113a5cfb5ed8822593ce7bf789b516f7d82189f2 Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:47:46 +0100 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=94=A5feat:=20stream=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/worker/domain/services/command.py | 63 +++++++++++-------- .../src/worker/providers/hashicorp/actions.py | 15 +++-- sld-dashboard/app/home/routes.py | 13 +++- .../app/home/templates/deploy-stream.html | 46 ++++++++++++++ .../app/home/templates/deploys-list.html | 1 + 5 files changed, 103 insertions(+), 35 deletions(-) create mode 100644 sld-dashboard/app/home/templates/deploy-stream.html diff --git a/sld-api-backend/src/worker/domain/services/command.py b/sld-api-backend/src/worker/domain/services/command.py index f65df2c2..4d09042b 100644 --- a/sld-api-backend/src/worker/domain/services/command.py +++ b/sld-api-backend/src/worker/domain/services/command.py @@ -1,35 +1,46 @@ import logging import subprocess -from subprocess import Popen, PIPE -from typing import Tuple, List - - -class SubprocessHandler: - def run_command(self, command: str) -> Tuple[int, List[str], List[str]]: - try: - process = Popen( - command, - shell=True, - stdout=PIPE, - stderr=PIPE, - universal_newlines=True - ) +import redis + +redis_client = redis.Redis(host='localhost', port=6379, db=15) + + +def command(command: str): + try: + output_lines = [] + + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + logging.info("#" * 80) + for line in process.stdout: + output_lines.append(line.strip()) + logging.info(line.strip()) + try: + redis_client.publish('channel_name', line.strip()) + except Exception as err: + logging.error(f"Error publish redis: {err}") - stdout_lines, stderr_lines = process.communicate() - output_lines = stdout_lines.strip().split('\n') if stdout_lines else [] - error_lines = stderr_lines.strip().split('\n') if stderr_lines else [] + logging.info("#" * 80) - if process.returncode == 0: - return process.returncode, output_lines - return process.returncode, error_lines + process.wait() - except subprocess.CalledProcessError as err: + if process.returncode == 0: + logging.info(f"Command {command} executed successfully.") + else: logging.error( - f"Error executing command, code: {err.returncode}. Error:\n{err.output}" + f"Error executing the command {command}. Exit code: {process.returncode}" ) - return err.returncode, [], [err.output] - except Exception as e: - logging.error(f"An error occurred: {e}") - return 1, [], [str(e)] + return process.returncode, output_lines + except subprocess.CalledProcessError as err: + logging.error( + f"Error execute command code: {err.returncode}. Error:\n{err.stderr}" + ) + return None diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py index 27a14af5..835999a7 100644 --- a/sld-api-backend/src/worker/providers/hashicorp/actions.py +++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py @@ -5,7 +5,7 @@ from config.api import settings from src.worker.security.providers_credentials import secret, unsecret -from src.worker.domain.services.command import SubprocessHandler +from src.worker.domain.services.command import command @@ -25,7 +25,7 @@ class Actions(StructBase): secreto: dict variables_file: str project_path: str - subprocess_handler: SubprocessHandler = SubprocessHandler() + subprocess_handler: command = command def execute_terraform_command(self, command: str) -> dict: try: @@ -48,13 +48,13 @@ def execute_terraform_command(self, command: str) -> dict: apply_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color {self.name}.tfplan" destroy_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color -var-file={variables_files}" - result, output = self.subprocess_handler.run_command(init_command) - result, output = self.subprocess_handler.run_command(plan_command) + result, output = self.subprocess_handler(init_command) + result, output = self.subprocess_handler(plan_command) if command == "apply": - result, output = self.subprocess_handler.run_command(apply_command) + result, output = self.subprocess_handler(apply_command) elif command == "destroy": - result, output = self.subprocess_handler.run_command(destroy_command) + result, output = self.subprocess_handler(destroy_command) unsecret(self.stack_name, self.environment, self.squad, self.name, self.secreto) @@ -77,7 +77,7 @@ def execute_terraform_command(self, command: str) -> dict: return output_data - except Exception as err: + except Exception: return { "command": command, "deploy": self.name, @@ -89,7 +89,6 @@ def execute_terraform_command(self, command: str) -> dict: "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}", "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}", "stdout": output, - "error_message": err, } diff --git a/sld-dashboard/app/home/routes.py b/sld-dashboard/app/home/routes.py index eefe5095..5bb2b494 100644 --- a/sld-dashboard/app/home/routes.py +++ b/sld-dashboard/app/home/routes.py @@ -3,7 +3,7 @@ import json import time import logging -from flask import jsonify, render_template, request, url_for, redirect, flash +from flask import jsonify, render_template, request, url_for, redirect, flash, Response import redis @@ -31,6 +31,7 @@ def decrypt(secreto): # Move to config file after testing r = redis.Redis(host="redis", port=6379, db=1, charset="utf-8", decode_responses=True) +s = redis.Redis(host="redis", port=6379, db=15, charset="utf-8", decode_responses=True) external_api_dns = settings.EXTERNAL_DNS_API @@ -46,6 +47,16 @@ def index(): "index.html", segment="index", external_api_dns=external_api_dns ) + +@blueprint.route('/stream') +def stream(): + def generate(): + pubsub = s.pubsub() + pubsub.subscribe('channel_name') + for message in pubsub.listen(): + yield f"data: {message['data']}\n\n" + return Response(generate(), mimetype='text/event-stream') + @blueprint.route('/status/') def status(task_id): try: diff --git a/sld-dashboard/app/home/templates/deploy-stream.html b/sld-dashboard/app/home/templates/deploy-stream.html new file mode 100644 index 00000000..ef743b12 --- /dev/null +++ b/sld-dashboard/app/home/templates/deploy-stream.html @@ -0,0 +1,46 @@ +{% extends "layouts/base.html" %} +{% from 'helpers/_forms.html' import render_field %} + +{% block title %} Edit Deploy {% endblock %} + + +{% block stylesheets %}{% endblock stylesheets %} +{% block content %} + + +
+ {% include 'includes/navigation.html' %} + +
+ + + +
+

Deployment Output:

+ 

+        

+    

+
+
+                {% include 'includes/footer.html' %}
+
+

+
+{% endblock content %}
+
+
+{% block javascripts %}
+
+{% endblock javascripts %}
\ No newline at end of file
diff --git a/sld-dashboard/app/home/templates/deploys-list.html b/sld-dashboard/app/home/templates/deploys-list.html
index 0acded8a..e0da19fd 100644
--- a/sld-dashboard/app/home/templates/deploys-list.html
+++ b/sld-dashboard/app/home/templates/deploys-list.html
@@ -537,4 +537,5 @@ 
                                             data-task-id="{{deploy.task_id}}"
                                             data-deploy-id="{{deploy.id}}"
                                         >
-                                            
+                                            
                                         
                                     
                                     
@@ -270,19 +289,7 @@ 
All Deploys

                                             
                                         
                                     
-                                    {% if "yoda" in current_user.role or "darth_vader" in current_user.role %}
-                                    
-                                    
-                                        
-                                            
-                                        
-                                    
-                                    {% endif %}
+
                                 
                             
                         

From f37c1955a39ac80ad548fb646bee6cd11c357678 Mon Sep 17 00:00:00 2001
From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com>
Date: Tue, 28 Nov 2023 01:38:30 +0100
Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=94=A5feat:=20stream=20output=20chang?=
 =?UTF-8?q?e=20style?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 sld-dashboard/app/home/templates/deploy-stream.html | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sld-dashboard/app/home/templates/deploy-stream.html b/sld-dashboard/app/home/templates/deploy-stream.html
index 26a7e19f..6022db7d 100644
--- a/sld-dashboard/app/home/templates/deploy-stream.html
+++ b/sld-dashboard/app/home/templates/deploy-stream.html
@@ -19,9 +19,10 @@
     }
 
     #sse-data {
-        background-color: #000; /* Black background for the SSE data */
+        background-color: #1e1e1e; /* Dark background for the main container */
         color: #fff; /* White text */
-        font-family: Consolas /* 'Monaco', 'Lucida Console', 'Courier New', monospace; */
+        font-size: medium;
+        font-family: "Courier New";
         white-space: pre-wrap; /* To maintain data formatting */
         padding: 10px; /* Internal spacing */
         border-radius: 4px; /* Rounded borders */

From eeb3e09fa025eb802a56bbb642df4f7038aa4af4 Mon Sep 17 00:00:00 2001
From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com>
Date: Tue, 28 Nov 2023 15:50:11 +0100
Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=94=A5feat:=20stream=20output=20indic?=
 =?UTF-8?q?ate=20commands?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../src/worker/domain/services/command.py     | 22 +++++++++-----
 .../src/worker/domain/services/pipeline.py    |  4 +--
 .../src/worker/providers/hashicorp/actions.py | 30 ++++++++-----------
 .../src/worker/tasks/terraform_worker.py      |  2 --
 sld-dashboard/app/home/routes.py              |  5 ++--
 5 files changed, 30 insertions(+), 33 deletions(-)

diff --git a/sld-api-backend/src/worker/domain/services/command.py b/sld-api-backend/src/worker/domain/services/command.py
index 156c47bb..ab33fac8 100644
--- a/sld-api-backend/src/worker/domain/services/command.py
+++ b/sld-api-backend/src/worker/domain/services/command.py
@@ -8,6 +8,11 @@
 def command(command: str, channel: str):
     try:
         output_lines = []
+        command_description = f"Executing command: {command}"
+        output_lines.append(command_description)
+        redis_client.publish(channel, "-" * 80)
+        redis_client.publish(channel, command_description)
+        redis_client.publish(channel, "-" * 80)
 
         process = subprocess.Popen(
             command,
@@ -19,12 +24,14 @@ def command(command: str, channel: str):
 
         logging.info("#" * 80)
         for line in process.stdout:
-            output_lines.append(line.strip())
-            logging.info(line.strip())
-            try:
-                redis_client.publish(f'{channel}', line.strip())
-            except Exception as err:
-                logging.error(f"Error publish redis: {err}")
+            if "[DEBUG]" not in line:
+                cleaned_line = line.strip()
+                output_lines.append(cleaned_line)
+                logging.info(cleaned_line)
+                try:
+                    redis_client.publish(f'{channel}', cleaned_line)
+                except Exception as err:
+                    logging.error(f"Error publish redis: {err}")
 
         logging.info("#" * 80)
 
@@ -36,8 +43,7 @@ def command(command: str, channel: str):
             logging.error(
                 f"Error executing the command {command}. Exit code: {process.returncode}"
             )
-
-        return process.returncode, output_lines  
+        return process.returncode, output_lines
     except subprocess.CalledProcessError as err:
         logging.error(
             f"Error execute command code: {err.returncode}. Error:\n{err.stderr}"
diff --git a/sld-api-backend/src/worker/domain/services/pipeline.py b/sld-api-backend/src/worker/domain/services/pipeline.py
index 4f5a3eb0..9109945a 100644
--- a/sld-api-backend/src/worker/domain/services/pipeline.py
+++ b/sld-api-backend/src/worker/domain/services/pipeline.py
@@ -249,7 +249,7 @@ def execute_apply(self):
             raise Exception(apply_result)
         return apply_result
 
-   # Destroy execute 
+    # Destroy execute 
     def execute_destroy(self):
         destroy_params = DeployParams(
             git_repo=self.params.git_repo,
@@ -271,5 +271,3 @@ def execute_destroy(self):
         if destroy_result["rc"] != 0:
             raise Exception(destroy_result)
         return destroy_result
-    
-    
\ No newline at end of file
diff --git a/sld-api-backend/src/worker/providers/hashicorp/actions.py b/sld-api-backend/src/worker/providers/hashicorp/actions.py
index 3daaab91..14cf8d77 100644
--- a/sld-api-backend/src/worker/providers/hashicorp/actions.py
+++ b/sld-api-backend/src/worker/providers/hashicorp/actions.py
@@ -7,9 +7,6 @@
 from src.worker.security.providers_credentials import secret, unsecret
 from src.worker.domain.services.command import command
 
-
-        
-
 @dataclass
 class StructBase:
     name: str
@@ -28,7 +25,7 @@ class Actions(StructBase):
     task_id: str
     subprocess_handler: command = command
 
-    def execute_terraform_command(self, command: str) -> dict:
+    def execute_terraform_command(self, action: str) -> dict:
         channel = self.task_id
         try:
             secret(self.stack_name, self.environment, self.squad, self.name, self.secreto)
@@ -46,16 +43,17 @@ def execute_terraform_command(self, command: str) -> dict:
                 os.chdir(f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}")
 
             init_command = f"/tmp/{self.version}/terraform init -no-color -input=false --upgrade"
-            plan_command = f"/tmp/{self.version}/terraform {command} -input=false -refresh -no-color -var-file={variables_files} -out={self.name}.tfplan"
-            apply_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color {self.name}.tfplan"
-            destroy_command = f"/tmp/{self.version}/terraform {command} -input=false -auto-approve -no-color -var-file={variables_files}"
-
-            result, output = self.subprocess_handler(init_command, channel=channel)
-            result, output = self.subprocess_handler(plan_command, channel=channel)
-
-            if command == "apply":
+            plan_command = f"/tmp/{self.version}/terraform plan -input=false -refresh -no-color -var-file={variables_files} -out={self.name}.tfplan"
+            apply_command = f"/tmp/{self.version}/terraform apply -input=false -auto-approve -no-color {self.name}.tfplan"
+            destroy_command = f"/tmp/{self.version}/terraform destroy -input=false -auto-approve -no-color -var-file={variables_files}"
+
+            if action == "plan":
+                result, output = command(init_command, channel=channel)
+                result, output = self.subprocess_handler(plan_command, channel=channel)
+            if action == "apply":
                 result, output = self.subprocess_handler(apply_command, channel=channel)
-            elif command == "destroy":
+            elif action == "destroy":
+                result, output = command(init_command, channel=channel)
                 result, output = self.subprocess_handler(destroy_command, channel=channel)
 
             unsecret(self.stack_name, self.environment, self.squad, self.name, self.secreto)
@@ -63,7 +61,7 @@ def execute_terraform_command(self, command: str) -> dict:
             rc = result
 
             output_data = {
-                "command": command,
+                "command": action,
                 "deploy": self.name,
                 "squad": self.squad,
                 "stack_name": self.stack_name,
@@ -73,15 +71,13 @@ def execute_terraform_command(self, command: str) -> dict:
                 "remote_state": f"http://remote-state:8080/terraform_state/{deploy_state}",
                 "project_path": f"/tmp/{self.stack_name}/{self.environment}/{self.squad}/{self.name}/{self.project_path}",
                 "stdout": output,
-
-
             }
 
             return output_data
 
         except Exception:
             return {
-                "command": command,
+                "command": action,
                 "deploy": self.name,
                 "squad": self.squad,
                 "stack_name": self.stack_name,
diff --git a/sld-api-backend/src/worker/tasks/terraform_worker.py b/sld-api-backend/src/worker/tasks/terraform_worker.py
index daa566de..a827bfaf 100644
--- a/sld-api-backend/src/worker/tasks/terraform_worker.py
+++ b/sld-api-backend/src/worker/tasks/terraform_worker.py
@@ -1,6 +1,4 @@
 import logging
-import traceback
-
 import redis
 from celery import states
 from celery.exceptions import Ignore
diff --git a/sld-dashboard/app/home/routes.py b/sld-dashboard/app/home/routes.py
index cf3f5f97..5ee6a1b8 100644
--- a/sld-dashboard/app/home/routes.py
+++ b/sld-dashboard/app/home/routes.py
@@ -101,15 +101,14 @@ def output(task_id):
             uri=f"tasks/id/{task_id}",
             headers={"Authorization": f"Bearer {token}"}
         )
-    
         if response.get("status_code") == 200:
             data = response.get("json").get("result").get("module").get("stdout")
             if not isinstance(data, list):
                 data = [data] 
             return data
     except Exception as err:
-        logging.error(err) 
-        return render_template("page-500.html"), 500
+        print(task_id)
+        raise err
 
 
 # Start Deploy

From 20dce701f80c997a19c61535392e95fc02a846b7 Mon Sep 17 00:00:00 2001
From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com>
Date: Tue, 28 Nov 2023 21:53:54 +0100
Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=94=A5feat:=20stream=20output=20set?=
 =?UTF-8?q?=20beta=20info?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../app/base/static/assets/css/volt.css         | 17 +++++++++++++++++
 .../app/home/templates/deploy-stream.html       | 13 +++++++++----
 sld-dashboard/gunicorn-cfg.py                   |  2 +-
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/sld-dashboard/app/base/static/assets/css/volt.css b/sld-dashboard/app/base/static/assets/css/volt.css
index 23f5bc13..0df768fd 100644
--- a/sld-dashboard/app/base/static/assets/css/volt.css
+++ b/sld-dashboard/app/base/static/assets/css/volt.css
@@ -41820,3 +41820,20 @@ pre {
   padding: 10px;
 }
 
+.beta-version {
+    color: var(--bs-black); /* Color de texto */
+    text-align: center;
+    padding: 10px;
+    border-radius: 4px;
+    margin-top: 10px;
+}
+
+.beta-icon {
+    font-weight: bold;
+    font-size: 18px;
+    background-color: #d9534f; /* Color de fondo para el ícono */
+    padding: 5px 10px;
+    border-radius: 4px;
+    margin-right: 5px;
+}
+
diff --git a/sld-dashboard/app/home/templates/deploy-stream.html b/sld-dashboard/app/home/templates/deploy-stream.html
index 6022db7d..c1bd8201 100644
--- a/sld-dashboard/app/home/templates/deploy-stream.html
+++ b/sld-dashboard/app/home/templates/deploy-stream.html
@@ -35,7 +35,10 @@
 {% block content %}
 

     {% include 'includes/navigation.html' %}
-
+

+    BETA
+    
This is a beta version of the feature. Feedback is welcome!

+

     

         Volt logo
     

@@ -73,9 +76,11 @@ 
Deployment Output:

             sseDataElement.textContent += e.data + '\n';
             // Delay scroll adjustment to ensure the browser has processed the new content
             setTimeout(() => {
-                sseDataElement.scrollTop = sseDataElement.scrollHeight;
-            }, 10); // Adjust the timeout as needed
-        }
+                const lastLine = document.createElement("div");
+                sseDataElement.appendChild(lastLine);
+                lastLine.scrollIntoView({ behavior: "smooth", block: "end" });
+            }, 30); 
+        } 
     };
 
 {% endblock javascripts %}
diff --git a/sld-dashboard/gunicorn-cfg.py b/sld-dashboard/gunicorn-cfg.py
index 068932d2..54e9f4c8 100644
--- a/sld-dashboard/gunicorn-cfg.py
+++ b/sld-dashboard/gunicorn-cfg.py
@@ -4,7 +4,7 @@
 """
 
 bind = "0.0.0.0:5000"
-workers = 3
+workers = 10
 accesslog = "-"
 loglevel = "info"
 capture_output = True

From 16129f718951f80a520207c78520d27f17b5b94b Mon Sep 17 00:00:00 2001
From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com>
Date: Tue, 28 Nov 2023 22:00:54 +0100
Subject: [PATCH 9/9] =?UTF-8?q?=E2=AC=86Bump:=20rela=20time=20worker=20log?=
 =?UTF-8?q?s=20=202.27.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/workflows/sld-api-docker-image.yml          | 4 ++--
 .github/workflows/sld-dashboard-docker-image.yml    | 4 ++--
 play-with-sld/kubernetes/k8s/sld-api-backend.yml    | 2 +-
 play-with-sld/kubernetes/k8s/sld-dashboard.yml      | 2 +-
 play-with-sld/kubernetes/k8s/sld-worker-default.yml | 2 +-
 play-with-sld/kubernetes/k8s/sld-worker-squad1.yml  | 2 +-
 play-with-sld/kubernetes/k8s/sld-worker-squad2.yml  | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/sld-api-docker-image.yml b/.github/workflows/sld-api-docker-image.yml
index 65ab723a..65dcebfe 100644
--- a/.github/workflows/sld-api-docker-image.yml
+++ b/.github/workflows/sld-api-docker-image.yml
@@ -24,11 +24,11 @@ jobs:
 
     - name: Build the Docker image with tag
       working-directory: ./sld-api-backend
-      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:2.26.1
+      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:2.27.0
       
     - name: Docker Push with tag
       #if: github.event.pull_request.merged == true
-      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:2.26.1
+      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:2.27.0
 
     - name: Build the Docker image
       working-directory: ./sld-api-backend
diff --git a/.github/workflows/sld-dashboard-docker-image.yml b/.github/workflows/sld-dashboard-docker-image.yml
index 4a362fd1..5648eb5d 100644
--- a/.github/workflows/sld-dashboard-docker-image.yml
+++ b/.github/workflows/sld-dashboard-docker-image.yml
@@ -24,11 +24,11 @@ jobs:
 
     - name: Build the Docker image with tags
       working-directory: ./sld-dashboard
-      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.26.0
+      run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.27.0
       
     - name: Docker Push with tags
       #if: github.event.pull_request.merged == true
-      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.26.0
+      run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:2.27.0
 
     - name: Build the Docker image
       working-directory: ./sld-dashboard
diff --git a/play-with-sld/kubernetes/k8s/sld-api-backend.yml b/play-with-sld/kubernetes/k8s/sld-api-backend.yml
index 4814f427..f3fe1a2f 100644
--- a/play-with-sld/kubernetes/k8s/sld-api-backend.yml
+++ b/play-with-sld/kubernetes/k8s/sld-api-backend.yml
@@ -17,7 +17,7 @@ spec:
       subdomain: primary
       containers:
       - name: api-backend
-        image: d10s0vsky/sld-api:2.26.1
+        image: d10s0vsky/sld-api:2.27.0
         imagePullPolicy: Always
         command: ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"]
         ports:
diff --git a/play-with-sld/kubernetes/k8s/sld-dashboard.yml b/play-with-sld/kubernetes/k8s/sld-dashboard.yml
index ed1b243b..4a78a9fa 100644
--- a/play-with-sld/kubernetes/k8s/sld-dashboard.yml
+++ b/play-with-sld/kubernetes/k8s/sld-dashboard.yml
@@ -17,7 +17,7 @@ spec:
       subdomain: primary
       containers:
         - name: sld-dashboard
-          image: d10s0vsky/sld-dashboard:2.26.0
+          image: d10s0vsky/sld-dashboard:2.27.0
           env:
           - name: PATH
             value: "/home/sld/.asdf/shims:/home/sld/.asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
diff --git a/play-with-sld/kubernetes/k8s/sld-worker-default.yml b/play-with-sld/kubernetes/k8s/sld-worker-default.yml
index b7506f0b..6c14fab3 100644
--- a/play-with-sld/kubernetes/k8s/sld-worker-default.yml
+++ b/play-with-sld/kubernetes/k8s/sld-worker-default.yml
@@ -17,7 +17,7 @@ spec:
       subdomain: primary
       containers:
         - name: stack-deploy-worker-default
-          image: d10s0vsky/sld-api:2.26.1
+          image: d10s0vsky/sld-api:2.27.0
           imagePullPolicy: Always
           env:
           - name: TF_WARN_OUTPUT_ERRORS
diff --git a/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml b/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml
index e9bae7d8..9a5f6284 100644
--- a/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml
+++ b/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml
@@ -17,7 +17,7 @@ spec:
       subdomain: primary
       containers:
         - name: stack-deploy-worker-squad1
-          image: d10s0vsky/sld-api:2.26.1
+          image: d10s0vsky/sld-api:2.27.0
           imagePullPolicy: Always
           env:
           - name: TF_WARN_OUTPUT_ERRORS
diff --git a/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml b/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml
index 898e6287..7d33fbbe 100644
--- a/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml
+++ b/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml
@@ -17,7 +17,7 @@ spec:
       subdomain: primary
       containers:
         - name: stack-deploy-worker-squad2
-          image: d10s0vsky/sld-api:2.26.1
+          image: d10s0vsky/sld-api:2.27.0
           imagePullPolicy: Always
           env:
           - name: TF_WARN_OUTPUT_ERRORS