From f6b272f51a3d17fddd7e0e5a0802b8ef322deecf Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Wed, 6 Dec 2023 23:11:28 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=94=A5feat:=20Add=20filter=20and=20me?= =?UTF-8?q?tadata=20vars?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/deploy/api/container/deploy/get.py | 19 ++-- sld-api-backend/src/deploy/api/v1/deploy.py | 6 +- .../src/deploy/domain/entities/repository.py | 37 +++++++ .../src/deploy/infrastructure/repositories.py | 14 ++- .../src/variables/api/container/get.py | 102 ++++++++++++++++-- 5 files changed, 158 insertions(+), 20 deletions(-) create mode 100644 sld-api-backend/src/deploy/domain/entities/repository.py diff --git a/sld-api-backend/src/deploy/api/container/deploy/get.py b/sld-api-backend/src/deploy/api/container/deploy/get.py index 4deb4efc..d4bb085a 100644 --- a/sld-api-backend/src/deploy/api/container/deploy/get.py +++ b/sld-api-backend/src/deploy/api/container/deploy/get.py @@ -7,27 +7,26 @@ from src.shared.security import deps from src.users.domain.entities import users as schemas_users from src.users.infrastructure import repositories as crud_users +from typing import List +from src.deploy.domain.entities.repository import DeployFilter, DeployFilterResponse from config.api import settings - - - async def get_all_deploys( current_user: schemas_users.User = Depends(deps.get_current_active_user), skip: int = 0, limit: int = 100, db: Session = Depends(deps.get_db), -): + filters: DeployFilter = Depends(DeployFilter), +) -> List[DeployFilterResponse]: try: + # Si el usuario no es un maestro, aplicar el filtro de escuadrón if not crud_users.is_master(db, current_user): - squad = current_user.squad - return crud_deploys.get_all_deploys_by_squad( - db=db, squad=squad, skip=skip, limit=limit - ) - return crud_deploys.get_all_deploys(db=db, skip=skip, limit=limit) + filters.squad = current_user.squad + + return crud_deploys.get_deploys(db=db, filters=filters, skip=skip, limit=limit) except Exception as err: - raise HTTPException(status_code=400, detail=f"{err}") + raise HTTPException(status_code=400, detail=str(err)) async def get_deploy_by_id( diff --git a/sld-api-backend/src/deploy/api/v1/deploy.py b/sld-api-backend/src/deploy/api/v1/deploy.py index 767abd16..1bee8f18 100644 --- a/sld-api-backend/src/deploy/api/v1/deploy.py +++ b/sld-api-backend/src/deploy/api/v1/deploy.py @@ -1,7 +1,9 @@ from fastapi import APIRouter, Depends +from typing import List from src.deploy.api.container.deploy import create, delete, destroy, get, update from src.deploy.domain.entities import deploy as schemas_deploy +from src.deploy.domain.entities.repository import DeployFilterResponse, DeployFilter router = APIRouter() @@ -36,9 +38,9 @@ async def delete_infra_by_id( return delete_deploy -@router.get("/") +@router.get("/", status_code=200, response_model=List[DeployFilterResponse]) async def get_all_deploys( - get_all_deploys: schemas_deploy.DeployBase = Depends(get.get_all_deploys), + get_all_deploys: DeployFilterResponse = Depends(get.get_all_deploys), ): return get_all_deploys diff --git a/sld-api-backend/src/deploy/domain/entities/repository.py b/sld-api-backend/src/deploy/domain/entities/repository.py new file mode 100644 index 00000000..9b156e89 --- /dev/null +++ b/sld-api-backend/src/deploy/domain/entities/repository.py @@ -0,0 +1,37 @@ +from pydantic import BaseModel +from typing import Optional +from datetime import datetime + + +class DeployFilter(BaseModel): + task_id: Optional[str] = None + name: Optional[str] = None + action: Optional[str] = None + stack_name: Optional[str] = None + stack_branch: Optional[str] = None + user_id: Optional[int] = None + username: Optional[str] = None + squad: Optional[str] = None + environment: Optional[str] = None + tfvar_file: Optional[str] = None + project_path: Optional[str] = None + + +class DeployFilterResponse(BaseModel): + id: int + task_id: str + name: str + action: str + start_time: Optional[str] + destroy_time: Optional[str] + stack_name: str + stack_branch: str + created_at: datetime + updated_at: Optional[datetime] + user_id: int + username: str + squad: str + variables: dict + environment: str + tfvar_file: Optional[str] + project_path: Optional[str] diff --git a/sld-api-backend/src/deploy/infrastructure/repositories.py b/sld-api-backend/src/deploy/infrastructure/repositories.py index c72ac30c..0d92d845 100644 --- a/sld-api-backend/src/deploy/infrastructure/repositories.py +++ b/sld-api-backend/src/deploy/infrastructure/repositories.py @@ -1,9 +1,10 @@ import datetime - +from typing import List from sqlalchemy.orm import Session from sqlalchemy import desc import src.deploy.domain.entities.deploy as schemas_deploy +from src.deploy.domain.entities.repository import DeployFilter, DeployFilterResponse import src.deploy.infrastructure.models as models @@ -204,3 +205,14 @@ def get_deploy_by_cloud_account(db: Session, squad: str, environment: str): ) except Exception as err: raise err + + +def get_deploys(db: Session, filters: DeployFilter, skip: int = 0, limit: int = 100) -> List[DeployFilterResponse]: + query = db.query(models.Deploy) + + for field, value in filters.model_dump().items(): + if value is not None: + query = query.filter(getattr(models.Deploy, field) == value) + + deploys = query.order_by(desc(models.Deploy.id)).offset(skip).limit(limit).all() + return [DeployFilterResponse(**deploy.__dict__) for deploy in deploys] diff --git a/sld-api-backend/src/variables/api/container/get.py b/sld-api-backend/src/variables/api/container/get.py index 6f40f360..4089d98d 100644 --- a/sld-api-backend/src/variables/api/container/get.py +++ b/sld-api-backend/src/variables/api/container/get.py @@ -1,3 +1,6 @@ +import re +from collections import defaultdict + from fastapi import Depends, HTTPException from sqlalchemy.orm import Session @@ -9,6 +12,89 @@ from src.users.infrastructure import repositories as crud_users +class MetadataProcessor: + def __init__(self, data): + self.data = data + self.grouped_metadata = defaultdict( + lambda: {"vars_group": "", "descriptions": []} + ) + + def add_metadata(self): + for key, value in self.data.items(): + original_description = value.get("description") + metadata = self._parse_description(original_description) + if metadata is not None and "_no_metadata" in metadata: + continue + self._group_metadata(metadata) + + if metadata is None or "error" in metadata: + value["metadata"] = { + "error": "Description format is incorrect or missing required fields." + } + else: + value["description"] = original_description + if "description_cleaned" in metadata: + metadata["description"] = metadata.pop("description_cleaned") + value["metadata"] = metadata + + self._update_group_descriptions() + + return self.data + + def _parse_description(self, description): + key_values = dict(re.findall(r"(\w+): ([^\n]*)", description)) + if all(key in key_values for key in ["order", "vars_group"]): + return self._process_newline_format(key_values) + + if "|" not in description: + return {"_no_metadata": "Basic configuration"} + + if description.count("|") >= 2: + return self._process_pipe_format(description) + + return {"error": "Invalid format"} + + def _process_pipe_format(self, description): + parts = [part.strip() for part in description.split("|")] + if parts[0]: + group_key = ( + re.match(r"([A-Za-z])\d*", parts[0]).group(1) if parts[0] else "" + ) + return { + "order": parts[0], + "vars_group": parts[1] + if parts[1] + else self.grouped_metadata[group_key]["vars_group"], + "description_cleaned": parts[2] if len(parts) > 2 else "", + "parameter_type": "pipe", + } + return {"error": "Invalid pipe format"} + + def _process_newline_format(self, key_values): + metadata = {k: v.strip('"') for k, v in key_values.items()} + if "query_type" in metadata and "query" in metadata and "query_key" in metadata: + metadata["parameter_type"] = "EzeParams" + else: + metadata["parameter_type"] = "EOF" + metadata["description"] = metadata.pop("description", "").strip("\"'") + return metadata + + def _group_metadata(self, metadata): + if "order" in metadata and not metadata.get("error"): + group_key = re.match(r"([A-Za-z])\d*", metadata["order"]).group(1) + if not self.grouped_metadata[group_key]["vars_group"]: + self.grouped_metadata[group_key]["vars_group"] = metadata.get( + "vars_group", "" + ) + self.grouped_metadata[group_key]["descriptions"].append( + metadata.get("description", "") + ) + + def _update_group_descriptions(self): + for key, group in self.grouped_metadata.items(): + group["description"] = "\n".join(group["descriptions"]) + + async def get_json( stack, current_user: schemas_users.User = Depends(deps.get_current_active_user), @@ -20,9 +106,9 @@ async def get_json( try: if stack.isdigit(): result = crud_stacks.get_stack_by_id(db=db, stack_id=stack) - if result == None: + if result is None: raise HTTPException( - status_code=404, detail=f"Not found" + status_code=404, detail="Not found" ) if not crud_users.is_master(db, current_user): if "*" not in result.squad_access: @@ -30,20 +116,22 @@ async def get_json( raise HTTPException( status_code=403, detail=f"Not enough permissions" ) - return result.var_json.get("variable") + metadata_result = MetadataProcessor(result.var_json.get("variable")) + return metadata_result.add_metadata() else: result = crud_stacks.get_stack_by_name(db=db, stack_name=stack) - if result == None: + if result is None: raise HTTPException( - status_code=404, detail=f"Not found" + status_code=404, detail="Not found" ) if not crud_users.is_master(db, current_user): if "*" not in result.squad_access: if not check_squad_user(current_user.squad, result.squad_access): raise HTTPException( - status_code=403, detail=f"Not enough permissions" + status_code=403, detail="Not enough permissions" ) - return result.var_json.get("variable") + metadata_result = MetadataProcessor(result.var_json.get("variable")) + return metadata_result.add_metadata() except Exception as err: raise err From 6baab7f2fcc4afe514451be11646c78a50719c6c Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Thu, 7 Dec 2023 00:26:13 +0100 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=94=A5feat:=20Add=20basic=20metrics?= =?UTF-8?q?=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/deploy/api/container/deploy/get.py | 28 +++++++ sld-api-backend/src/deploy/api/v1/deploy.py | 7 ++ .../src/deploy/domain/entities/metrics.py | 54 +++++++++++++ .../src/deploy/domain/entities/repository.py | 2 + .../src/deploy/infrastructure/repositories.py | 78 ++++++++++++++++++- 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 sld-api-backend/src/deploy/domain/entities/metrics.py diff --git a/sld-api-backend/src/deploy/api/container/deploy/get.py b/sld-api-backend/src/deploy/api/container/deploy/get.py index d4bb085a..c6393921 100644 --- a/sld-api-backend/src/deploy/api/container/deploy/get.py +++ b/sld-api-backend/src/deploy/api/container/deploy/get.py @@ -9,6 +9,8 @@ from src.users.infrastructure import repositories as crud_users from typing import List from src.deploy.domain.entities.repository import DeployFilter, DeployFilterResponse +from src.deploy.domain.entities.metrics import AllMetricsResponse +from src.deploy.infrastructure.repositories import MetricsFetcher from config.api import settings @@ -141,3 +143,29 @@ async def lock_deploy( return response.json() except Exception as err: raise err + + +async def get_all_metrics( + db: Session = Depends(deps.get_db), + _current_user: schemas_users.User = Depends(deps.get_current_active_user), +) -> AllMetricsResponse: + metrics_fetcher = MetricsFetcher(db) + user_activity = metrics_fetcher.get_deploy_count_by_user() + action_count = metrics_fetcher.get_deploy_count_by_action() + environment_count = metrics_fetcher.get_deploy_count_by_environment() + stack_usage = metrics_fetcher.get_deploy_count_by_stack_name() + monthly_deploy_count = metrics_fetcher.get_monthly_deploy_count() + squad_deploy_count = metrics_fetcher.get_deploy_count_by_squad() + cloud_provider_usage = metrics_fetcher.get_cloud_provider_usage() + squad_environment_usage = metrics_fetcher.get_squad_environment_usage() + + return AllMetricsResponse( + user_activity=user_activity, + action_count=action_count, + environment_count=environment_count, + stack_usage=stack_usage, + monthly_deploy_count=monthly_deploy_count, + squad_deploy_count=squad_deploy_count, + cloud_provider_usage=cloud_provider_usage, + squad_environment_usage=squad_environment_usage, + ) diff --git a/sld-api-backend/src/deploy/api/v1/deploy.py b/sld-api-backend/src/deploy/api/v1/deploy.py index 1bee8f18..a9c8ac2b 100644 --- a/sld-api-backend/src/deploy/api/v1/deploy.py +++ b/sld-api-backend/src/deploy/api/v1/deploy.py @@ -77,3 +77,10 @@ async def get_show( get_show: schemas_deploy.DeployBase = Depends(get.get_show), ): return get_show + + +@router.get("/metrics/all") +async def get_metrics( + get_metrics: schemas_deploy.DeployBase = Depends(get.get_all_metrics), +): + return get_metrics diff --git a/sld-api-backend/src/deploy/domain/entities/metrics.py b/sld-api-backend/src/deploy/domain/entities/metrics.py new file mode 100644 index 00000000..38cbdf12 --- /dev/null +++ b/sld-api-backend/src/deploy/domain/entities/metrics.py @@ -0,0 +1,54 @@ +from pydantic import BaseModel +from typing import List + + +class UserActivity(BaseModel): + username: str + deploy_count: int + + +class ActionCount(BaseModel): + action: str + count: int + + +class EnvironmentCount(BaseModel): + environment: str + count: int + + +class StackUsage(BaseModel): + stack_name: str + count: int + + +class MonthlyDeployCount(BaseModel): + month: int + count: int + + +class SquadDeployCount(BaseModel): + squad: str + count: int + + +class CloudProviderUsage(BaseModel): + provider: str + count: int + + +class SquadEnvironmentUsage(BaseModel): + squad: str + environment: str + count: int + + +class AllMetricsResponse(BaseModel): + user_activity: List[UserActivity] + action_count: List[ActionCount] + environment_count: List[EnvironmentCount] + stack_usage: List[StackUsage] + monthly_deploy_count: List[MonthlyDeployCount] + squad_deploy_count: List[SquadDeployCount] + cloud_provider_usage: List[CloudProviderUsage] + squad_environment_usage: List[SquadEnvironmentUsage] diff --git a/sld-api-backend/src/deploy/domain/entities/repository.py b/sld-api-backend/src/deploy/domain/entities/repository.py index 9b156e89..175d8d59 100644 --- a/sld-api-backend/src/deploy/domain/entities/repository.py +++ b/sld-api-backend/src/deploy/domain/entities/repository.py @@ -35,3 +35,5 @@ class DeployFilterResponse(BaseModel): environment: str tfvar_file: Optional[str] project_path: Optional[str] + + diff --git a/sld-api-backend/src/deploy/infrastructure/repositories.py b/sld-api-backend/src/deploy/infrastructure/repositories.py index 0d92d845..d0f5a27a 100644 --- a/sld-api-backend/src/deploy/infrastructure/repositories.py +++ b/sld-api-backend/src/deploy/infrastructure/repositories.py @@ -1,10 +1,11 @@ import datetime from typing import List from sqlalchemy.orm import Session -from sqlalchemy import desc +from sqlalchemy import func, extract, desc, case import src.deploy.domain.entities.deploy as schemas_deploy from src.deploy.domain.entities.repository import DeployFilter, DeployFilterResponse +from src.deploy.domain.entities.metrics import ActionCount, EnvironmentCount, MonthlyDeployCount, SquadDeployCount, StackUsage, UserActivity, CloudProviderUsage, SquadEnvironmentUsage import src.deploy.infrastructure.models as models @@ -216,3 +217,78 @@ def get_deploys(db: Session, filters: DeployFilter, skip: int = 0, limit: int = deploys = query.order_by(desc(models.Deploy.id)).offset(skip).limit(limit).all() return [DeployFilterResponse(**deploy.__dict__) for deploy in deploys] + + +class MetricsFetcher: + def __init__(self, db: Session): + self.db = db + + def get_deploy_count_by_user(self) -> List[UserActivity]: + query_result = self.db.query( + models.Deploy.username, + func.count(models.Deploy.id) + ).group_by(models.Deploy.username).all() + return [UserActivity(username=username, deploy_count=count) for username, count in query_result] + + def get_deploy_count_by_action(self) -> List[ActionCount]: + query_result = self.db.query( + models.Deploy.action, + func.count(models.Deploy.id) + ).group_by(models.Deploy.action).all() + return [ActionCount(action=action, count=count) for action, count in query_result] + + def get_deploy_count_by_environment(self) -> List[EnvironmentCount]: + query_result = self.db.query( + models.Deploy.environment, + func.count(models.Deploy.id) + ).group_by(models.Deploy.environment).all() + return [EnvironmentCount(environment=environment, count=count) for environment, count in query_result] + + def get_deploy_count_by_stack_name(self) -> List[StackUsage]: + query_result = self.db.query( + models.Deploy.stack_name, + func.count(models.Deploy.id) + ).group_by(models.Deploy.stack_name).all() + return [StackUsage(stack_name=stack_name, count=count) for stack_name, count in query_result] + + def get_monthly_deploy_count(self) -> List[MonthlyDeployCount]: + query_result = self.db.query( + extract('month', models.Deploy.created_at), + func.count(models.Deploy.id) + ).group_by(extract('month', models.Deploy.created_at)).all() + return [MonthlyDeployCount(month=month, count=count) for month, count in query_result] + + def get_deploy_count_by_squad(self) -> List[SquadDeployCount]: + query_result = self.db.query( + models.Deploy.squad, + func.count(models.Deploy.id) + ).group_by(models.Deploy.squad).all() + return [SquadDeployCount(squad=squad, count=count) for squad, count in query_result] + + def get_cloud_provider_usage(self) -> List[CloudProviderUsage]: + stacks = self.db.query(models.Deploy.stack_name).all() + counts = {'aws': 0, 'azure': 0, 'gcp': 0, 'custom': 0} + + for (stack_name,) in stacks: + for provider in counts.keys(): + if stack_name.startswith(provider + '_'): + counts[provider] += 1 + break + + return [CloudProviderUsage(provider=provider, count=count) + for provider, count in counts.items() if count > 0] + + def get_squad_environment_usage(self) -> List[SquadEnvironmentUsage]: + results = self.db.query( + models.Deploy.squad, + models.Deploy.environment, + func.count(models.Deploy.id) + ).group_by( + models.Deploy.squad, + models.Deploy.environment + ).all() + + return [ + SquadEnvironmentUsage(squad=squad, environment=environment, count=count) + for squad, environment, count in results + ] \ No newline at end of file From c252b72095a5da007ba716ac3a43b68d3294313e Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Thu, 7 Dec 2023 02:39:19 +0100 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=94=A5feat:=20Add=20Metrics=20charts?= =?UTF-8?q?=20dashboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/deploy/domain/entities/deploy.py | 4 +- .../src/deploy/domain/entities/repository.py | 1 + .../src/deploy/infrastructure/repositories.py | 6 +- .../src/stacks/infrastructure/repositories.py | 21 ++- sld-dashboard/app/home/routes.py | 23 +-- .../app/home/templates/dashboard.html | 152 ++++++++++++++++-- .../app/home/templates/stacks-list.html | 4 +- 7 files changed, 173 insertions(+), 38 deletions(-) diff --git a/sld-api-backend/src/deploy/domain/entities/deploy.py b/sld-api-backend/src/deploy/domain/entities/deploy.py index fb3c8e47..3b1e3ff1 100644 --- a/sld-api-backend/src/deploy/domain/entities/deploy.py +++ b/sld-api-backend/src/deploy/domain/entities/deploy.py @@ -41,8 +41,8 @@ class DeployDeleteMaster(BaseModel): class DeployUpdate(BaseModel): - start_time: constr(strip_whitespace=True) - destroy_time: constr(strip_whitespace=True) + start_time: Optional[constr(strip_whitespace=True)] = Field("") + destroy_time: Optional[constr(strip_whitespace=True)] = Field("") stack_branch: Optional[constr(strip_whitespace=True)] = Field("", example="") tfvar_file: Optional[constr(strip_whitespace=True)] = Field( "", example="terraform.tfvars" diff --git a/sld-api-backend/src/deploy/domain/entities/repository.py b/sld-api-backend/src/deploy/domain/entities/repository.py index 175d8d59..246e8c2a 100644 --- a/sld-api-backend/src/deploy/domain/entities/repository.py +++ b/sld-api-backend/src/deploy/domain/entities/repository.py @@ -4,6 +4,7 @@ class DeployFilter(BaseModel): + id: Optional[str] = None task_id: Optional[str] = None name: Optional[str] = None action: Optional[str] = None diff --git a/sld-api-backend/src/deploy/infrastructure/repositories.py b/sld-api-backend/src/deploy/infrastructure/repositories.py index d0f5a27a..fab4e455 100644 --- a/sld-api-backend/src/deploy/infrastructure/repositories.py +++ b/sld-api-backend/src/deploy/infrastructure/repositories.py @@ -207,13 +207,15 @@ def get_deploy_by_cloud_account(db: Session, squad: str, environment: str): except Exception as err: raise err - def get_deploys(db: Session, filters: DeployFilter, skip: int = 0, limit: int = 100) -> List[DeployFilterResponse]: query = db.query(models.Deploy) for field, value in filters.model_dump().items(): if value is not None: - query = query.filter(getattr(models.Deploy, field) == value) + if field == 'squad' and isinstance(value, list): + query = query.filter(getattr(models.Deploy, field).in_(value)) + else: + query = query.filter(getattr(models.Deploy, field) == value) deploys = query.order_by(desc(models.Deploy.id)).offset(skip).limit(limit).all() return [DeployFilterResponse(**deploy.__dict__) for deploy in deploys] diff --git a/sld-api-backend/src/stacks/infrastructure/repositories.py b/sld-api-backend/src/stacks/infrastructure/repositories.py index 5e6ae707..4a883130 100644 --- a/sld-api-backend/src/stacks/infrastructure/repositories.py +++ b/sld-api-backend/src/stacks/infrastructure/repositories.py @@ -92,19 +92,20 @@ def update_stack( raise err -def get_all_stacks_by_squad( - db: Session, squad_access: str, skip: int = 0, limit: int = 100 -): +def get_all_stacks_by_squad(db: Session, squad_access: str, skip: int = 0, limit: int = 100): try: + from sqlalchemy import func + + order_by_clause = models.Stack.id.desc() + filter_all = ( db.query(models.Stack) .filter(models.Stack.squad_access.contains("*")) - .order_by(models.Stack.created_at.desc()) + .order_by(order_by_clause) .offset(skip) .limit(limit) .all() ) - from sqlalchemy import func result = [] for i in squad_access: @@ -112,15 +113,19 @@ def get_all_stacks_by_squad( result.extend( db.query(models.Stack) .filter(func.json_contains(models.Stack.squad_access, a) == 1) - .order_by(models.Stack.created_at.desc()) + .order_by(order_by_clause) .all() ) - merge_query = result + filter_all - return set(merge_query) + + merge_query = list({v.id: v for v in (result + filter_all)}.values()) + merge_query.sort(key=lambda x: x.id, reverse=True) + + return merge_query except Exception as err: raise err + def get_all_stacks(db: Session, squad_access: str, skip: int = 0, limit: int = 100): try: return db.query(models.Stack).order_by(models.Stack.created_at.desc()).offset(skip).limit(limit).all() diff --git a/sld-dashboard/app/home/routes.py b/sld-dashboard/app/home/routes.py index e99dd2d0..8b5c303b 100644 --- a/sld-dashboard/app/home/routes.py +++ b/sld-dashboard/app/home/routes.py @@ -52,11 +52,8 @@ def index(): @login_required def deploy_stream(deploy_id): token = decrypt(r.get(current_user.id)) - # Check if token no expired check_unauthorized_token(token) - # Get defaults vars by deploy endpoint = f"deploy/{deploy_id}" - # Get deploy data vars and set var for render response = request_url( verb="GET", uri=f"{endpoint}", headers={"Authorization": f"Bearer {token}"} ) @@ -135,7 +132,7 @@ def list_deploys(limit): # get stack info endpoint = f"stacks/?limit={limit}" if limit == 0: - endpoint = f"stacks/" + endpoint = "stacks/" stack_response = request_url( verb="GET", uri=f"{endpoint}", headers={"Authorization": f"Bearer {token}"} ) @@ -1674,26 +1671,33 @@ def route_template(template): # Get Api data deploy_response = request_url( verb="GET", - uri=f"deploy", + uri="deploy", headers={"Authorization": f"Bearer {token}"}, ) stack_response = request_url( verb="GET", - uri=f"stacks/", + uri="stacks/", headers={"Authorization": f"Bearer {token}"}, ) tasks_response = request_url( verb="GET", - uri=f"tasks/all", + uri="tasks/all", + headers={"Authorization": f"Bearer {token}"}, + ) + + metrics = request_url( + verb="GET", + uri="deploy/metrics/all", headers={"Authorization": f"Bearer {token}"}, ) + try: api_healthy = request_url(verb="GET", uri=f"") schedule_healthy = request_url( - verb="GET", uri=f"", server=settings.SCHEDULE + verb="GET", uri="", server=settings.SCHEDULE ) remote_state_healthy = request_url( - verb="GET", uri=f"", server=settings.REMOTE_STATE + verb="GET", uri="", server=settings.REMOTE_STATE ) except Exception as err: return err @@ -1713,6 +1717,7 @@ def route_template(template): api_healthy=api_healthy["json"], schedule_healthy=schedule_healthy["json"], remote_state_healthy=remote_state_healthy["json"], + metrics=metrics["json"], ) except TemplateNotFound: return render_template("page-404.html"), 404 diff --git a/sld-dashboard/app/home/templates/dashboard.html b/sld-dashboard/app/home/templates/dashboard.html index b75b7c43..fbd420b0 100644 --- a/sld-dashboard/app/home/templates/dashboard.html +++ b/sld-dashboard/app/home/templates/dashboard.html @@ -32,24 +32,11 @@
+

Services Status

-
-
-
-
-
-
-
-
-
-
- -
-
+
@@ -216,7 +203,49 @@

down

+ +

Metrics

+ +
+ +
+
+
+ +
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+ +
+
+
+
+ + {% include 'includes/footer.html' %} @@ -227,4 +256,97 @@

down

{% block javascripts %} + + + {% endblock javascripts %} diff --git a/sld-dashboard/app/home/templates/stacks-list.html b/sld-dashboard/app/home/templates/stacks-list.html index d736438a..711c2007 100644 --- a/sld-dashboard/app/home/templates/stacks-list.html +++ b/sld-dashboard/app/home/templates/stacks-list.html @@ -49,10 +49,10 @@

All Stacks

{% if "yoda" in current_user.role or "darth_vader" in current_user.role %} Stack Repo Branch + {% endif %} IaC Type IaC version squad_access - {% endif %} Description @@ -76,6 +76,7 @@

All Stacks

{% if "yoda" in current_user.role or "darth_vader" in current_user.role %} {{ stack.git_repo }} {{ stack.branch }} + {% endif %} {% if stack.iac_type == 'terraform' %} Terraform Icon @@ -85,7 +86,6 @@

All Stacks

 {{ stack.tf_version }} {{ stack.squad_access }} - {% endif %} {{ stack.description}} {% if "yoda" in current_user.role or "darth_vader" in current_user.role %} From 22bf6b25af4a1bafb84aa0ae1dcef52071a173fe Mon Sep 17 00:00:00 2001 From: D10S0VSkY-OSS <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Thu, 7 Dec 2023 17:11:42 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=94=A5feat:=20Add=20metrics=20and=20f?= =?UTF-8?q?ix=20vars=20render?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/variables/api/container/get.py | 11 +- .../app/home/templates/dashboard.html | 140 ++++++++++++++---- 2 files changed, 123 insertions(+), 28 deletions(-) diff --git a/sld-api-backend/src/variables/api/container/get.py b/sld-api-backend/src/variables/api/container/get.py index 4089d98d..8215e568 100644 --- a/sld-api-backend/src/variables/api/container/get.py +++ b/sld-api-backend/src/variables/api/container/get.py @@ -42,18 +42,23 @@ def add_metadata(self): return self.data def _parse_description(self, description): + # Asegurarse de que description sea una cadena de texto + description = str(description) + key_values = dict(re.findall(r"(\w+): ([^\n]*)", description)) + if all(key in key_values for key in ["order", "vars_group"]): return self._process_newline_format(key_values) - + if "|" not in description: return {"_no_metadata": "Basic configuration"} - + if description.count("|") >= 2: return self._process_pipe_format(description) - + return {"error": "Invalid format"} + def _process_pipe_format(self, description): parts = [part.strip() for part in description.split("|")] if parts[0]: diff --git a/sld-dashboard/app/home/templates/dashboard.html b/sld-dashboard/app/home/templates/dashboard.html index fbd420b0..88725fcc 100644 --- a/sld-dashboard/app/home/templates/dashboard.html +++ b/sld-dashboard/app/home/templates/dashboard.html @@ -207,43 +207,75 @@

down

Metrics

-
- -
-
-
- -
+ +
+ +
+
+
+
+
- -
-
-
- -
+ +
+
+
+
+
- -
-
-
- -
+ +
+
+
+
+
- -
-
-
- -
+ +
+
+
+
+
+ + +
+
+
+ +
+
+
+ + + +
+
+
+ +
+
+
+ + +
+
+
+ +
+
+
+ + +
@@ -346,6 +378,64 @@

down

} }); +// Gráfico "squad_deploy_count" (Bar Chart) +var ctx = document.getElementById('squadDeployCountChart').getContext('2d'); +var squadDeployCountChart = new Chart(ctx, { + type: 'bar', + data: { + labels: metrics.squad_deploy_count.map(function(a) { return a.squad; }), + datasets: [{ + label: 'Deployments by squad', + data: metrics.squad_deploy_count.map(function(a) { return a.count; }), + backgroundColor: metrics.squad_deploy_count.map(function() { return randomColor(); }) + }] + }, + options: { + scales: { + yAxes: [{ + ticks: { + beginAtZero: true + } + }] + } + } +}); + +// Gráfico "cloud_provider_usage" (Pie Chart) +var ctx = document.getElementById('cloudProviderUsageChart').getContext('2d'); +var cloudProviderUsageChart = new Chart(ctx, { + type: 'pie', + data: { + labels: metrics.cloud_provider_usage.map(function(a) { return a.provider; }), + datasets: [{ + data: metrics.cloud_provider_usage.map(function(a) { return a.count; }), + backgroundColor: metrics.cloud_provider_usage.map(function() { return randomColor(); }) + }] + } +}); + +// Gráfico "squad_environment_usage" (Bar Chart) +var ctx = document.getElementById('squadEnvironmentUsageChart').getContext('2d'); +var squadEnvironmentUsageChart = new Chart(ctx, { + type: 'bar', + data: { + labels: metrics.squad_environment_usage.map(function(a) { return a.squad + ' - ' + a.environment; }), + datasets: [{ + label: 'Deployments by squad and environment', + data: metrics.squad_environment_usage.map(function(a) { return a.count; }), + backgroundColor: metrics.squad_environment_usage.map(function() { return randomColor(); }) + }] + }, + options: { + scales: { + yAxes: [{ + ticks: { + beginAtZero: true + } + }] + } + } +}); From 5527db0e1f6d98da4c73192a33bf85a7bdcae6e2 Mon Sep 17 00:00:00 2001 From: d10s <79284025+D10S0VSkY-OSS@users.noreply.github.com> Date: Thu, 7 Dec 2023 23:12:43 +0100 Subject: [PATCH 5/5] =?UTF-8?q?=E2=AC=86Bump:=20add=20charts=20dashboard?= =?UTF-8?q?=20and=20metrics=20path=20operator=20v3.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/sld-api-docker-image.yml | 4 ++-- .github/workflows/sld-dashboard-docker-image.yml | 4 ++-- play-with-sld/kubernetes/k8s/sld-api-backend.yml | 2 +- play-with-sld/kubernetes/k8s/sld-dashboard.yml | 2 +- play-with-sld/kubernetes/k8s/sld-worker-default.yml | 2 +- play-with-sld/kubernetes/k8s/sld-worker-squad1.yml | 2 +- play-with-sld/kubernetes/k8s/sld-worker-squad2.yml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sld-api-docker-image.yml b/.github/workflows/sld-api-docker-image.yml index efe031cd..308dcd71 100644 --- a/.github/workflows/sld-api-docker-image.yml +++ b/.github/workflows/sld-api-docker-image.yml @@ -24,11 +24,11 @@ jobs: - name: Build the Docker image with tag working-directory: ./sld-api-backend - run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:3.0.0 + run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-api:3.1.0 - name: Docker Push with tag #if: github.event.pull_request.merged == true - run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:3.0.0 + run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-api:3.1.0 - name: Build the Docker image working-directory: ./sld-api-backend diff --git a/.github/workflows/sld-dashboard-docker-image.yml b/.github/workflows/sld-dashboard-docker-image.yml index d0a19e52..9305c1d7 100644 --- a/.github/workflows/sld-dashboard-docker-image.yml +++ b/.github/workflows/sld-dashboard-docker-image.yml @@ -24,11 +24,11 @@ jobs: - name: Build the Docker image with tags working-directory: ./sld-dashboard - run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:3.0.0 + run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:3.1.0 - name: Docker Push with tags #if: github.event.pull_request.merged == true - run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:3.0.0 + run: docker push ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:3.1.0 - name: Build the Docker image working-directory: ./sld-dashboard run: docker build . --file Dockerfile --tag ${{ secrets.DOCKER_USERNAME }}/sld-dashboard:latest diff --git a/play-with-sld/kubernetes/k8s/sld-api-backend.yml b/play-with-sld/kubernetes/k8s/sld-api-backend.yml index e8532320..ea7bd2df 100644 --- a/play-with-sld/kubernetes/k8s/sld-api-backend.yml +++ b/play-with-sld/kubernetes/k8s/sld-api-backend.yml @@ -17,7 +17,7 @@ spec: subdomain: primary containers: - name: api-backend - image: d10s0vsky/sld-api:3.0.0 + image: d10s0vsky/sld-api:3.1.0 imagePullPolicy: Always command: ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"] ports: diff --git a/play-with-sld/kubernetes/k8s/sld-dashboard.yml b/play-with-sld/kubernetes/k8s/sld-dashboard.yml index bf8bf05d..3a135baf 100644 --- a/play-with-sld/kubernetes/k8s/sld-dashboard.yml +++ b/play-with-sld/kubernetes/k8s/sld-dashboard.yml @@ -17,7 +17,7 @@ spec: subdomain: primary containers: - name: sld-dashboard - image: d10s0vsky/sld-dashboard:3.0.0 + image: d10s0vsky/sld-dashboard:3.1.0 env: - name: PATH value: "/home/sld/.asdf/shims:/home/sld/.asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/play-with-sld/kubernetes/k8s/sld-worker-default.yml b/play-with-sld/kubernetes/k8s/sld-worker-default.yml index 84a31b25..c90d5c05 100644 --- a/play-with-sld/kubernetes/k8s/sld-worker-default.yml +++ b/play-with-sld/kubernetes/k8s/sld-worker-default.yml @@ -17,7 +17,7 @@ spec: subdomain: primary containers: - name: stack-deploy-worker-default - image: d10s0vsky/sld-api:3.0.0 + image: d10s0vsky/sld-api:3.1.0 imagePullPolicy: Always env: - name: TF_WARN_OUTPUT_ERRORS diff --git a/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml b/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml index 7aee6331..47780690 100644 --- a/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml +++ b/play-with-sld/kubernetes/k8s/sld-worker-squad1.yml @@ -17,7 +17,7 @@ spec: subdomain: primary containers: - name: stack-deploy-worker-squad1 - image: d10s0vsky/sld-api:3.0.0 + image: d10s0vsky/sld-api:3.1.0 imagePullPolicy: Always env: - name: TF_WARN_OUTPUT_ERRORS diff --git a/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml b/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml index 878dbcad..1314e728 100644 --- a/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml +++ b/play-with-sld/kubernetes/k8s/sld-worker-squad2.yml @@ -17,7 +17,7 @@ spec: subdomain: primary containers: - name: stack-deploy-worker-squad2 - image: d10s0vsky/sld-api:3.0.0 + image: d10s0vsky/sld-api:3.1.0 imagePullPolicy: Always env: - name: TF_WARN_OUTPUT_ERRORS