From 6524d38a73a72f0daf1dab88a29f6908d6626def Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Fri, 20 Sep 2024 12:25:08 +0530 Subject: [PATCH 1/3] - Fixed data source name missing on Dashboard => Safe Loader => Documents With Findings - Fixed size empty issue on Dashboard => Safe Loader => Data Source --- pebblo/app/models/db_models.py | 1 + pebblo/app/service/loader/loader_doc_service.py | 1 + pebblo/app/service/local_ui/loader_apps.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/pebblo/app/models/db_models.py b/pebblo/app/models/db_models.py index ed9dce0e..128dc967 100644 --- a/pebblo/app/models/db_models.py +++ b/pebblo/app/models/db_models.py @@ -149,6 +149,7 @@ class AiDataSource(BaseModel): loadId: str metadata: Metadata sourcePath: str + sourceSize: int sourceType: str loader: str diff --git a/pebblo/app/service/loader/loader_doc_service.py b/pebblo/app/service/loader/loader_doc_service.py index 5549b0a6..e66a4466 100644 --- a/pebblo/app/service/loader/loader_doc_service.py +++ b/pebblo/app/service/loader/loader_doc_service.py @@ -275,6 +275,7 @@ def _get_or_create_data_source(self): "modifiedAt": get_current_time(), }, "sourcePath": loader_details.get("source_path"), + "sourceSize": loader_details.get("source_aggregate_size", 0), "sourceType": loader_details.get("source_type"), "loader": loader_details.get("loader"), } diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index e58e9534..0c5e37fc 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -230,6 +230,7 @@ def _update_loader_datasource( "name": ds_data["loader"], "sourcePath": ds_data["sourcePath"], "sourceType": ds_data["sourceType"], + "sourceSize": ds_data.get("sourceSize", "-"), "findingsEntities": entity_count, "findingsTopics": topic_count, "totalSnippetCount": total_snippet_count, @@ -246,6 +247,19 @@ def _update_loader_datasource( self.loader_details.get("loader_data_source_list", []) ) + def _get_data_source_name(self, document_detail: dict): + """ + Get data source name for given data source id from db. + """ + data_source_id = document_detail.get("dataSourceId") + source_name = "-" + if data_source_id: + _, datasource = self.db.query(AiDataSourceTable, {"id": data_source_id}) + if datasource: + source_name = datasource[0].data.get("loader", "-") + + return source_name + def _get_documents_with_findings(self, app_data: AiDataLoaderTable) -> None: """ Fetch required data for DocumentWithFindings @@ -268,9 +282,12 @@ def _get_documents_with_findings(self, app_data: AiDataLoaderTable) -> None: for topic, topic_data in document_detail.get("topics", {}).items(): topic_count += topic_data.get("count", 0) + source_name = self._get_data_source_name(document_detail) + if document_detail["sourcePath"] in loader_document_with_findings: document_obj = { "appName": document_detail["appName"], + "sourceName": source_name, "owner": document_detail.get("owner", "-"), "sourceSize": document_detail.get("sourceSize", 0), "sourceFilePath": document_detail["sourcePath"], From ac1e3a22ce6f7b4d5e1a3de27143c3184b1c0a0d Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Fri, 20 Sep 2024 13:57:27 +0530 Subject: [PATCH 2/3] Ruff fix. --- pebblo/app/service/local_ui/loader_apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 0c5e37fc..574d46da 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -287,7 +287,7 @@ def _get_documents_with_findings(self, app_data: AiDataLoaderTable) -> None: if document_detail["sourcePath"] in loader_document_with_findings: document_obj = { "appName": document_detail["appName"], - "sourceName": source_name, + "sourceName": source_name, "owner": document_detail.get("owner", "-"), "sourceSize": document_detail.get("sourceSize", 0), "sourceFilePath": document_detail["sourcePath"], From 7431b279093e0ad3caf2b86ceca87bf6982ef22d Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Fri, 20 Sep 2024 16:34:02 +0530 Subject: [PATCH 3/3] Riff fix. --- pebblo/app/service/local_ui/loader_apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 574d46da..2c0dfa47 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -247,7 +247,7 @@ def _update_loader_datasource( self.loader_details.get("loader_data_source_list", []) ) - def _get_data_source_name(self, document_detail: dict): + def _get_data_source_name(self, document_detail: dict) -> str: """ Get data source name for given data source id from db. """