From 1d0fb9025b468a3918c13cdfe004b542f86a0cfc Mon Sep 17 00:00:00 2001 From: eric-intuitem <71850047+eric-intuitem@users.noreply.github.com> Date: Sat, 2 Mar 2024 03:19:57 +0100 Subject: [PATCH 1/3] Fix for CA-258 fill order_id at library loading Use order_id for sorting When order_id is missing, use created_at instead --- backend/core/helpers.py | 20 +++++++++++++++----- backend/library/utils.py | 7 ++++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/backend/core/helpers.py b/backend/core/helpers.py index 99e5a1d74..1307e49a8 100644 --- a/backend/core/helpers.py +++ b/backend/core/helpers.py @@ -196,6 +196,8 @@ def get_sorted_requirement_nodes( requirement_nodes: the list of all requirement_nodes requirements_assessed: the list of all requirements_assessed Returns a dictionary containing key=name and value={"description": description, "style": "leaf|node"}} + Values are correctly sorted based on order_id + If order_id is missing, sorting is based on created_at """ requirement_assessment_from_requirement_id = ( {str(ra.requirement.id): ra for ra in requirements_assessed} @@ -209,7 +211,7 @@ def get_sorted_requirement_nodes_rec( start: list, ) -> dict: """ - Recursive function to build framework groups tree, within get_sorted_requirements_and_groups + Recursive function to build framework groups tree, within get_sorted_requirements_nodes start: the initial list """ result = {} @@ -231,11 +233,12 @@ def get_sorted_requirement_nodes_rec( requirement_nodes, requirements_assessed, children ), } - for req in [ + for req in sorted([ requirement_node for requirement_node in requirement_nodes if requirement_node.parent_urn == node.urn - ]: + ], key=lambda x: x.order_id): + if requirements_assessed: req_as = requirement_assessment_from_requirement_id[str(req.id)] result[str(node.id)]["children"][str(req.id)].update( @@ -275,10 +278,17 @@ def get_sorted_requirement_nodes_rec( ) return result + # cope for old version not creating order_id correctly + requirement_nodes_with_order_id = list(requirement_nodes.all()) + for req in requirement_nodes_with_order_id: + if req.order_id is None: + req.order_id = req.created_at + tree = get_sorted_requirement_nodes_rec( - requirement_nodes, + requirement_nodes_with_order_id, requirements_assessed, - [rg for rg in requirement_nodes if not rg.parent_urn], + sorted([rg for rg in requirement_nodes_with_order_id if not rg.parent_urn], + key=lambda x: x.order_id), ) return tree diff --git a/backend/library/utils.py b/backend/library/utils.py index 0c7327a9e..e9b7204b7 100644 --- a/backend/library/utils.py +++ b/backend/library/utils.py @@ -103,8 +103,9 @@ def get_library_items(library, type: str) -> list[dict]: class RequirementNodeImporter: REQUIRED_FIELDS = {"urn"} - def __init__(self, requirement_data: dict): + def __init__(self, requirement_data: dict, index: int): self.requirement_data = requirement_data + self.index = index def is_valid(self) -> Union[str, None]: if missing_fields := self.REQUIRED_FIELDS - set(self.requirement_data.keys()): @@ -121,7 +122,7 @@ def import_requirement_node(self, framework_object: Framework): ref_id=self.requirement_data.get("ref_id"), annotation=self.requirement_data.get("annotation"), provider=framework_object.provider, - order_id=self.requirement_data.get("order_id"), + order_id=self.index, level=self.requirement_data.get("level"), name=self.requirement_data.get("name"), description=self.requirement_data.get("description"), @@ -154,7 +155,7 @@ def init_requirement_nodes(self, requirement_nodes: List[dict]) -> Union[str, No requirement_node_importers = [] import_errors = [] for index, requirement_node_data in enumerate(requirement_nodes): - requirement_node_importer = RequirementNodeImporter(requirement_node_data) + requirement_node_importer = RequirementNodeImporter(requirement_node_data, index) requirement_node_importers.append(requirement_node_importer) if ( requirement_node_error := requirement_node_importer.is_valid() From 7ce7004ce1d8dfc71973b6513584c2b9c80ba8c2 Mon Sep 17 00:00:00 2001 From: eric-intuitem <71850047+eric-intuitem@users.noreply.github.com> Date: Sun, 3 Mar 2024 00:20:28 +0100 Subject: [PATCH 2/3] optimizations remove unused function improve perf by loading all objects once add order_id for library preview optimize get_sorted_requirement_nodes --- backend/core/helpers.py | 22 ++++++++++++---------- backend/core/models.py | 11 ----------- backend/core/views.py | 6 +++--- backend/library/helpers.py | 3 +++ 4 files changed, 18 insertions(+), 24 deletions(-) diff --git a/backend/core/helpers.py b/backend/core/helpers.py index 1307e49a8..146a81d10 100644 --- a/backend/core/helpers.py +++ b/backend/core/helpers.py @@ -199,12 +199,21 @@ def get_sorted_requirement_nodes( Values are correctly sorted based on order_id If order_id is missing, sorting is based on created_at """ + + # cope for old version not creating order_id correctly + for req in requirement_nodes: + if req.order_id is None: + req.order_id = req.created_at + if req.order_id is None: + print("arghh", req) + requirement_assessment_from_requirement_id = ( - {str(ra.requirement.id): ra for ra in requirements_assessed} + {str(ra.requirement_id): ra for ra in requirements_assessed} if requirements_assessed else {} ) + def get_sorted_requirement_nodes_rec( requirement_nodes: list, requirements_assessed: list, @@ -278,19 +287,12 @@ def get_sorted_requirement_nodes_rec( ) return result - # cope for old version not creating order_id correctly - requirement_nodes_with_order_id = list(requirement_nodes.all()) - for req in requirement_nodes_with_order_id: - if req.order_id is None: - req.order_id = req.created_at - tree = get_sorted_requirement_nodes_rec( - requirement_nodes_with_order_id, + requirement_nodes, requirements_assessed, - sorted([rg for rg in requirement_nodes_with_order_id if not rg.parent_urn], + sorted([rn for rn in requirement_nodes if not rn.parent_urn], key=lambda x: x.order_id), ) - return tree diff --git a/backend/core/models.py b/backend/core/models.py index 12be40c1b..1ec44034e 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -284,17 +284,6 @@ class Meta: verbose_name = _("Framework") verbose_name_plural = _("Frameworks") - def get_next_order_id(self, obj_type: models.Model, _parent_urn: str = None) -> int: - """ - Returns the next order id for a given object type - """ - if _parent_urn: - return ( - obj_type.objects.filter(framework=self, parent_urn=_parent_urn).count() - + 1 - ) - else: - return obj_type.objects.filter(framework=self).count() + 1 def is_deletable(self) -> bool: """ diff --git a/backend/core/views.py b/backend/core/views.py index e15b5f2bd..466414f38 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -1015,7 +1015,7 @@ def tree(self, request, pk): _framework = Framework.objects.get(id=pk) return Response( get_sorted_requirement_nodes( - RequirementNode.objects.filter(framework=_framework), None + RequirementNode.objects.filter(framework=_framework).all(), None ) ) @@ -1181,10 +1181,10 @@ def tree(self, request, pk): _framework = self.get_object().framework return Response( get_sorted_requirement_nodes( - RequirementNode.objects.filter(framework=_framework), + RequirementNode.objects.filter(framework=_framework).all(), RequirementAssessment.objects.filter( compliance_assessment=self.get_object() - ), + ).all(), ) ) diff --git a/backend/library/helpers.py b/backend/library/helpers.py index 8e24403ff..48dabb196 100644 --- a/backend/library/helpers.py +++ b/backend/library/helpers.py @@ -10,7 +10,9 @@ def preview_library(library) -> dict[str, list]: preview = {} requirement_nodes_list = [] if library["objects"]["framework"].get("requirement_nodes"): + index=0 for requirement_node in library["objects"]["framework"]["requirement_nodes"]: + index+=1 requirement_nodes_list.append( RequirementNode( description=requirement_node.get("description"), @@ -18,6 +20,7 @@ def preview_library(library) -> dict[str, list]: name=requirement_node.get("name"), urn=requirement_node["urn"], parent_urn=requirement_node.get("parent_urn"), + order_id=index, ) ) preview["requirement_nodes"] = requirement_nodes_list From 6170398c1b08b0d6f160b962adc1b31053d6e703 Mon Sep 17 00:00:00 2001 From: Mohamed-Hacene Date: Mon, 4 Mar 2024 16:44:28 +0100 Subject: [PATCH 3/3] chore: remove debug print --- backend/core/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/core/helpers.py b/backend/core/helpers.py index 146a81d10..494a3e816 100644 --- a/backend/core/helpers.py +++ b/backend/core/helpers.py @@ -204,8 +204,6 @@ def get_sorted_requirement_nodes( for req in requirement_nodes: if req.order_id is None: req.order_id = req.created_at - if req.order_id is None: - print("arghh", req) requirement_assessment_from_requirement_id = ( {str(ra.requirement_id): ra for ra in requirements_assessed}