Bulk update a component's statements across all systems

CHANGELOG.md

@@ -1,6 +1,17 @@
 GovReady-Q Release Notes
 ========================
 
+v0.12.0-dev (February 4, 2022)
+---------------------------
+
+**Developer changes**
+
+* Add API endpoint and Element (component) model method to force update all Element consuming systems's control implementation statements with library Elements content.
+* Add parameter createOSCAL API endpoint to indicate update existing components.
+* Upgrade Python libraries.
+* Update NPM libraries.
+
+
 v0.11.4 (December 17, 2022)
 ---------------------------
 
@@ -14,8 +25,7 @@ v0.11.3 (December 10, 2022)
 
 **Developer changes**
 
-* Add processing for question actions targeted at system to handle `system/add_baseline/<value>` to add additional baseline set of controls to a system without deleting already assigned controls.A
-
+* Add processing for question actions targeted at system to handle `system/add_baseline/<value>` to add additional baseline set of controls to a system without deleting already assigned controls.
 
 
 v0.11.2 (December 10, 2022)

VERSION

@@ -1 +1 @@
-v0.11.4
+v0.12.0-dev

api/controls/serializers/element.py

@@ -36,9 +36,18 @@ class Meta:
 
 class WriteElementOscalSerializer(WriteOnlySerializer):
     oscal = serializers.JSONField()
+    update = serializers.NullBooleanField()
     class Meta:
         model = Element
-        fields = ['oscal']
+        fields = ['oscal', 'update']
+
+class WriteSynchConsumingSystemsImplementationStatementsSerializer(WriteOnlySerializer):
+    # oscal = serializers.JSONField()
+    componentId = serializers.IntegerField(min_value=1, max_value=None)
+    class Meta:
+        model = Element
+        fields = ['componentId']
+
 class ReadElementOscalSerializer(ReadOnlySerializer):
     oscal = serializers.SerializerMethodField('get_oscal')
 
@@ -276,4 +285,5 @@ class ElementCreateAndSetRequestSerializer(WriteOnlySerializer):
     status = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True)
     class Meta:
         model = Element
-        fields = ['proposalId', 'userId', 'systemId', 'criteria_comment', 'criteria_reject_comment', 'status']
+        fields = ['proposalId', 'userId', 'systemId', 'criteria_comment', 'criteria_reject_comment', 'status']
+

api/controls/views/element.py

@@ -19,7 +19,7 @@
     WriteElementTagsSerializer, ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer, \
     WriteElementAppointPartySerializer, ElementPartySerializer, DeletePartyAppointmentsFromElementSerializer, CreateMultipleAppointmentsFromRoleIds, \
     ElementRequestsSerializer, ElementSetRequestsSerializer, ElementCreateAndSetRequestSerializer, \
-    WriteElementOscalSerializer, ReadElementOscalSerializer, SimpleGetElementByNameSerializer
+    WriteElementOscalSerializer, ReadElementOscalSerializer, SimpleGetElementByNameSerializer, WriteSynchConsumingSystemsImplementationStatementsSerializer
 from controls.models import Element, System
 from siteapp.models import Appointment, Party, Proposal, Role, Request, User
 from controls.views import ComponentImporter, OSCALComponentSerializer
@@ -52,7 +52,9 @@ class ElementViewSet(ReadWriteViewSet):
                                            CreateAndSetRequest=ElementCreateAndSetRequestSerializer,
                                            createOSCAL=WriteElementOscalSerializer,
                                            getOSCAL=ReadElementOscalSerializer,
-                                           downloadOSCAL=ReadElementOscalSerializer)
+                                           downloadOSCAL=ReadElementOscalSerializer,
+                                           synchConsumingSystemsImplementationStatements=WriteSynchConsumingSystemsImplementationStatementsSerializer                                       
+                                           )
 
     @action(detail=False, url_path="createOSCAL", methods=["POST"])
     def createOSCAL(self, request, **kwargs):
@@ -61,11 +63,17 @@ def createOSCAL(self, request, **kwargs):
         if "metadata" in request.data["oscal"]["component-definition"]:
             title = request.data["oscal"]["component-definition"]["metadata"]["title"]
         date_string = datetime.now().strftime("%Y-%m-%d-%H-%M")
+
+        # check if update value set to True
+        if "update" in request.data and request.data["update"]:
+            update = True
+        else:
+            update = False
 
         import_record_name = title + "_api-import_" + date_string
         oscal_component_json = json.dumps(request.data["oscal"])
 
-        import_record_result = ComponentImporter().import_components_as_json(import_record_name, oscal_component_json, request)
+        import_record_result = ComponentImporter().import_components_as_json(import_record_name, oscal_component_json, request, update=update)
         element = Element.objects.filter(import_record=import_record_result).first()
 
         serializer_class = self.get_serializer_class('retrieve')
@@ -84,6 +92,24 @@ def createOSCAL(self, request, **kwargs):
     #     serializer = self.get_serializer(serializer_class, element)
     #     return Response(serializer.data)
 
+    @action(detail=False, url_path="synchConsumingSystemsImplementationStatements", methods=["POST"])
+    def synchConsumingSystemsImplementationStatements(self, request, **kwargs):
+        """
+        Force update all element consuming system control impl smts with content of protoype component control impl smt
+        """
+        if "componentId" in request.data:
+            component_id = request.data['componentId']
+            element = Element.objects.filter(id=component_id).first()
+            if element is not None:
+                # TODO: check user permisson
+                system_smts_updated = element.synch_consuming_systems_implementation_statements()
+                result = {"system_smts_updated": system_smts_updated}
+                return Response(result)
+            else:
+                # element not found
+                result = {"system_smts_updated": 0}
+                return Response(result2)
+
     @action(detail=True, url_path="getOSCAL", methods=["GET"])
     def getOSCAL(self, request, **kwargs):
         element, validated_data = self.validate_serializer_and_get_object(request)

controls/models.py

@@ -11,6 +11,7 @@
                                 get_perms_for_model, get_user_perms,
                                 get_users_with_perms, remove_perm)
 from simple_history.models import HistoricalRecords
+from simple_history.utils import bulk_update_with_history
 from jsonfield import JSONField
 from natsort import natsorted
 
@@ -394,6 +395,7 @@ def assign_user_permissions(self, user, permissions):
                 user={"id": user.id, "username": user.username}
             )
             return False
+
     def remove_all_permissions_from_user(self, user):
         try:
             current_permissions = get_user_perms(user, self)
@@ -417,6 +419,7 @@ def remove_all_permissions_from_user(self, user):
                 user={"id": user.id, "username": user.username}
             )
             return False
+
     def get_permissible_users(self):
         return get_users_with_perms(self, attach_perms=True)
 
@@ -597,6 +600,70 @@ def copy(self, name=None):
             smt_copy.save()
         return e_copy
 
+    @transaction.atomic
+    def synch_consuming_systems_implementation_statements(self):
+        """
+        Force update all Element's consuming systems' control implementation statements to be the same
+        as the Element's control implementation prototype statements
+        """
+
+        # get Element's consuming_systems
+        consuming_systems = self.consuming_systems()
+        # get Element's control_implementation_prototype statements
+        element_prototype_smts = self.statements(StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name)
+        # track system control implementation statements touched via synchronization (whether changed or not)
+        total_system_smts_updated = 0
+        consuming_systems_updated = []
+        # loop through Element's control_implementation_prototype statements
+        for prototype_smt in element_prototype_smts:
+            # find the consuming systems' control implementation statements to be updated with current control_implementation_prototype
+            system_smts_to_update = Statement.objects.filter(statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION.name, prototype_id=prototype_smt.id)
+            # track updated smts for bulk update
+            system_smts_updated = []
+            # determine list of all consuming systems to be updated
+            consuming_systems_to_update = Statement.objects.filter(statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION.name, prototype_id=prototype_smt.id).values('consumer_element')
+
+            # consuming systems that have a statement that has been removed from the producing Element
+
+            # update the related control_implementation statements
+            for smt in system_smts_to_update:
+
+                # update the system if not already synced with prototype
+                # TODO: improve Statement.protype_synched() to check pid, status, etc
+                if smt.prototype_synched == STATEMENT_NOT_SYNCHED:
+                    smt.body = prototype_smt.body
+                    smt.pid = prototype_smt.pid
+                    # smt.status = prototype_smt.status
+                    # TODO: add changelog
+                    # TODO: log change
+                    # record a reason for the change in simple_history
+                    smt._change_reason = 'Forced synchronization with library component statement'
+                    system_smts_updated.append(smt)
+            # bulk save the changes and update simple_history records to reduce database calls
+            bulk_update_with_history(system_smts_updated, Statement, ['body'], batch_size=500)
+            total_system_smts_updated += len(system_smts_updated)
+
+            # add this prototype smt to any consuming system not currently having a child smt
+            # determine which consuming systems are missing the prototype smt
+            consuming_systems_missing_smt = [cs for cs in consuming_systems if cs not in consuming_systems_to_update]
+            for cs in consuming_systems_missing_smt:
+                # add statement to consuming system's root element
+                prototype_smt.create_system_control_smt_from_component_prototype_smt(cs.root_element.id)
+                total_system_smts_updated =+ 1
+
+        # remove any statements deleted from element in consuming systems
+        # by searching through consuming systems's to delete orphaned statements
+        # associated with the this element
+        for consuming_system in consuming_systems:
+            consumed_smts = consuming_system.root_element.statements_consumed.filter(statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION.name, producer_element=self)
+            for consumed_smt in consumed_smts:
+                if consumed_smt.prototype_synched == STATEMENT_ORPHANED:
+                    # delete statement
+                    consumed_smt.delete()
+                    total_system_smts_updated =+ 1
+                    # TODO: add count for deleted smt
+        return total_system_smts_updated
+
     @property
     def selected_controls_oscal_ctl_ids(self):
         """Return array of selected controls oscal ids"""

controls/tests.py

@@ -581,6 +581,19 @@ def test_component_type_state(self):
         self.assertTrue(e2.component_type == "hardware")
         self.assertTrue(e2.component_state == "disposition")
 
+    def test_element_update_control_implementation_with_prototype(self):
+        e = Element.objects.create(name="New component",  element_type="system")
+        self.assertTrue(e.id is not None)
+        self.assertTrue(e.component_type == "software")
+        # add two statements
+        # create two systems
+        # assign element to two systems
+        # check statements
+        # modify element statements
+        # execute element_update_control_implementation_with_prototype
+        # assert system statements changed
+
+
 class ElementUITests(OrganizationSiteFunctionalTests):
 
     def test_element_create_form(self):

controls/views.py

@@ -1100,7 +1100,7 @@ def as_yaml(self):
 
 class ComponentImporter(object):
 
-    def import_components_as_json(self, import_name, json_object, request=None, existing_import_record=False, stopinvalid=True):
+    def import_components_as_json(self, import_name, json_object, request=None, existing_import_record=False, stopinvalid=True, update=False):
         """Imports Components from a JSON object
 
         @type import_name: str
@@ -1151,7 +1151,6 @@ def import_components_as_json(self, import_name, json_object, request=None, exis
                     import sys
                     sys.exit()
 
-
         # If importing from importcomponents script print issues
         if len(issues) > 0:
             print("\nNOTICE - ISSUES DURING COMPONENT IMPORT\n")
@@ -1162,7 +1161,7 @@ def import_components_as_json(self, import_name, json_object, request=None, exis
             user_owner = request.user
         else:
             user_owner = User.objects.filter(is_superuser=True)[0]
-        created_components = self.create_components(oscal_json, user_owner)
+        created_components = self.create_or_update_components(oscal_json, user_owner, update)
         new_import_record = self.create_import_record(import_name, created_components, existing_import_record=existing_import_record)
         return new_import_record
 
@@ -1191,16 +1190,26 @@ def create_import_record(self, import_name, components, existing_import_record=F
 
         return import_record
 
-    def create_components(self, oscal_json, user_owner=None):
-        """Creates Elements (Components) from valid OSCAL JSON"""
-        components_created = []
+    def create_or_update_components(self, oscal_json, user_owner=None, update=False):
+        """
+        Creates or updates Elements (Components) from valid OSCAL JSON
+        """
+        components_created_or_updated = []
         components = oscal_json['component-definition']['components']
         for component in components:
-            new_component = self.create_component(component, user_owner)
-            if new_component is not None:
-                components_created.append(new_component)
+            # update existing component if update set to true and a component exists with same name
+            component_name = component['title']
+            if update and Element.objects.filter(name=component_name).count() > 0:
+                print(f"[DEBUG] ****** update equals 4: {update}; 1 or more components matching name exists")
+                updated_component = self.update_component(component, user_owner)
+                if updated_component is not None:
+                    components_created_or_updated.append(updated_component)
+            else:
+                new_component = self.create_component(component, user_owner)
+                if new_component is not None:
+                    components_created_or_updated.append(new_component)
 
-        return components_created
+        return components_created_or_updated
 
     def create_component(self, component_json, user_owner=None, private=False):
         """Creates a component from a JSON dict
@@ -1252,11 +1261,6 @@ def create_component(self, component_json, user_owner=None, private=False):
             for control_element in control_implementation_statements:
                 catalog = oscalize_catalog_key(control_element.get('source', None))
                 created_statements = self.create_control_implementation_statements(catalog, control_element, new_component)
-        # If there are no valid statements in the json object
-        if created_statements == []:
-            logger.info(f"The Component {new_component.name} will be deleted as there were no valid statements provided.")
-            new_component.delete()
-            new_component = None
 
         return new_component
 
@@ -1294,6 +1298,73 @@ def create_control_implementation_statements(self, catalog_key, control_element,
         statements_created = Statement.objects.bulk_create(new_statements)
         return statements_created
 
+        #TODO: add atomic
+    def update_component(self, component_json, user_owner=None, private=False):
+        """Updates an existing component from a JSON dict
+
+        @type component_json: dict
+        @param component_json: Component attributes from JSON object
+        @param user_owner: Django user
+        @rtype: Element
+        @returns: Element object updated, None otherwise
+        """
+
+        component_name = component_json['title']
+
+        # while Element.objects.filter(name=component_name).count() > 0:
+        #     component_name = increment_element_name(component_name)
+        if Element.objects.filter(name=component_name).count() > 0:
+            existing_component = Element.objects.filter(name=component_name)[0]
+
+        # update basic details
+        #new_component = Element.objects.create(
+        existing_component.name = component_name
+        existing_component.description = component_json['description'] if 'description' in component_json else 'Description missing'
+        # Components uploaded to the Component Library are all system_element types
+        existing_component.element_type = "system_element"
+        #existing_component.uuid=component_json['uuid'] if 'uuid' in component_json else uuid.uuid4(),
+        existing_component.component_type=component_json['type'] if 'type' in component_json else "software"
+        existing_component.private=private
+        #)
+        existing_component.save()
+
+        logger.info(f"Component {existing_component.name} with UUID {existing_component.uuid} updated.")
+        # TODO: Should change of ownership be allowed?
+        if user_owner:
+            existing_component.assign_owner_permissions(user_owner)
+            logger.info(
+                event="new_element with user as owner",
+                object={"object": "element", "id": existing_component.id, "name":existing_component.name},
+                user={"id": user_owner.id, "username": user_owner.username}
+            )
+
+        component_props = component_json.get('props', None)
+        if component_props:
+            desired_tags = set([prop['value'] for prop in component_props if prop['name'] == 'tag' and 'ns' in prop and prop['ns'] == "https://govready.com/ns/oscal"])
+            existing_tags = Tag.objects.filter(label__in=desired_tags).values('id', 'label')
+            tags_to_create = desired_tags.difference(set([tag['label'] for tag in existing_tags]))
+            new_tags = Tag.objects.bulk_create([Tag(label=tag) for tag in tags_to_create])
+            all_tag_ids = [tag.id for tag in new_tags] + [tag['id'] for tag in existing_tags]
+            existing_component.add_tags(all_tag_ids)
+            existing_component.save()
+        created_statements = []
+        control_implementation_statements = component_json.get('control-implementations', None)
+        # If there data exists the OSCAL component's control-implementations key
+
+        # delete all existing control implementation prototype statements for component
+        statements_deleted = Statement.objects.filter(producer_element_id=existing_component.id, statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name).delete()
+        print(f"[DEBUG] deleting existing control impl prototype smts: {statements_deleted}")
+        # add new statements from OSCAL
+        print(f"[DEBUG] adding new smts from OSCAL")
+        if control_implementation_statements:
+            for control_element in control_implementation_statements:
+                catalog = oscalize_catalog_key(control_element.get('source', None))
+                created_statements = self.create_control_implementation_statements(catalog, control_element, existing_component)
+
+        # TODO: Sync statements to consuming_elements
+
+        return existing_component
+
 def add_selected_components(system, import_record):
         """Add a component from the library or a compliance app to the project and its statements using the import record"""