Bulk update a component's statements across all systems

GovReady#1797

CHANGELOG.md

@@ -1,30 +1,31 @@
 GovReady-Q Release Notes
 ========================
 
-v0.11.4 (December 17, 2022)
+v0.12.0-dev (February 4, 2022)
 ---------------------------
 
 **Developer changes**
 
-* Dynamically set Internet schme (http or https) for swagger interface to support proper URL strings in swagger.
+* Add API endpoint and Element (component) model method to force update all Element consuming systems's control implementation statements with library Elements content.
+* Add parameter createOSCAL API endpoint to indicate update existing components.
+* Upgrade Python libraries.
+* Update NPM libraries.
 
 
-v0.11.3 (December 10, 2022)
+v0.11.4 (December 17, 2022)
 ---------------------------
 
 **Developer changes**
 
-* Add processing for question actions targeted at system to handle `system/add_baseline/<value>` to add additional baseline set of controls to a system without deleting already assigned controls.A
-
+* Dynamically set Internet schme (http or https) for swagger interface to support proper URL strings in swagger.
 
 
 v0.11.3 (December 10, 2022)
 ---------------------------
 
 **Developer changes**
 
-* Add processing for question actions targeted at system to handle `system/add_baseline/<value>` to add additional baseline set of controls to a system without deleting already assigned controls.A
-
+* Add processing for question actions targeted at system to handle `system/add_baseline/<value>` to add additional baseline set of controls to a system without deleting already assigned controls.
 
 
 v0.11.2 (December 10, 2022)
@@ -2868,4 +2869,4 @@ Development changes:
 v0.7.0-rc2 (January 8, 2018)
 ----------------------------
 
-First release.
+First release.

VERSION

@@ -1 +1 @@
-v0.11.4
+v0.12.0-dev

api/base/urls.py

@@ -21,4 +21,3 @@ def get_swagger_urls():
         url(r'^docs/swagger(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'),
         url(r'^docs/swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
     ]
-

api/controls/serializers/element.py

@@ -36,9 +36,18 @@ class Meta:
 
 class WriteElementOscalSerializer(WriteOnlySerializer):
     oscal = serializers.JSONField()
+    update = serializers.NullBooleanField()
     class Meta:
         model = Element
-        fields = ['oscal']
+        fields = ['oscal', 'update']
+
+class WriteSynchConsumingSystemsImplementationStatementsSerializer(WriteOnlySerializer):
+    # oscal = serializers.JSONField()
+    componentId = serializers.IntegerField(min_value=1, max_value=None)
+    class Meta:
+        model = Element
+        fields = ['componentId']
+
 class ReadElementOscalSerializer(ReadOnlySerializer):
     oscal = serializers.SerializerMethodField('get_oscal')
 
@@ -276,4 +285,4 @@ class ElementCreateAndSetRequestSerializer(WriteOnlySerializer):
     status = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True)
     class Meta:
         model = Element
-        fields = ['proposalId', 'userId', 'systemId', 'criteria_comment', 'criteria_reject_comment', 'status']
+        fields = ['proposalId', 'userId', 'systemId', 'criteria_comment', 'criteria_reject_comment', 'status']

api/controls/views/element.py

@@ -19,7 +19,7 @@
     WriteElementTagsSerializer, ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer, \
     WriteElementAppointPartySerializer, ElementPartySerializer, DeletePartyAppointmentsFromElementSerializer, CreateMultipleAppointmentsFromRoleIds, \
     ElementRequestsSerializer, ElementSetRequestsSerializer, ElementCreateAndSetRequestSerializer, \
-    WriteElementOscalSerializer, ReadElementOscalSerializer, SimpleGetElementByNameSerializer
+    WriteElementOscalSerializer, ReadElementOscalSerializer, SimpleGetElementByNameSerializer, WriteSynchConsumingSystemsImplementationStatementsSerializer
 from controls.models import Element, System
 from siteapp.models import Appointment, Party, Proposal, Role, Request, User
 from controls.views import ComponentImporter, OSCALComponentSerializer
@@ -52,7 +52,9 @@ class ElementViewSet(ReadWriteViewSet):
                                            CreateAndSetRequest=ElementCreateAndSetRequestSerializer,
                                            createOSCAL=WriteElementOscalSerializer,
                                            getOSCAL=ReadElementOscalSerializer,
-                                           downloadOSCAL=ReadElementOscalSerializer)
+                                           downloadOSCAL=ReadElementOscalSerializer,
+                                           synchConsumingSystemsImplementationStatements=WriteSynchConsumingSystemsImplementationStatementsSerializer                                       
+                                           )
 
     @action(detail=False, url_path="createOSCAL", methods=["POST"])
     def createOSCAL(self, request, **kwargs):
@@ -61,11 +63,17 @@ def createOSCAL(self, request, **kwargs):
         if "metadata" in request.data["oscal"]["component-definition"]:
             title = request.data["oscal"]["component-definition"]["metadata"]["title"]
         date_string = datetime.now().strftime("%Y-%m-%d-%H-%M")
+
+        # check if update value set to True
+        if "update" in request.data and request.data["update"]:
+            update = True
+        else:
+            update = False
 
         import_record_name = title + "_api-import_" + date_string
         oscal_component_json = json.dumps(request.data["oscal"])
 
-        import_record_result = ComponentImporter().import_components_as_json(import_record_name, oscal_component_json, request)
+        import_record_result = ComponentImporter().import_components_as_json(import_record_name, oscal_component_json, request, update=update)
         element = Element.objects.filter(import_record=import_record_result).first()
 
         serializer_class = self.get_serializer_class('retrieve')
@@ -84,6 +92,24 @@ def createOSCAL(self, request, **kwargs):
     #     serializer = self.get_serializer(serializer_class, element)
     #     return Response(serializer.data)
 
+    @action(detail=False, url_path="synchConsumingSystemsImplementationStatements", methods=["POST"])
+    def synchConsumingSystemsImplementationStatements(self, request, **kwargs):
+        """
+        Force update all element consuming system control impl smts with content of protoype component control impl smt
+        """
+        if "componentId" in request.data:
+            component_id = request.data['componentId']
+            element = Element.objects.filter(id=component_id).first()
+            if element is not None:
+                # TODO: check user permisson
+                system_smts_updated = element.synch_consuming_systems_implementation_statements()
+                result = {"system_smts_updated": system_smts_updated}
+                return Response(result)
+            else:
+                # element not found
+                result = {"system_smts_updated": 0}
+                return Response(result2)
+
     @action(detail=True, url_path="getOSCAL", methods=["GET"])
     def getOSCAL(self, request, **kwargs):
         element, validated_data = self.validate_serializer_and_get_object(request)

controls/models.py

@@ -11,6 +11,7 @@
                                 get_perms_for_model, get_user_perms,
                                 get_users_with_perms, remove_perm)
 from simple_history.models import HistoricalRecords
+from simple_history.utils import bulk_update_with_history
 from jsonfield import JSONField
 from natsort import natsorted
 
@@ -394,6 +395,7 @@ def assign_user_permissions(self, user, permissions):
                 user={"id": user.id, "username": user.username}
             )
             return False
+
     def remove_all_permissions_from_user(self, user):
         try:
             current_permissions = get_user_perms(user, self)
@@ -417,6 +419,7 @@ def remove_all_permissions_from_user(self, user):
                 user={"id": user.id, "username": user.username}
             )
             return False
+
     def get_permissible_users(self):
         return get_users_with_perms(self, attach_perms=True)
 
@@ -597,6 +600,70 @@ def copy(self, name=None):
             smt_copy.save()
         return e_copy
 
+    @transaction.atomic
+    def synch_consuming_systems_implementation_statements(self):
+        """
+        Force update all Element's consuming systems' control implementation statements to be the same
+        as the Element's control implementation prototype statements
+        """
+
+        # get Element's consuming_systems
+        consuming_systems = self.consuming_systems()
+        # get Element's control_implementation_prototype statements
+        element_prototype_smts = self.statements(StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name)
+        # track system control implementation statements touched via synchronization (whether changed or not)
+        total_system_smts_updated = 0
+        consuming_systems_updated = []
+        # loop through Element's control_implementation_prototype statements
+        for prototype_smt in element_prototype_smts:
+            # find the consuming systems' control implementation statements to be updated with current control_implementation_prototype
+            system_smts_to_update = Statement.objects.filter(statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION.name, prototype_id=prototype_smt.id)
+            # track updated smts for bulk update
+            system_smts_updated = []
+            # determine list of all consuming systems to be updated
+            consuming_systems_to_update = Statement.objects.filter(statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION.name, prototype_id=prototype_smt.id).values('consumer_element')
+
+            # consuming systems that have a statement that has been removed from the producing Element
+
+            # update the related control_implementation statements
+            for smt in system_smts_to_update:
+
+                # update the system if not already synced with prototype
+                # TODO: improve Statement.protype_synched() to check pid, status, etc
+                if smt.prototype_synched == STATEMENT_NOT_SYNCHED:
+                    smt.body = prototype_smt.body
+                    smt.pid = prototype_smt.pid
+                    # smt.status = prototype_smt.status
+                    # TODO: add changelog
+                    # TODO: log change
+                    # record a reason for the change in simple_history
+                    smt._change_reason = 'Forced synchronization with library component statement'
+                    system_smts_updated.append(smt)
+            # bulk save the changes and update simple_history records to reduce database calls
+            bulk_update_with_history(system_smts_updated, Statement, ['body'], batch_size=500)
+            total_system_smts_updated += len(system_smts_updated)
+
+            # add this prototype smt to any consuming system not currently having a child smt
+            # determine which consuming systems are missing the prototype smt
+            consuming_systems_missing_smt = [cs for cs in consuming_systems if cs not in consuming_systems_to_update]
+            for cs in consuming_systems_missing_smt:
+                # add statement to consuming system's root element
+                prototype_smt.create_system_control_smt_from_component_prototype_smt(cs.root_element.id)
+                total_system_smts_updated =+ 1
+
+        # remove any statements deleted from element in consuming systems
+        # by searching through consuming systems's to delete orphaned statements
+        # associated with the this element
+        for consuming_system in consuming_systems:
+            consumed_smts = consuming_system.root_element.statements_consumed.filter(statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION.name, producer_element=self)
+            for consumed_smt in consumed_smts:
+                if consumed_smt.prototype_synched == STATEMENT_ORPHANED:
+                    # delete statement
+                    consumed_smt.delete()
+                    total_system_smts_updated =+ 1
+                    # TODO: add count for deleted smt
+        return total_system_smts_updated
+
     @property
     def selected_controls_oscal_ctl_ids(self):
         """Return array of selected controls oscal ids"""

controls/tests.py

@@ -581,6 +581,19 @@ def test_component_type_state(self):
         self.assertTrue(e2.component_type == "hardware")
         self.assertTrue(e2.component_state == "disposition")
 
+    def test_element_update_control_implementation_with_prototype(self):
+        e = Element.objects.create(name="New component",  element_type="system")
+        self.assertTrue(e.id is not None)
+        self.assertTrue(e.component_type == "software")
+        # add two statements
+        # create two systems
+        # assign element to two systems
+        # check statements
+        # modify element statements
+        # execute element_update_control_implementation_with_prototype
+        # assert system statements changed
+
+
 class ElementUITests(OrganizationSiteFunctionalTests):
 
     def test_element_create_form(self):
@@ -1252,4 +1265,3 @@ def create_simple_import_record(self):
         statement.save()
 
         return import_record
-

controls/utilities.py

@@ -82,7 +82,7 @@ def oscalize_control_id(cl_id):
     cl_id = re.sub(r'^([A-Za-z][A-Za-z]-)([0-9]*)([ ]*)\(([0-9]*)\)$', r'\1\2.\4', cl_id)
     # Remove trailing space
     cl_id = cl_id.strip(" ")
-    # makes ure lowercase
+    # makes sure lowercase
     cl_id = cl_id.lower()
 
     return cl_id