From b32e62004f00b464eeb55242d665ee251c95f9ea Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Mon, 7 Mar 2022 04:22:23 -0600 Subject: [PATCH 001/115] feat(Integrations): Support for integrations with APIs. --- controls/migrations/0068_system_info.py | 24 +++ controls/models.py | 1 + integrations/__init__.py | 0 integrations/admin.py | 3 + integrations/apps.py | 6 + integrations/csam/README.py | 0 integrations/csam/__init__.py | 0 integrations/csam/communicate.py | 63 ++++++++ integrations/csam/config-validator.json | 0 integrations/csam/mock.py | 141 ++++++++++++++++++ integrations/github/README.py | 0 integrations/github/__init__.py | 0 integrations/github/communicate.py | 64 ++++++++ integrations/github/config-validator.json | 0 integrations/github/mock.py | 0 integrations/jsonplaceholder/README.py | 0 integrations/jsonplaceholder/__init__.py | 0 integrations/jsonplaceholder/communicate.py | 66 ++++++++ .../jsonplaceholder/config-validator.json | 0 integrations/jsonplaceholder/mock.py | 0 integrations/migrations/__init__.py | 0 integrations/models.py | 3 + integrations/tests.py | 3 + integrations/urls.py | 14 ++ integrations/utils/__init__.py | 0 integrations/utils/integration.py | 88 +++++++++++ integrations/views.py | 90 +++++++++++ integrations/wazuh/README.py | 0 integrations/wazuh/__init__.py | 0 integrations/wazuh/communicate.py | 0 integrations/wazuh/config-validator.json | 0 integrations/wazuh/mock.py | 0 siteapp/settings_application.py | 1 + siteapp/urls.py | 1 + .../0011_alter_systemsettings_details.py | 18 +++ system_settings/models.py | 2 +- 36 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 controls/migrations/0068_system_info.py create mode 100644 integrations/__init__.py create mode 100644 integrations/admin.py create mode 100644 integrations/apps.py create mode 100644 integrations/csam/README.py create mode 100644 integrations/csam/__init__.py create mode 100644 integrations/csam/communicate.py create mode 100644 integrations/csam/config-validator.json create mode 100644 integrations/csam/mock.py create mode 100644 integrations/github/README.py create mode 100644 integrations/github/__init__.py create mode 100644 integrations/github/communicate.py create mode 100644 integrations/github/config-validator.json create mode 100644 integrations/github/mock.py create mode 100644 integrations/jsonplaceholder/README.py create mode 100644 integrations/jsonplaceholder/__init__.py create mode 100644 integrations/jsonplaceholder/communicate.py create mode 100644 integrations/jsonplaceholder/config-validator.json create mode 100644 integrations/jsonplaceholder/mock.py create mode 100644 integrations/migrations/__init__.py create mode 100644 integrations/models.py create mode 100644 integrations/tests.py create mode 100644 integrations/urls.py create mode 100644 integrations/utils/__init__.py create mode 100644 integrations/utils/integration.py create mode 100644 integrations/views.py create mode 100644 integrations/wazuh/README.py create mode 100644 integrations/wazuh/__init__.py create mode 100644 integrations/wazuh/communicate.py create mode 100644 integrations/wazuh/config-validator.json create mode 100644 integrations/wazuh/mock.py create mode 100644 system_settings/migrations/0011_alter_systemsettings_details.py diff --git a/controls/migrations/0068_system_info.py b/controls/migrations/0068_system_info.py new file mode 100644 index 000000000..577db06b1 --- /dev/null +++ b/controls/migrations/0068_system_info.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.12 on 2022-03-05 18:36 + +from django.db import migrations, models + +def system_info_schema(apps, schema_editor): + SystemSettings = apps.get_model('system_settings', 'SystemSettings') + system_info_schema = SystemSettings.objects.create(setting='system_info_schema') + system_info_schema.details = dict() + system_info_schema.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0067_alter_element_private'), + ] + + operations = [ + migrations.AddField( + model_name='system', + name='info', + field=models.JSONField(blank=True, default=dict, help_text='JSON object representing additional system information'), + ), + migrations.RunPython(system_info_schema, migrations.RunPython.noop) + ] diff --git a/controls/models.py b/controls/models.py index cf80af43f..554e5c15d 100644 --- a/controls/models.py +++ b/controls/models.py @@ -522,6 +522,7 @@ class System(auto_prefetch.Model, TagModelMixin): help_text="The Element that is this System. Element must be type [Application, General Support System]") fisma_id = models.CharField(max_length=40, help_text="The FISMA Id of the system", unique=False, blank=True, null=True) + info = models.JSONField(blank=True, default=dict, help_text="JSON object representing additional system information") created = models.DateTimeField(auto_now_add=True, db_index=True) updated = models.DateTimeField(auto_now_add=True, db_index=True) diff --git a/integrations/__init__.py b/integrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/admin.py b/integrations/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/integrations/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/integrations/apps.py b/integrations/apps.py new file mode 100644 index 000000000..73adb7a53 --- /dev/null +++ b/integrations/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class IntegrationsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'integrations' diff --git a/integrations/csam/README.py b/integrations/csam/README.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/csam/__init__.py b/integrations/csam/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/csam/communicate.py b/integrations/csam/communicate.py new file mode 100644 index 000000000..527ccb4c5 --- /dev/null +++ b/integrations/csam/communicate.py @@ -0,0 +1,63 @@ +import requests +import json +from base64 import b64encode +from urllib.parse import urlparse +from integrations.utils.integration import Communication + + +class CSAMCommunication(Communication): + + DESCRIPTION = { + "name": "CSAM", + "description": "CSAM API Service", + "version": "0.1", + "base_url": "http://localhost:9002", + } + + def __init__(self, **kwargs): + assert self.DESCRIPTION, "Developer must assign a description dict" + self.__is_authenticated = False + self.error_msg = {} + self.auth_dict = {} + self.data = None + self.base_url = self.DESCRIPTION['base_url'] + + # def identify(self): + # """Identify which Communication subclass""" + # super().identify() + + def setup(self, **kwargs): + pass + + def get_response(self, endpoint, headers=None, verify=False): + response = requests.get(f"{self.base_url}{endpoint}") + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + + def authenticate(self, user=None, passwd=None): + """Authenticate with service""" + pass + + @property + def is_authenticated(self): + return self.__is_authenticated + + @is_authenticated.setter + def is_authenticated(self, value): + self.__is_authenticated = value + + def extract_data(self, authentication, identifiers): + """Extract data""" + pass + + def transform_data(self, data, system_id=None, title=None, description=None, deployment_uuid=None): + pass + + def load_data(self, data): + pass diff --git a/integrations/csam/config-validator.json b/integrations/csam/config-validator.json new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py new file mode 100644 index 000000000..58ddf4732 --- /dev/null +++ b/integrations/csam/mock.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +######################################################## +# +# A simple Python webserver to generate mock CSAM API +# results +# +# Usage: +# python3 integrations/csam/mock.py +# +# Accessing: +# curl localhost:9002 +# +####################################################### + +# Parse command-line arguments +import click + +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse, parse_qs +import json +import random +import uuid +from pprint import pprint + +PORT = 9002 + +class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + + def mk_csam_system_info_response(self): + csam_system_info_response = { + "system_id": 111, + "name": "My IT System" + } + return csam_system_info_response + + # def mk_sar(self): + # + # sar = { + # "id": f"{random.randint(20431, 34554)}", + # "name": "oscap-scan", + # "description": "SCAP scan Results", + # "ip": f"10.10.0.{str(random.randint(2, 250))}", + # "uuid": f"{str(uuid.uuid4())}", + # "pass": random.randint(200, 300), + # "fail": random.randint(0, 20), + # "error": random.randint(0, 4) + # } + # + # return sar + + def do_GET(self, method=None): + + # Parse path + # print("urlparse(self.path)", urlparse(self.path)) + parsed_path = urlparse(self.path) + # print("parsed_path", parsed_path) + # Parse query params + # print("parsed_path.query",parsed_path.query) + params = parse_qs(parsed_path.query) + print("** params", params) + # params are received as arrays, so get first element in array + system_id = params.get('system_id', [0])[0] + deployment_uuid = params.get('deployment_uuid', ['None'])[0] + if deployment_uuid == 'None': + deployment_uuid = None + + # Route and handle request + if parsed_path.path == "/hello": + """Reply with "hello""""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + data = {"message":"hello"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif parsed_path.path == "/systems": + """Reply with sample CSAM API response for system information""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + # Generate a random small number of endpoint assessments + sar_list = [] + for endpoints in range(0,random.randint(1, 4)): + sar_list.append(self.mk_csam_system_info_response()) + data = {"schema": "GovReadySimpleSAR", + "version": "0.2", + "sar": sar_list, + "assessment-results": {}, + "uuid": f"{str(uuid.uuid4())}", + "metadata": { + "title": random.choice([f"Weekly Scan {random.randint(1, 50)}", + f"Daily Scan {random.randint(1, 365)}", + f"Build Scan {random.randint(25, 100)}" + ]), + "description": None, + "published": "dateTime-with-timezone", + "last-modified": "dateTime-with-timezone", + "schema": "GovReadySimpleSAR", + "version": "0.2", + "system_id": system_id, + "deployment_uuid": deployment_uuid + } + } + + # Temporarily set descriptio to title + data["metadata"]["description"] = data["metadata"]["title"] + pprint(data) + + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif parsed_path.path == "/system/111": + """Reply with system information""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + data = self.mk_csam_system_info_response() + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + else: + """Reply with Path not found""" + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + data = {"message":"Path not found"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + +def main(): + httpd = HTTPServer(('localhost', PORT), SimpleHTTPRequestHandler) + httpd.serve_forever() + +if __name__ == "__main__": + main() diff --git a/integrations/github/README.py b/integrations/github/README.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/github/__init__.py b/integrations/github/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/github/communicate.py b/integrations/github/communicate.py new file mode 100644 index 000000000..bf843e6ec --- /dev/null +++ b/integrations/github/communicate.py @@ -0,0 +1,64 @@ +import requests +import json +from base64 import b64encode +from urllib.parse import urlparse +from integrations.utils.integration import Communication + + +class GithubCommunication(Communication): + + DESCRIPTION = { + "name": "GitHub", + "description": "GitHub API Service", + "version": "0.1" + } + + def __init__(self, **kwargs): + assert self.DESCRIPTION, "Developer must assign a description dict" + self.__is_authenticated = False + self.error_msg = {} + self.auth_dict = {} + self.data = None + self.base_url = "https://github.com/api" + + def identify(self): + """Identify which Communication subclass""" + identity_str = f"This is {self.DESCRIPTION['name']} version {self.DESCRIPTION['version']}" + print(identity_str) + return identity_str + + def setup(self, **kwargs): + pass + + def get_response(self, endpoint, headers=None, verify=False): + response = requests.get(f"{self.base_url}{endpoint}") + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + + def authenticate(self, user=None, passwd=None): + """Authenticate with service""" + pass + + @property + def is_authenticated(self): + return self.__is_authenticated + + @is_authenticated.setter + def is_authenticated(self, value): + self.__is_authenticated = value + + def extract_data(self, authentication, identifiers): + """Extract data""" + pass + + def transform_data(self, data, system_id=None, title=None, description=None, deployment_uuid=None): + pass + + def load_data(self, data): + pass diff --git a/integrations/github/config-validator.json b/integrations/github/config-validator.json new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/github/mock.py b/integrations/github/mock.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/jsonplaceholder/README.py b/integrations/jsonplaceholder/README.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/jsonplaceholder/__init__.py b/integrations/jsonplaceholder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/jsonplaceholder/communicate.py b/integrations/jsonplaceholder/communicate.py new file mode 100644 index 000000000..de23237b0 --- /dev/null +++ b/integrations/jsonplaceholder/communicate.py @@ -0,0 +1,66 @@ +import requests +import json +from base64 import b64encode +from urllib.parse import urlparse +from integrations.utils.integration import Communication + + +class JsonplaceholderCommunication(Communication): + + DESCRIPTION = { + "name": "Jsonplaceholder", + "description": "Test API Jsonplaceholder", + "version": "0.1", + "base_url": "https://jsonplaceholder.typicode.com", + } + + def __init__(self, **kwargs): + assert self.DESCRIPTION, "Developer must assign a description dict" + self.__is_authenticated = False + self.error_msg = {} + self.auth_dict = {} + self.data = None + self.status_code = None + self.base_url = self.DESCRIPTION['base_url'] + + # def identify(self): + # """Identify which Communication subclass""" + # identity_str = f"This is {self.DESCRIPTION['name']} version {self.DESCRIPTION['version']}" + # print(identity_str) + # return identity_str + + def setup(self, **kwargs): + pass + + def get_response(self, endpoint, headers=None, verify=False): + response = requests.get(f"{self.base_url}{endpoint}") + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + + def authenticate(self, user=None, passwd=None): + """Authenticate with service""" + pass + + @property + def is_authenticated(self): + return self.__is_authenticated + + @is_authenticated.setter + def is_authenticated(self, value): + self.__is_authenticated = value + + def extract_data(self, authentication, identifiers): + """Extract data""" + pass + + def transform_data(self, data, system_id=None, title=None, description=None, deployment_uuid=None): + pass + + def load_data(self, data): + pass diff --git a/integrations/jsonplaceholder/config-validator.json b/integrations/jsonplaceholder/config-validator.json new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/jsonplaceholder/mock.py b/integrations/jsonplaceholder/mock.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/migrations/__init__.py b/integrations/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/models.py b/integrations/models.py new file mode 100644 index 000000000..71a836239 --- /dev/null +++ b/integrations/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/integrations/tests.py b/integrations/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/integrations/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/integrations/urls.py b/integrations/urls.py new file mode 100644 index 000000000..ae460ec40 --- /dev/null +++ b/integrations/urls.py @@ -0,0 +1,14 @@ +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import admin +from django.conf import settings +from django.urls import path, re_path + +from . import views + +urlpatterns = [ + # url(r"^jsonplaceholder/users$", views.jsonplaceholders_users, name='jsonplaceholders_users'), + + url(r"^(?P.*)/identify$", views.integration_identify, name='integration_identify'), + url(r"^(?P.*)/endpoint(?P.*)$", views.integration_endpoint, name='integration_endpoint'), +] \ No newline at end of file diff --git a/integrations/utils/__init__.py b/integrations/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/utils/integration.py b/integrations/utils/integration.py new file mode 100644 index 000000000..de145dd43 --- /dev/null +++ b/integrations/utils/integration.py @@ -0,0 +1,88 @@ +import json +import os +import subprocess +import sys +from abc import ABC +from urllib.parse import urlparse + + +class HelperMixin: + + def __init__(self): + pass + + def create_secret(self): + import secrets + alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' + return ''.join(secrets.choice(alphabet) for i in range(50)) + + def check_if_valid_uri(self, x): + try: + result = urlparse(x) + return all([result.scheme, result.netloc]) + except: + return False + + def cleanup(self): + raise NotImplementedError() + + def on_complete(self): + raise NotImplementedError() + + +class Communication(HelperMixin, ABC): + + DESCRIPTION = { + "name": "abstract class", + "version": "0.0" + } + + def __init__(self, **kwargs): + assert self.DESCRIPTION, "Developer must assign a description dict" + self.__is_authenticated = False + self.error_msg = {} + self.auth_dict = {} + self.data = None + self.base_url = None + + def identify(self): + """Identify which Communication subclass""" + identity_str = f"This is {self.DESCRIPTION['name']} version {self.DESCRIPTION['version']}" + print(identity_str) + return identity_str + + def setup(self, **kwargs): + raise NotImplementedError() + + def get_response(self, endpoint, headers=None, verify=False): + response = requests.get(f"{self.base_url}{endpoint}") + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + + def authenticate(self, user=None, passwd=None): + """Authenticate with service""" + raise NotImplementedError() + + @property + def is_authenticated(self): + return self.__is_authenticated + + @is_authenticated.setter + def is_authenticated(self, value): + self.__is_authenticated = value + + def extract_data(self, authentication, identifiers): + """Extract data from Security Service using list of identifiers""" + raise NotImplementedError() + + def transform_data(self, data, system_id=None, title=None, description=None, deployment_uuid=None): + raise NotImplementedError() + + def load_data(self, data): + raise NotImplementedError() diff --git a/integrations/views.py b/integrations/views.py new file mode 100644 index 000000000..e57fb103a --- /dev/null +++ b/integrations/views.py @@ -0,0 +1,90 @@ +from django.shortcuts import render +from django.http import HttpResponse, HttpResponseNotFound +import requests +import argparse +import os +import importlib +from .utils.integration import Communication +# from .models import Endpoint +from .csam.communicate import CSAMCommunication +from .github.communicate import GithubCommunication +from .jsonplaceholder.communicate import JsonplaceholderCommunication + +def set_integration(integration): + """Select correct integration""" + + if integration == "csam": + return CSAMCommunication() + elif integration == "github": + return GithubCommunication() + elif integration == "jsonplaceholder": + return JsonplaceholderCommunication() + else: + return None + + +# def jsonplaceholders_users(request): +# #pull data from third party rest api +# print(1,"========", "view_users") +# endpoint_name = 'Test API Jsonplaceholder' +# endpoint_url = 'https://jsonplaceholder.typicode.com/users' +# endpoint_description = 'Sample json results listing users' + +# from integrations.jsonplaceholder.communicate import JsonplaceholderCommunication +# comms = JsonplaceholderCommunication() +# users = comms.get_response(endpoint_url) + +# # response = requests.get(endpoint_url) +# # convert reponse data into json +# # users = response.json() +# print(users) +# return HttpResponse(users) + +# # add results into Endpoint model +# # ep, created = Endpoint.objects.update_or_create( +# # name=endpoint_name, +# # url=endpoint_url, +# # description=endpoint_description, +# # api_data=users +# # ) + +# # return render(request, "api_client/users.html", {"users": users}) +# pass + +def integration_identify(request, integration): + """Integration returns an identification""" + + # QUESTION: Is there a better way to make dynamic assignments? + # CAN'T USE BELOW APPROACH BECAUSE IMPORT MODULES ONLY DONE ON FIRST INTERPRETATON + # load correct module (directory path) for integration + # path = f"integrations.{integration}.communicate" + # check if path exists + # import ipdb; ipdb.set_trace() + # if not os.path.isdir(os.path.join(f"{os.path.sep}".join(path.split('.')[:-1]))): + # print(f"Path '{path}' to integration module {integration} does not exits") + # raise error + # importlib.invalidate_caches() + # importlib.import_module(path) + # load module's Communication subclass for integration + # communication_classes = Communication.__subclasses__() + # if not communication_classes: + # print(f"Unable to find class inheriting `Communication` in {path}") + # report/raise error + # import ipdb; ipdb.set_trace() + # communication = communication_classes[0]() + + communication = set_integration(integration) + if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') + identified = communication.identify() + return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}") + +def integration_endpoint(request, integration, endpoint=None): + """Communicate with an integrated service""" + + communication = set_integration(integration) + if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') + identified = communication.identify() + data = communication.get_response(endpoint) + + return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + diff --git a/integrations/wazuh/README.py b/integrations/wazuh/README.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/wazuh/__init__.py b/integrations/wazuh/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/wazuh/communicate.py b/integrations/wazuh/communicate.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/wazuh/config-validator.json b/integrations/wazuh/config-validator.json new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/wazuh/mock.py b/integrations/wazuh/mock.py new file mode 100644 index 000000000..e69de29bb diff --git a/siteapp/settings_application.py b/siteapp/settings_application.py index c321b5f29..151a89226 100644 --- a/siteapp/settings_application.py +++ b/siteapp/settings_application.py @@ -15,6 +15,7 @@ 'controls', 'system_settings', 'nlp', + 'integrations', 'loadtesting', ] diff --git a/siteapp/urls.py b/siteapp/urls.py index 68dc6148b..cbdb22b5a 100644 --- a/siteapp/urls.py +++ b/siteapp/urls.py @@ -141,6 +141,7 @@ ] urlpatterns += [url(r'^api/v2/', include('api.urls'))] +urlpatterns += [url(r'^integrations/', include('integrations.urls'))] if settings.OKTA_CONFIG or settings.OIDC_CONFIG: urlpatterns += [ diff --git a/system_settings/migrations/0011_alter_systemsettings_details.py b/system_settings/migrations/0011_alter_systemsettings_details.py new file mode 100644 index 000000000..aa91e80fe --- /dev/null +++ b/system_settings/migrations/0011_alter_systemsettings_details.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2022-03-05 19:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('system_settings', '0010_auto_20210713_1031'), + ] + + operations = [ + migrations.AlterField( + model_name='systemsettings', + name='details', + field=models.JSONField(blank=True, default=dict, help_text='Value objects of System Setting in JSON', null=True), + ), + ] diff --git a/system_settings/models.py b/system_settings/models.py index f03f8f77f..70fc944f5 100644 --- a/system_settings/models.py +++ b/system_settings/models.py @@ -30,7 +30,7 @@ class SystemSettings(models.Model): setting = models.CharField(max_length=200, unique=True, help_text="Name of System Setting") active = models.BooleanField(default=False) - details = models.JSONField(blank=True, null=True, help_text="Value objects of System Setting in JSON") + details = models.JSONField(blank=True, null=True, default=dict, help_text="Value objects of System Setting in JSON") def __str__(self): return self.setting From 2289fdd54dd883d989b578d9e7a9fb0826ba74c8 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Mon, 7 Mar 2022 08:25:39 -0600 Subject: [PATCH 002/115] feat(Integrations): Add Authorization to mock csam service --- integrations/csam/README.md | 29 ++++++ integrations/csam/communicate.py | 12 ++- integrations/csam/mock.py | 92 ++++++++++++++----- .../{csam/README.py => github/README.md} | 0 .../README.py => jsonplaceholder/README.md} | 0 integrations/jsonplaceholder/README.py | 0 6 files changed, 111 insertions(+), 22 deletions(-) create mode 100644 integrations/csam/README.md rename integrations/{csam/README.py => github/README.md} (100%) rename integrations/{github/README.py => jsonplaceholder/README.md} (100%) delete mode 100644 integrations/jsonplaceholder/README.py diff --git a/integrations/csam/README.md b/integrations/csam/README.md new file mode 100644 index 000000000..693db814a --- /dev/null +++ b/integrations/csam/README.md @@ -0,0 +1,29 @@ +# ABOUT CSAM Integreation + +A simple Python webserver to generate mock CSAM API +results +# +Usage: +# + +``` +# Start mock service +python3 integrations/csam/mock.py +``` + +``` +Accessing: + curl localhost:9002/endpoint + curl localhost:9002/hello + curl localhost:9002/system/111 # requires authentication +``` + + + +``` +Accessing with simple authentication: + curl -X 'GET' 'http://localhost:9002/system/111' \ + -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ + -H 'Authorization: Bearer FAD619' +``` + diff --git a/integrations/csam/communicate.py b/integrations/csam/communicate.py index 527ccb4c5..47d309844 100644 --- a/integrations/csam/communicate.py +++ b/integrations/csam/communicate.py @@ -30,7 +30,17 @@ def setup(self, **kwargs): pass def get_response(self, endpoint, headers=None, verify=False): - response = requests.get(f"{self.base_url}{endpoint}") + # set headers + self.PAT_TOKEN = "FAD619BF4A06903215E59A626XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + # accept_header = f'accept: application/json;odata.metadata=minimal;odata.streaming=true' + # auth_header = f'Authorization: Bearer {self.PAT_TOKEN}' + #-H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' + #-H 'Authorization: Bearer FAD619BF4A06903215E59A626XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + headers={"accept":"application/json;odata.metadata=minimal;odata.streaming=true", + "Authorization": f"Bearer {self.PAT_TOKEN}" + } + # get response + response = requests.get(f"{self.base_url}{endpoint}", headers=headers) self.status_code = response.status_code if self.status_code == 200: self.data = response.json() diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index 58ddf4732..fc8fcd883 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -6,10 +6,19 @@ # results # # Usage: +# +# # Start mock service # python3 integrations/csam/mock.py # # Accessing: -# curl localhost:9002 +# curl localhost:9002/endpoint +# curl localhost:9002/hello +# curl localhost:9002/system/111 # requires authentication +# +# Accessing with simple authentication: +# curl -X 'GET' 'http://localhost:9002/system/111' \ +# -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ +# -H 'Authorization: Bearer FAD619' # ####################################################### @@ -34,21 +43,6 @@ def mk_csam_system_info_response(self): } return csam_system_info_response - # def mk_sar(self): - # - # sar = { - # "id": f"{random.randint(20431, 34554)}", - # "name": "oscap-scan", - # "description": "SCAP scan Results", - # "ip": f"10.10.0.{str(random.randint(2, 250))}", - # "uuid": f"{str(uuid.uuid4())}", - # "pass": random.randint(200, 300), - # "fail": random.randint(0, 20), - # "error": random.randint(0, 4) - # } - # - # return sar - def do_GET(self, method=None): # Parse path @@ -67,17 +61,19 @@ def do_GET(self, method=None): # Route and handle request if parsed_path.path == "/hello": - """Reply with "hello""""" + """Reply with 'hello'""" + self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - data = {"message":"hello"} + # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) elif parsed_path.path == "/systems": """Reply with sample CSAM API response for system information""" + self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() @@ -113,13 +109,67 @@ def do_GET(self, method=None): # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - elif parsed_path.path == "/system/111": - """Reply with system information""" + elif parsed_path.path == "/authenticate-test": + """Test authentication""" + + # Test authentication by reading headers and looking for 'Authentication' self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - data = self.mk_csam_system_info_response() + # Read headers + header_type = type(self.headers) + header_dict = dict(self.headers) + # headers = "|||".join(self.headers.split("\n")) + print("headers ======\n", self.headers) + print("header_type ======\n", header_type) + print("header_dict ======\n", header_dict) + print("Authorization header:", self.headers['Authorization']) + + data = { + "reply": "yes", + "Authorization": self.headers['Authorization'], + "pat": self.headers['Authorization'].split("Bearer ")[-1] + } + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif parsed_path.path == "/system/111": + """Reply with system information""" + + # Usage: + # + # # authorized example + # curl -X 'GET' 'http://localhost:9002/system/111' \ + # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ + # -H 'Authorization: Bearer FAD619' + # + # # unauthorized example: + # + # curl -X 'GET' 'http://localhost:9002/system/111' \ + # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' + # + + pat = None + if 'Authorization' in self.headers: + pat = self.headers['Authorization'].split("Bearer ")[-1] + + if pat is None or pat != "FAD619": + # Reply with unauthorized + self.send_response(401) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = { + "message": "Unauthorized request", + "endpoint": parsed_path.path + } + else: + # Request is authenticated + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = self.mk_csam_system_info_response() + # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) diff --git a/integrations/csam/README.py b/integrations/github/README.md similarity index 100% rename from integrations/csam/README.py rename to integrations/github/README.md diff --git a/integrations/github/README.py b/integrations/jsonplaceholder/README.md similarity index 100% rename from integrations/github/README.py rename to integrations/jsonplaceholder/README.md diff --git a/integrations/jsonplaceholder/README.py b/integrations/jsonplaceholder/README.py deleted file mode 100644 index e69de29bb..000000000 From fb4e4f4ad134c6bc57a2a960ebdd965a7cc4ea6a Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 8 Mar 2022 05:25:31 -0600 Subject: [PATCH 003/115] feat(Integrations): Add Integration, Endpoint models for storing data --- CHANGELOG.md | 6 +- integrations/admin.py | 35 ++++++++- integrations/csam/README.md | 20 ++--- integrations/csam/communicate.py | 31 +++++--- integrations/csam/mock.py | 12 +-- integrations/github/communicate.py | 1 + integrations/jsonplaceholder/communicate.py | 1 + integrations/migrations/0001_initial.py | 81 +++++++++++++++++++++ integrations/models.py | 70 +++++++++++++++++- integrations/urls.py | 4 +- integrations/views.py | 60 ++++++--------- 11 files changed, 249 insertions(+), 72 deletions(-) create mode 100644 integrations/migrations/0001_initial.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c1d6ef5fe..95e53e0a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,11 @@ GovReady-Q Release Notes ======================== -v0.9.14-dev (February xx, 2022) +v0.9.14-dev (March xx, 2022) ------------------------------ +Release adds support for private components and integrations with third party services. + **UI changes** * Change label 'certified statement' to 'reference statement'. @@ -19,6 +21,8 @@ v0.9.14-dev (February xx, 2022) * Support for private components by adding 'private' boolean field to controls.models.Element. * Create new components as private and assign owner permissions to user who created the component. * Added tests for component creation form user interface. +* Added extensible Integrations Django appplication to support communication with third-party services via APIs, etc. +* Added initial support for DoJ's CSAM integration. **Bug fixes** diff --git a/integrations/admin.py b/integrations/admin.py index 8c38f3f3d..ffc32a1c1 100644 --- a/integrations/admin.py +++ b/integrations/admin.py @@ -1,3 +1,36 @@ +import csv from django.contrib import admin +from django.http import HttpResponse +from .models import Integration, Endpoint +from guardian.admin import GuardedModelAdmin +from simple_history.admin import SimpleHistoryAdmin +from django_json_widget.widgets import JSONEditorWidget +from jsonfield import JSONField +from django.db import models +from controls.admin import ExportCsvMixin -# Register your models here. + +class IntegrationAdmin(SimpleHistoryAdmin, ExportCsvMixin): + list_display = ('name',) + search_fields = ('id', 'name') + actions = ["export_as_csv"] + readonly_fields = ('created', 'updated') + formfield_overrides = { + models.JSONField: {'widget': JSONEditorWidget}, + } + +class EndpointAdmin(SimpleHistoryAdmin, ExportCsvMixin): + list_display = ('endpoint_path', 'get_integration_name', 'updated') + + @admin.display(ordering='integration__name', description='Name') + def get_integration_name(self, obj): + return obj.integration.name + search_fields = ('id', 'endpoint_path') + actions = ["export_as_csv"] + readonly_fields = ('created', 'updated') + formfield_overrides = { + models.JSONField: {'widget': JSONEditorWidget}, + } + +admin.site.register(Integration, IntegrationAdmin) +admin.site.register(Endpoint, EndpointAdmin) diff --git a/integrations/csam/README.md b/integrations/csam/README.md index 693db814a..b2e80149f 100644 --- a/integrations/csam/README.md +++ b/integrations/csam/README.md @@ -2,9 +2,9 @@ A simple Python webserver to generate mock CSAM API results -# + Usage: -# + ``` # Start mock service @@ -12,18 +12,18 @@ python3 integrations/csam/mock.py ``` ``` -Accessing: - curl localhost:9002/endpoint - curl localhost:9002/hello - curl localhost:9002/system/111 # requires authentication +# Accessing mock service +curl localhost:9002/endpoint +curl localhost:9002/hello +curl localhost:9002/system/111 # requires authentication ``` ``` -Accessing with simple authentication: - curl -X 'GET' 'http://localhost:9002/system/111' \ - -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ - -H 'Authorization: Bearer FAD619' +# Accessing mock service with authentication + curl -X 'GET' 'http://localhost:9002/system/111' \ + -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ + -H 'Authorization: Bearer FAD619' ``` diff --git a/integrations/csam/communicate.py b/integrations/csam/communicate.py index 47d309844..5a347fb00 100644 --- a/integrations/csam/communicate.py +++ b/integrations/csam/communicate.py @@ -2,25 +2,35 @@ import json from base64 import b64encode from urllib.parse import urlparse +from django.shortcuts import get_object_or_404 from integrations.utils.integration import Communication +from integrations.models import Integration, Endpoint class CSAMCommunication(Communication): DESCRIPTION = { - "name": "CSAM", + "name": "csam", "description": "CSAM API Service", "version": "0.1", - "base_url": "http://localhost:9002", + "integration_db_record": True, + "mock": { + "base_url": "http:/localhost:9002", + "personal_access_token": "FAD619" + } } def __init__(self, **kwargs): - assert self.DESCRIPTION, "Developer must assign a description dict" + self.integration_name = self.DESCRIPTION['name'] + self.integration = get_object_or_404(Integration, name=self.integration_name) + self.description = self.integration.description + self.config = self.integration.config + self.base_url = self.config['base_url'] + self.personal_access_token = self.config['personal_access_token'] self.__is_authenticated = False self.error_msg = {} self.auth_dict = {} self.data = None - self.base_url = self.DESCRIPTION['base_url'] # def identify(self): # """Identify which Communication subclass""" @@ -30,14 +40,11 @@ def setup(self, **kwargs): pass def get_response(self, endpoint, headers=None, verify=False): - # set headers - self.PAT_TOKEN = "FAD619BF4A06903215E59A626XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - # accept_header = f'accept: application/json;odata.metadata=minimal;odata.streaming=true' - # auth_header = f'Authorization: Bearer {self.PAT_TOKEN}' - #-H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' - #-H 'Authorization: Bearer FAD619BF4A06903215E59A626XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' - headers={"accept":"application/json;odata.metadata=minimal;odata.streaming=true", - "Authorization": f"Bearer {self.PAT_TOKEN}" + # PAT for mock service is 'FAD619' + # print("1 ===== endpoint",endpoint) + headers={ + "accept":"application/json;odata.metadata=minimal;odata.streaming=true", + "Authorization": f"Bearer {self.personal_access_token}" } # get response response = requests.get(f"{self.base_url}{endpoint}", headers=headers) diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index fc8fcd883..5dffc51c7 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -52,6 +52,7 @@ def do_GET(self, method=None): # Parse query params # print("parsed_path.query",parsed_path.query) params = parse_qs(parsed_path.query) + print("parsed_path.path:", parsed_path.path) print("** params", params) # params are received as arrays, so get first element in array system_id = params.get('system_id', [0])[0] @@ -118,12 +119,11 @@ def do_GET(self, method=None): self.end_headers() # Read headers - header_type = type(self.headers) - header_dict = dict(self.headers) - # headers = "|||".join(self.headers.split("\n")) - print("headers ======\n", self.headers) - print("header_type ======\n", header_type) - print("header_dict ======\n", header_dict) + # header_type = type(self.headers) + # header_dict = dict(self.headers) + # print("headers ======\n", self.headers) + # print("header_type ======\n", header_type) + # print("header_dict ======\n", header_dict) print("Authorization header:", self.headers['Authorization']) data = { diff --git a/integrations/github/communicate.py b/integrations/github/communicate.py index bf843e6ec..bb2a6aaae 100644 --- a/integrations/github/communicate.py +++ b/integrations/github/communicate.py @@ -3,6 +3,7 @@ from base64 import b64encode from urllib.parse import urlparse from integrations.utils.integration import Communication +from integrations.models import Integration, Endpoint class GithubCommunication(Communication): diff --git a/integrations/jsonplaceholder/communicate.py b/integrations/jsonplaceholder/communicate.py index de23237b0..8ce8450f4 100644 --- a/integrations/jsonplaceholder/communicate.py +++ b/integrations/jsonplaceholder/communicate.py @@ -3,6 +3,7 @@ from base64 import b64encode from urllib.parse import urlparse from integrations.utils.integration import Communication +from integrations.models import Integration, Endpoint class JsonplaceholderCommunication(Communication): diff --git a/integrations/migrations/0001_initial.py b/integrations/migrations/0001_initial.py new file mode 100644 index 000000000..f78c5fe83 --- /dev/null +++ b/integrations/migrations/0001_initial.py @@ -0,0 +1,81 @@ +# Generated by Django 3.2.12 on 2022-03-08 10:00 + +import auto_prefetch +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.db.models.manager +import jsonfield.fields +import simple_history.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Integration', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated', models.DateTimeField(auto_now=True, db_index=True, null=True)), + ('name', models.CharField(help_text='Endpoint name in lowercase', max_length=250)), + ('description', models.TextField(blank=True, default='', help_text='Brief description of the Integration', null=True)), + ('config', jsonfield.fields.JSONField(blank=True, default=dict, help_text='Integration configuration', null=True)), + ('config_schema', jsonfield.fields.JSONField(blank=True, default=dict, help_text='Integration schema', null=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='HistoricalEndpoint', + fields=[ + ('id', models.BigIntegerField(auto_created=True, blank=True, db_index=True, verbose_name='ID')), + ('created', models.DateTimeField(blank=True, db_index=True, editable=False)), + ('updated', models.DateTimeField(blank=True, db_index=True, editable=False, null=True)), + ('endpoint_path', models.CharField(blank=True, help_text='Path to the Endpoint', max_length=250, null=True)), + ('description', models.TextField(blank=True, default='', help_text='Brief description of the endpoint', null=True)), + ('element_type', models.CharField(blank=True, help_text='Component type', max_length=150, null=True)), + ('data', jsonfield.fields.JSONField(blank=True, default=dict, help_text='JSON object representing the API results.', null=True)), + ('history_id', models.AutoField(primary_key=True, serialize=False)), + ('history_date', models.DateTimeField()), + ('history_change_reason', models.CharField(max_length=100, null=True)), + ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)), + ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)), + ('integration', auto_prefetch.ForeignKey(blank=True, db_constraint=False, help_text="Endpoint's Integration", null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='integrations.integration')), + ], + options={ + 'verbose_name': 'historical endpoint', + 'ordering': ('-history_date', '-history_id'), + 'get_latest_by': 'history_date', + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name='Endpoint', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated', models.DateTimeField(auto_now=True, db_index=True, null=True)), + ('endpoint_path', models.CharField(blank=True, help_text='Path to the Endpoint', max_length=250, null=True)), + ('description', models.TextField(blank=True, default='', help_text='Brief description of the endpoint', null=True)), + ('element_type', models.CharField(blank=True, help_text='Component type', max_length=150, null=True)), + ('data', jsonfield.fields.JSONField(blank=True, default=dict, help_text='JSON object representing the API results.', null=True)), + ('integration', auto_prefetch.ForeignKey(help_text="Endpoint's Integration", on_delete=django.db.models.deletion.CASCADE, related_name='endpoints', to='integrations.integration')), + ], + options={ + 'abstract': False, + 'base_manager_name': 'prefetch_manager', + }, + managers=[ + ('objects', django.db.models.manager.Manager()), + ('prefetch_manager', django.db.models.manager.Manager()), + ], + ), + ] diff --git a/integrations/models.py b/integrations/models.py index 71a836239..9d25140a2 100644 --- a/integrations/models.py +++ b/integrations/models.py @@ -1,3 +1,71 @@ from django.db import models +from django.db.models import Count +from django.utils.functional import cached_property +from guardian.shortcuts import (assign_perm, get_objects_for_user, + get_perms_for_model, get_user_perms, + get_users_with_perms, remove_perm) +from simple_history.models import HistoricalRecords +from jsonfield import JSONField +from natsort import natsorted +import uuid +import auto_prefetch +from django.db import transaction -# Create your models here. +from api.base.models import BaseModel + + +class Integration(BaseModel): + name = models.CharField(max_length=250, help_text="Endpoint name in lowercase", unique=False, blank=False, null=False) + description = models.TextField(default="", help_text="Brief description of the Integration", unique=False, blank=True, null=True) + config = JSONField(blank=True, null=True, default=dict, help_text="Integration configuration") + config_schema = JSONField(blank=True, null=True, default=dict, help_text="Integration schema") + + def __str__(self): + return f"'{self.name} id={self.id}'" + + def __repr__(self): + return f"'{self.name} id={self.id}'" + + # { + # "integrations": [ + # "csam": { + # "authenticate": { + # "personal_access_token": "fsdfkjlkjsdfljk" + # } + # "map_endpoints": { + # "system_info": "/system/id/{id}" + # }, + # "data": { + # "system_info": { + # "name": "My IT System", + # "system_id": 111 + # } + # } + # "map_data": [ + # { + # "local_value": "System.name", + # "integration_value": "system_info['name']" + # } + # ] + + # } + # ] + # } + +class Endpoint(auto_prefetch.Model, BaseModel): + integration = auto_prefetch.ForeignKey(Integration, related_name="endpoints", on_delete=models.CASCADE, + help_text="Endpoint's Integration") + endpoint_path = models.CharField(max_length=250, help_text="Path to the Endpoint", unique=False, blank=True, null=True) + description = models.TextField(default="", help_text="Brief description of the endpoint", unique=False, blank=True, null=True) + element_type = models.CharField(max_length=150, help_text="Component type", unique=False, blank=True, null=True) + data = JSONField(blank=True, null=True, default=dict, help_text="JSON object representing the API results.") + history = HistoricalRecords(cascade_delete_history=True) + + def __str__(self): + return f"'{self.integration.name} {self.endpoint_path} id={self.id}'" + + def __repr__(self): + return f"'{self.integration.name}{self.endpoint_path} id={self.id}'" + + # def get_absolute_url(self): + # return f"/integrations/{integration}/data/endpoint{endpoint_path}" diff --git a/integrations/urls.py b/integrations/urls.py index ae460ec40..db09f7a86 100644 --- a/integrations/urls.py +++ b/integrations/urls.py @@ -9,6 +9,6 @@ urlpatterns = [ # url(r"^jsonplaceholder/users$", views.jsonplaceholders_users, name='jsonplaceholders_users'), - url(r"^(?P.*)/identify$", views.integration_identify, name='integration_identify'), - url(r"^(?P.*)/endpoint(?P.*)$", views.integration_endpoint, name='integration_endpoint'), + url(r"^(?P.*)/identify$", views.integration_identify, name='integration_identify'), + url(r"^(?P.*)/endpoint(?P.*)$", views.integration_endpoint, name='integration_endpoint'), # /integrations/csam/endpoint/system/111 ] \ No newline at end of file diff --git a/integrations/views.py b/integrations/views.py index e57fb103a..8c58aa63a 100644 --- a/integrations/views.py +++ b/integrations/views.py @@ -1,15 +1,16 @@ -from django.shortcuts import render +from django.shortcuts import get_object_or_404, redirect, render from django.http import HttpResponse, HttpResponseNotFound import requests import argparse import os import importlib from .utils.integration import Communication -# from .models import Endpoint +from .models import Integration, Endpoint from .csam.communicate import CSAMCommunication from .github.communicate import GithubCommunication from .jsonplaceholder.communicate import JsonplaceholderCommunication + def set_integration(integration): """Select correct integration""" @@ -22,38 +23,6 @@ def set_integration(integration): else: return None - -# def jsonplaceholders_users(request): -# #pull data from third party rest api -# print(1,"========", "view_users") -# endpoint_name = 'Test API Jsonplaceholder' -# endpoint_url = 'https://jsonplaceholder.typicode.com/users' -# endpoint_description = 'Sample json results listing users' - -# from integrations.jsonplaceholder.communicate import JsonplaceholderCommunication -# comms = JsonplaceholderCommunication() -# users = comms.get_response(endpoint_url) - -# # response = requests.get(endpoint_url) -# # convert reponse data into json -# # users = response.json() -# print(users) -# return HttpResponse(users) - -# # add results into Endpoint model -# # ep, created = Endpoint.objects.update_or_create( -# # name=endpoint_name, -# # url=endpoint_url, -# # description=endpoint_description, -# # api_data=users -# # ) - -# # return render(request, "api_client/users.html", {"users": users}) -# pass - -def integration_identify(request, integration): - """Integration returns an identification""" - # QUESTION: Is there a better way to make dynamic assignments? # CAN'T USE BELOW APPROACH BECAUSE IMPORT MODULES ONLY DONE ON FIRST INTERPRETATON # load correct module (directory path) for integration @@ -73,18 +42,31 @@ def integration_identify(request, integration): # import ipdb; ipdb.set_trace() # communication = communication_classes[0]() - communication = set_integration(integration) +def integration_identify(request, integration_name): + """Integration returns an identification""" + + integration = get_object_or_404(Integration, name=integration_name) if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') identified = communication.identify() - return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}") + return HttpResponse(f"Attempting to communicate with '{integration_name}' integration: {identified}") -def integration_endpoint(request, integration, endpoint=None): +def integration_endpoint(request, integration_name, endpoint=None): """Communicate with an integrated service""" - communication = set_integration(integration) - if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') + integration = get_object_or_404(Integration, name=integration_name) + if integration is None: return HttpResponseNotFound(f'

404 - Integration configuration not found.

') + communication = set_integration(integration_name) + if communication is None: return HttpResponseNotFound(f'

404 - Integration not found.

') identified = communication.identify() data = communication.get_response(endpoint) + # add results into Endpoint model + + ep, created = Endpoint.objects.update_or_create( + integration=integration, + endpoint_path=endpoint, + data=data + ) + return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") From 8def54fcf9e79520e9d13d7187f7569dd0a96bce Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 8 Mar 2022 10:14:35 -0600 Subject: [PATCH 004/115] feat(Integrations): Clean up --- integrations/github/communicate.py | 7 ++++++- integrations/utils/integration.py | 28 ++++++++++++++++++++++++---- integrations/views.py | 1 + 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/integrations/github/communicate.py b/integrations/github/communicate.py index bb2a6aaae..03375cbec 100644 --- a/integrations/github/communicate.py +++ b/integrations/github/communicate.py @@ -11,7 +11,12 @@ class GithubCommunication(Communication): DESCRIPTION = { "name": "GitHub", "description": "GitHub API Service", - "version": "0.1" + "version": "0.1", + "integration_db_record": False, + "mock": { + "base_url": "http:/localhost:9003", + "personal_access_token": None + } } def __init__(self, **kwargs): diff --git a/integrations/utils/integration.py b/integrations/utils/integration.py index de145dd43..b2ffe1f73 100644 --- a/integrations/utils/integration.py +++ b/integrations/utils/integration.py @@ -33,17 +33,32 @@ def on_complete(self): class Communication(HelperMixin, ABC): DESCRIPTION = { - "name": "abstract class", - "version": "0.0" + "name": "abstract class", + "description": "Abstract integration class", + "version": "0.0", + "integration_db_record": False, + "mock": { + "base_url": "http:/localhost:9001", + "personal_access_token": None + } } def __init__(self, **kwargs): - assert self.DESCRIPTION, "Developer must assign a description dict" + self.integration_name = self.DESCRIPTION['name'] + + if self.DESCRIPTION['integration_db_record']: + self.integration = get_object_or_404(Integration, name=self.integration_name) + self.description = self.integration.description + self.config = self.integration.config + self.base_url = self.config['base_url'] + else: + self.description = self.DESCRIPTION['description'] + self.personal_access_token = self.config['personal_access_token'] self.__is_authenticated = False self.error_msg = {} self.auth_dict = {} + self.version = self.DESCRIPTION['version'] self.data = None - self.base_url = None def identify(self): """Identify which Communication subclass""" @@ -54,6 +69,11 @@ def identify(self): def setup(self, **kwargs): raise NotImplementedError() + def msg_missing_configuration(self): + """Message that the integration is not properly configured""" + + msg = f"The {self.integration_name} integration is not fully configured. The problem could be no database record or no integration subclass." + def get_response(self, endpoint, headers=None, verify=False): response = requests.get(f"{self.base_url}{endpoint}") self.status_code = response.status_code diff --git a/integrations/views.py b/integrations/views.py index 8c58aa63a..5b5d952b7 100644 --- a/integrations/views.py +++ b/integrations/views.py @@ -46,6 +46,7 @@ def integration_identify(request, integration_name): """Integration returns an identification""" integration = get_object_or_404(Integration, name=integration_name) + communication = set_integration(integration_name) if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') identified = communication.identify() return HttpResponse(f"Attempting to communicate with '{integration_name}' integration: {identified}") From 4455bf6eb6b83b75b19053f1af15b8b69f1b9cbd Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 8 Mar 2022 12:20:09 -0600 Subject: [PATCH 005/115] feat(Integrations): Default csam integration ssl verify to False --- integrations/csam/communicate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integrations/csam/communicate.py b/integrations/csam/communicate.py index 5a347fb00..d6e8e0381 100644 --- a/integrations/csam/communicate.py +++ b/integrations/csam/communicate.py @@ -26,6 +26,7 @@ def __init__(self, **kwargs): self.description = self.integration.description self.config = self.integration.config self.base_url = self.config['base_url'] + self.ssl_verify = self.config.get('ssl_verify', False) self.personal_access_token = self.config['personal_access_token'] self.__is_authenticated = False self.error_msg = {} @@ -47,7 +48,9 @@ def get_response(self, endpoint, headers=None, verify=False): "Authorization": f"Bearer {self.personal_access_token}" } # get response - response = requests.get(f"{self.base_url}{endpoint}", headers=headers) + if self.ssl_verify: + verify = self.ssl_verify + response = requests.get(f"{self.base_url}{endpoint}", headers=headers, verify=verify) self.status_code = response.status_code if self.status_code == 200: self.data = response.json() From f0cf67a1993705c0ffaf3a074ff92061ee2d69d5 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 11 Mar 2022 04:55:24 -0600 Subject: [PATCH 006/115] feat(Integrations): Improve integrations architecture, support POST --- integrations/csam/communicate.py | 26 ++++++- integrations/csam/config-validator.json | 0 integrations/csam/mock.py | 77 +++++++------------ integrations/csam/urls.py | 10 +++ integrations/csam/views.py | 56 ++++++++++++++ integrations/github/config-validator.json | 0 integrations/jsonplaceholder/communicate.py | 7 ++ .../jsonplaceholder/config-validator.json | 0 integrations/jsonplaceholder/urls.py | 10 +++ integrations/jsonplaceholder/views.py | 65 ++++++++++++++++ integrations/urls.py | 13 ++-- integrations/views.py | 19 +++++ 12 files changed, 225 insertions(+), 58 deletions(-) delete mode 100644 integrations/csam/config-validator.json create mode 100644 integrations/csam/urls.py create mode 100644 integrations/csam/views.py delete mode 100644 integrations/github/config-validator.json delete mode 100644 integrations/jsonplaceholder/config-validator.json create mode 100644 integrations/jsonplaceholder/urls.py create mode 100644 integrations/jsonplaceholder/views.py diff --git a/integrations/csam/communicate.py b/integrations/csam/communicate.py index d6e8e0381..28ebcaec4 100644 --- a/integrations/csam/communicate.py +++ b/integrations/csam/communicate.py @@ -21,6 +21,7 @@ class CSAMCommunication(Communication): } def __init__(self, **kwargs): + self.status_code = None self.integration_name = self.DESCRIPTION['name'] self.integration = get_object_or_404(Integration, name=self.integration_name) self.description = self.integration.description @@ -40,9 +41,10 @@ def __init__(self, **kwargs): def setup(self, **kwargs): pass - def get_response(self, endpoint, headers=None, verify=False): + def get_response(self, endpoint, headers=None, params=None, verify=False): + """Send request using GET""" + # PAT for mock service is 'FAD619' - # print("1 ===== endpoint",endpoint) headers={ "accept":"application/json;odata.metadata=minimal;odata.streaming=true", "Authorization": f"Bearer {self.personal_access_token}" @@ -60,6 +62,26 @@ def get_response(self, endpoint, headers=None, verify=False): pass return self.data + def post_response(self, endpoint, data=None, params=None, verify=False): + """Send request using POST""" + + headers = { + "accept": "application/json;odata.metadata=minimal;odata.streaming=true", + "Authorization": f"Bearer {self.personal_access_token}" + } + # get response + if self.ssl_verify: + verify = self.ssl_verify + response = requests.post(f"{self.base_url}{endpoint}", headers=headers, data=data, verify=verify) + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + def authenticate(self, user=None, passwd=None): """Authenticate with service""" pass diff --git a/integrations/csam/config-validator.json b/integrations/csam/config-validator.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index 5dffc51c7..dadffaf01 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -28,9 +28,6 @@ from http.server import HTTPServer, BaseHTTPRequestHandler from urllib.parse import urlparse, parse_qs import json -import random -import uuid -from pprint import pprint PORT = 9002 @@ -46,19 +43,12 @@ def mk_csam_system_info_response(self): def do_GET(self, method=None): # Parse path - # print("urlparse(self.path)", urlparse(self.path)) parsed_path = urlparse(self.path) - # print("parsed_path", parsed_path) - # Parse query params - # print("parsed_path.query",parsed_path.query) params = parse_qs(parsed_path.query) print("parsed_path.path:", parsed_path.path) print("** params", params) # params are received as arrays, so get first element in array system_id = params.get('system_id', [0])[0] - deployment_uuid = params.get('deployment_uuid', ['None'])[0] - if deployment_uuid == 'None': - deployment_uuid = None # Route and handle request if parsed_path.path == "/hello": @@ -72,44 +62,6 @@ def do_GET(self, method=None): # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - elif parsed_path.path == "/systems": - """Reply with sample CSAM API response for system information""" - - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - - # Generate a random small number of endpoint assessments - sar_list = [] - for endpoints in range(0,random.randint(1, 4)): - sar_list.append(self.mk_csam_system_info_response()) - data = {"schema": "GovReadySimpleSAR", - "version": "0.2", - "sar": sar_list, - "assessment-results": {}, - "uuid": f"{str(uuid.uuid4())}", - "metadata": { - "title": random.choice([f"Weekly Scan {random.randint(1, 50)}", - f"Daily Scan {random.randint(1, 365)}", - f"Build Scan {random.randint(25, 100)}" - ]), - "description": None, - "published": "dateTime-with-timezone", - "last-modified": "dateTime-with-timezone", - "schema": "GovReadySimpleSAR", - "version": "0.2", - "system_id": system_id, - "deployment_uuid": deployment_uuid - } - } - - # Temporarily set descriptio to title - data["metadata"]["description"] = data["metadata"]["title"] - pprint(data) - - # Send the JSON response - self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - elif parsed_path.path == "/authenticate-test": """Test authentication""" @@ -183,6 +135,35 @@ def do_GET(self, method=None): # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + def do_POST(self): + + parsed_path = urlparse(self.path) + params = parse_qs(parsed_path.query) + print("parsed_path.path:", parsed_path.path) + print("** params", params) + + # Route and handle request + if parsed_path.path == "/hello": + """Reply with 'hello'""" + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message": "hello, POST"} + + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + else: + """Reply with Path not found""" + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + data = {"message":"Path not found"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + def main(): httpd = HTTPServer(('localhost', PORT), SimpleHTTPRequestHandler) httpd.serve_forever() diff --git a/integrations/csam/urls.py b/integrations/csam/urls.py new file mode 100644 index 000000000..e209e81b4 --- /dev/null +++ b/integrations/csam/urls.py @@ -0,0 +1,10 @@ +from django.conf.urls import include, url +from . import views + +urlpatterns = [ + url(r"^identify$", views.integration_identify, name='integration_identify'), + url(r"^endpoint(?P.*)$", views.integration_endpoint, + name='integration_endpoint'), # Ex: /integrations/csam/endpoint/system/111 + url(r"^post_endpoint(?P.*)$", views.integration_endpoint_post, + name='integration_endpoint'), +] diff --git a/integrations/csam/views.py b/integrations/csam/views.py new file mode 100644 index 000000000..c6d29936a --- /dev/null +++ b/integrations/csam/views.py @@ -0,0 +1,56 @@ +from django.shortcuts import get_object_or_404, redirect, render +from django.http import HttpResponse, HttpResponseNotFound +from integrations.models import Integration, Endpoint +from .communicate import CSAMCommunication + + +def set_integration(): + """Select correct integration""" + return CSAMCommunication() + +def integration_identify(request): + """Integration returns an identification""" + + communication = set_integration() + if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') + identified = communication.identify() + return HttpResponse(f"Attempting to communicate with csam integration: {identified}") + +def integration_endpoint(request, integration_name='csam', endpoint=None): + """Communicate with an integrated service""" + + integration = get_object_or_404(Integration, name=integration_name) + if integration is None: return HttpResponseNotFound(f'

404 - Integration configuration not found.

') + communication = set_integration() + if communication is None: return HttpResponseNotFound(f'

404 - Integration not found.

') + identified = communication.identify() + data = communication.get_response(endpoint) + + # add results into Endpoint model + ep, created = Endpoint.objects.update_or_create( + integration=integration, + endpoint_path=endpoint, + data=data + ) + + return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + +def integration_endpoint_post(request, integration_name='csam', endpoint=None): + """Communicate with an integrated service using POST""" + + integration = get_object_or_404(Integration, name=integration_name) + if integration is None: return HttpResponseNotFound(f'

404 - Integration configuration not found.

') + communication = set_integration() + if communication is None: return HttpResponseNotFound(f'

404 - Integration not found.

') + identified = communication.identify() + data = communication.post_response(endpoint) + + # add results into Endpoint model + + ep, created = Endpoint.objects.update_or_create( + integration=integration, + endpoint_path=endpoint, + data=data + ) + + return HttpResponse(f"Attempting to communicate using POST with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") diff --git a/integrations/github/config-validator.json b/integrations/github/config-validator.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/integrations/jsonplaceholder/communicate.py b/integrations/jsonplaceholder/communicate.py index 8ce8450f4..6e95eda06 100644 --- a/integrations/jsonplaceholder/communicate.py +++ b/integrations/jsonplaceholder/communicate.py @@ -13,8 +13,15 @@ class JsonplaceholderCommunication(Communication): "description": "Test API Jsonplaceholder", "version": "0.1", "base_url": "https://jsonplaceholder.typicode.com", + "personal_access_token": None } + # integrations record config value: + # { + # "base_url": "http://jsonplaceholder.typicode.com", + # "personal_access_token": null + # } + def __init__(self, **kwargs): assert self.DESCRIPTION, "Developer must assign a description dict" self.__is_authenticated = False diff --git a/integrations/jsonplaceholder/config-validator.json b/integrations/jsonplaceholder/config-validator.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/integrations/jsonplaceholder/urls.py b/integrations/jsonplaceholder/urls.py new file mode 100644 index 000000000..e209e81b4 --- /dev/null +++ b/integrations/jsonplaceholder/urls.py @@ -0,0 +1,10 @@ +from django.conf.urls import include, url +from . import views + +urlpatterns = [ + url(r"^identify$", views.integration_identify, name='integration_identify'), + url(r"^endpoint(?P.*)$", views.integration_endpoint, + name='integration_endpoint'), # Ex: /integrations/csam/endpoint/system/111 + url(r"^post_endpoint(?P.*)$", views.integration_endpoint_post, + name='integration_endpoint'), +] diff --git a/integrations/jsonplaceholder/views.py b/integrations/jsonplaceholder/views.py new file mode 100644 index 000000000..5bce09e01 --- /dev/null +++ b/integrations/jsonplaceholder/views.py @@ -0,0 +1,65 @@ +from django.shortcuts import get_object_or_404, redirect, render +from django.http import HttpResponse, HttpResponseNotFound +from integrations.models import Integration, Endpoint +from .communicate import JsonplaceholderCommunication + +INTEGRATION_NAME = 'jsonplaceholder' + + +def set_integration(): + """Select correct integration""" + return JsonplaceholderCommunication() + + +def integration_identify(request): + """Integration returns an identification""" + + communication = set_integration() + identified = communication.identify() + return HttpResponse(f"Attempting to communicate with {INTEGRATION_NAME} integration: {identified}") + + +def integration_endpoint(request, integration_name=INTEGRATION_NAME, endpoint=None): + """Communicate with an integrated service""" + + try: + integration = get_object_or_404(Integration, name=integration_name) + except: + return HttpResponseNotFound(f'

404 - Integration configuration missing. Create Integration database record.

') + communication = set_integration() + identified = communication.identify() + data = communication.get_response(endpoint) + + # add results into Endpoint model + ep, created = Endpoint.objects.update_or_create( + integration=integration, + endpoint_path=endpoint, + data=data + ) + + return HttpResponse( + f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + + +def integration_endpoint_post(request, integration_name=INTEGRATION_NAME, endpoint=None): + """Communicate with an integrated service using POST""" + + try: + integration = get_object_or_404(Integration, name=integration_name) + except: + return HttpResponseNotFound( + f'

404 - Integration configuration missing. Create Integration database record.

') + communication = set_integration() + identified = communication.identify() + data = communication.post_response(endpoint) + + # add results into Endpoint model + + ep, created = Endpoint.objects.update_or_create( + integration=integration, + endpoint_path=endpoint, + data=data + ) + + return HttpResponse( + f"Attempting to communicate using POST with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") diff --git a/integrations/urls.py b/integrations/urls.py index db09f7a86..8f3163ed2 100644 --- a/integrations/urls.py +++ b/integrations/urls.py @@ -1,14 +1,11 @@ -from django.conf import settings from django.conf.urls import include, url -from django.contrib import admin -from django.conf import settings -from django.urls import path, re_path - from . import views urlpatterns = [ # url(r"^jsonplaceholder/users$", views.jsonplaceholders_users, name='jsonplaceholders_users'), - - url(r"^(?P.*)/identify$", views.integration_identify, name='integration_identify'), - url(r"^(?P.*)/endpoint(?P.*)$", views.integration_endpoint, name='integration_endpoint'), # /integrations/csam/endpoint/system/111 + # url(r"^(?P.*)/identify$", views.integration_identify, name='integration_identify'), + # url(r"^(?P.*)/endpoint(?P.*)$", views.integration_endpoint, + # name='integration_endpoint'), # Ex: /integrations/csam/endpoint/system/111 + url(r"^csam/", include("integrations.csam.urls")), + url(r"^jsonplaceholder/", include("integrations.jsonplaceholder.urls")), ] \ No newline at end of file diff --git a/integrations/views.py b/integrations/views.py index 5b5d952b7..a4d03d10b 100644 --- a/integrations/views.py +++ b/integrations/views.py @@ -71,3 +71,22 @@ def integration_endpoint(request, integration_name, endpoint=None): return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") +def integration_endpoint_post(request, integration_name, endpoint=None): + """Communicate with an integrated service using POST""" + + integration = get_object_or_404(Integration, name=integration_name) + if integration is None: return HttpResponseNotFound(f'

404 - Integration configuration not found.

') + communication = set_integration(integration_name) + if communication is None: return HttpResponseNotFound(f'

404 - Integration not found.

') + identified = communication.identify() + data = communication.post_response(endpoint) + + # add results into Endpoint model + + ep, created = Endpoint.objects.update_or_create( + integration=integration, + endpoint_path=endpoint, + data=data + ) + + return HttpResponse(f"Attempting to communicate using POST with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") From 5c42fe1832160b3de5f1163b6794b1e77dbfe05e Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sat, 12 Mar 2022 04:29:04 -0600 Subject: [PATCH 007/115] feat(Integrations): Add POST data to csam; refactoring --- integrations/csam/mock.py | 65 ++++++++++++++++++++----- integrations/csam/views.py | 70 +++++++++++++++------------ integrations/jsonplaceholder/views.py | 60 ++++++++++++----------- 3 files changed, 126 insertions(+), 69 deletions(-) diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index dadffaf01..88446f826 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -33,12 +33,14 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + SYSTEM = { + "system_id": 111, + "name": "My IT System", + "description": "This is a simple test system" + } + def mk_csam_system_info_response(self): - csam_system_info_response = { - "system_id": 111, - "name": "My IT System" - } - return csam_system_info_response + return self.SYSTEM def do_GET(self, method=None): @@ -71,11 +73,6 @@ def do_GET(self, method=None): self.end_headers() # Read headers - # header_type = type(self.headers) - # header_dict = dict(self.headers) - # print("headers ======\n", self.headers) - # print("header_type ======\n", header_type) - # print("header_dict ======\n", header_dict) print("Authorization header:", self.headers['Authorization']) data = { @@ -97,7 +94,6 @@ def do_GET(self, method=None): # -H 'Authorization: Bearer FAD619' # # # unauthorized example: - # # curl -X 'GET' 'http://localhost:9002/system/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' # @@ -154,6 +150,53 @@ def do_POST(self): # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + if parsed_path.path == "/system/111": + """Update system information""" + + # Usage: + # + # # authorized example + # curl -X 'POST' 'http://localhost:9002/system/111' \ + # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ + # -H 'Authorization: Bearer FAD619' + # + # # unauthorized example: + # curl -X 'POST' 'http://localhost:9002/system/111' \ + # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' + # + + pat = None + if 'Authorization' in self.headers: + pat = self.headers['Authorization'].split("Bearer ")[-1] + + if pat is None or pat != "FAD619": + # Reply with unauthorized + self.send_response(401) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = { + "message": "Unauthorized request", + "endpoint": parsed_path.path + } + else: + # Request is authenticated - read POST data and update system info + content_length = int(self.headers['Content-Length']) + self.post_data = self.rfile.read(content_length) + self.post_data_json = json.loads(self.post_data) + # post_data_decoded = post_data.decode('utf-8') + self.SYSTEM['name'] = self.post_data_json.get('name', self.SYSTEM['name']) + self.SYSTEM['description'] = self.post_data_json.get('description', self.SYSTEM['description']) + + # Send response + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + + data = self.mk_csam_system_info_response() + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + else: """Reply with Path not found""" self.send_response(404) diff --git a/integrations/csam/views.py b/integrations/csam/views.py index c6d29936a..b5eec62c8 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -1,56 +1,66 @@ +import json from django.shortcuts import get_object_or_404, redirect, render from django.http import HttpResponse, HttpResponseNotFound from integrations.models import Integration, Endpoint from .communicate import CSAMCommunication +INTEGRATION_NAME = 'csam' +try: + INTEGRATION = get_object_or_404(Integration, name=INTEGRATION_NAME) +except: + HttpResponseNotFound(f'

404 - Integration configuration missing. Create Integration database record.

') def set_integration(): - """Select correct integration""" return CSAMCommunication() def integration_identify(request): """Integration returns an identification""" communication = set_integration() - if communication is None: return HttpResponseNotFound('

404 - Integration not found.

') - identified = communication.identify() - return HttpResponse(f"Attempting to communicate with csam integration: {identified}") + return HttpResponse(f"Attempting to communicate with csam integration: {communication.identify()}") -def integration_endpoint(request, integration_name='csam', endpoint=None): +def integration_endpoint(request, endpoint=None): """Communicate with an integrated service""" - integration = get_object_or_404(Integration, name=integration_name) - if integration is None: return HttpResponseNotFound(f'

404 - Integration configuration not found.

') communication = set_integration() - if communication is None: return HttpResponseNotFound(f'

404 - Integration not found.

') - identified = communication.identify() data = communication.get_response(endpoint) - - # add results into Endpoint model - ep, created = Endpoint.objects.update_or_create( - integration=integration, - endpoint_path=endpoint, - data=data + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint ) + ep.data = data + ep.save() - return HttpResponse(f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") -def integration_endpoint_post(request, integration_name='csam', endpoint=None): +def integration_endpoint_post(request, endpoint=None): """Communicate with an integrated service using POST""" - integration = get_object_or_404(Integration, name=integration_name) - if integration is None: return HttpResponseNotFound(f'

404 - Integration configuration not found.

') + post_data = { + "name": "My IT System2", + "description": "This is a more complex test system" + } communication = set_integration() - if communication is None: return HttpResponseNotFound(f'

404 - Integration not found.

') - identified = communication.identify() - data = communication.post_response(endpoint) - - # add results into Endpoint model - - ep, created = Endpoint.objects.update_or_create( - integration=integration, - endpoint_path=endpoint, - data=data + data = communication.post_response(endpoint, data=json.dumps(post_data)) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint ) + ep.data = data + ep.save() - return HttpResponse(f"Attempting to communicate using POST with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + return HttpResponse( + f"

Attempting to communicate using POST with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

endpoint: {endpoint}.

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") \ No newline at end of file diff --git a/integrations/jsonplaceholder/views.py b/integrations/jsonplaceholder/views.py index 5bce09e01..d4927bf75 100644 --- a/integrations/jsonplaceholder/views.py +++ b/integrations/jsonplaceholder/views.py @@ -1,9 +1,14 @@ +import json from django.shortcuts import get_object_or_404, redirect, render from django.http import HttpResponse, HttpResponseNotFound from integrations.models import Integration, Endpoint from .communicate import JsonplaceholderCommunication INTEGRATION_NAME = 'jsonplaceholder' +try: + INTEGRATION = get_object_or_404(Integration, name=INTEGRATION_NAME) +except: + HttpResponseNotFound(f'

404 - Integration configuration missing. Create Integration database record.

') def set_integration(): @@ -15,51 +20,50 @@ def integration_identify(request): """Integration returns an identification""" communication = set_integration() - identified = communication.identify() - return HttpResponse(f"Attempting to communicate with {INTEGRATION_NAME} integration: {identified}") + return HttpResponse(f"Attempting to communicate with {INTEGRATION_NAME} integration: {communication.identify()}") -def integration_endpoint(request, integration_name=INTEGRATION_NAME, endpoint=None): +def integration_endpoint(request, endpoint=None): """Communicate with an integrated service""" - try: - integration = get_object_or_404(Integration, name=integration_name) - except: - return HttpResponseNotFound(f'

404 - Integration configuration missing. Create Integration database record.

') communication = set_integration() - identified = communication.identify() data = communication.get_response(endpoint) - # add results into Endpoint model - ep, created = Endpoint.objects.update_or_create( - integration=integration, - endpoint_path=endpoint, - data=data + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint ) + ep.data = data + ep.save() return HttpResponse( - f"Attempting to communicate with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data, indent=4)}
" + f"") -def integration_endpoint_post(request, integration_name=INTEGRATION_NAME, endpoint=None): +def integration_endpoint_post(request, endpoint=None): """Communicate with an integrated service using POST""" - try: - integration = get_object_or_404(Integration, name=integration_name) - except: - return HttpResponseNotFound( - f'

404 - Integration configuration missing. Create Integration database record.

') communication = set_integration() - identified = communication.identify() data = communication.post_response(endpoint) - # add results into Endpoint model - - ep, created = Endpoint.objects.update_or_create( - integration=integration, - endpoint_path=endpoint, - data=data + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint ) + ep.data = data + ep.save() return HttpResponse( - f"Attempting to communicate using POST with '{integration}' integration: {identified}. endpoint: {endpoint}.
Returned data: {data}") + f"

Attempting to communicate using POST with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

endpoint: {endpoint}.

" + f"

Returned data:

" + f"
{json.dumps(data, indent=4)}
" + f"") From 068967052215fb680ee3607e511f1337bf25521d Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Sat, 12 Mar 2022 10:56:14 -0600 Subject: [PATCH 008/115] feat(Integrations): Begin abstracting csam calls --- integrations/csam/views.py | 23 ++++++++++++++++++++- integrations/models.py | 42 +++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/integrations/csam/views.py b/integrations/csam/views.py index b5eec62c8..e116a00e9 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -63,4 +63,25 @@ def integration_endpoint_post(request, endpoint=None): f"

endpoint: {endpoint}.

" f"

Returned data:

" f"
{json.dumps(data,indent=4)}
" - f"") \ No newline at end of file + f"") + +def update_system_description(params={"src_obj_type": "system", "src_obj_id": 2}): + """Update System description in CSAM""" + + result = dict() + from controls.models import System + system_id = params['src_obj_id'] + system = System.objects.get(pk=system_id) + csam_system_id = system.info.get('csam_system_id', None) + print("10, ========== csam_system_id", csam_system_id) + if csam_system_id is not None: + new_description = "This is the new system description." + endpoint = f"/system/{csam_system_id}" + post_data = { + "description": new_description + } + communication = set_integration() + data = communication.post_response(endpoint, data=json.dumps(post_data)) + result = data + return result + diff --git a/integrations/models.py b/integrations/models.py index 9d25140a2..d6ffc350d 100644 --- a/integrations/models.py +++ b/integrations/models.py @@ -15,8 +15,10 @@ class Integration(BaseModel): - name = models.CharField(max_length=250, help_text="Endpoint name in lowercase", unique=False, blank=False, null=False) - description = models.TextField(default="", help_text="Brief description of the Integration", unique=False, blank=True, null=True) + name = models.CharField(max_length=250, help_text="Endpoint name in lowercase", unique=False, blank=False, + null=False) + description = models.TextField(default="", help_text="Brief description of the Integration", unique=False, + blank=True, null=True) config = JSONField(blank=True, null=True, default=dict, help_text="Integration configuration") config_schema = JSONField(blank=True, null=True, default=dict, help_text="Integration schema") @@ -41,22 +43,38 @@ def __repr__(self): # "system_id": 111 # } # } - # "map_data": [ - # { - # "local_value": "System.name", - # "integration_value": "system_info['name']" - # } - # ] - # } # ] # } + +# class ObjectMap(BaseModel): +# src_obj_type = models.CharField(max_length=100, unique=False, blank=False, null=False, +# help_text="GovReady object type") +# src_obj_id = models.IntegerField(max_length=100, unique=False, blank=False, null=False, +# help_text="GovReady object's ID/primary_key") +# integration = auto_prefetch.ForeignKey(Integration, related_name="object_maps", on_delete=models.CASCADE, +# unique=False, blank=False, null=False, +# help_text="The Integration") +# trg_obj_type = models.CharField(max_length=100, unique=False, blank=True, null=True, +# help_text="Integration object type") +# trg_obj_id = models.CharField(max_length=100, unique=False, blank=True, null=True, +# help_text="Integration object's ID/primary_key") +# +# def __str__(self): +# return f"'{self.src_obj_type} {self.src_obj_id} to {self.integration} {self.trg_obj_type} id={self.id}'" +# +# def __repr__(self): +# return f"'{self.src_obj_type} {self.src_obj_id} to {self.integration} {self.trg_obj_type} id={self.id}'" + + class Endpoint(auto_prefetch.Model, BaseModel): integration = auto_prefetch.ForeignKey(Integration, related_name="endpoints", on_delete=models.CASCADE, - help_text="Endpoint's Integration") - endpoint_path = models.CharField(max_length=250, help_text="Path to the Endpoint", unique=False, blank=True, null=True) - description = models.TextField(default="", help_text="Brief description of the endpoint", unique=False, blank=True, null=True) + help_text="Endpoint's Integration") + endpoint_path = models.CharField(max_length=250, help_text="Path to the Endpoint", unique=False, blank=True, + null=True) + description = models.TextField(default="", help_text="Brief description of the endpoint", unique=False, blank=True, + null=True) element_type = models.CharField(max_length=150, help_text="Component type", unique=False, blank=True, null=True) data = JSONField(blank=True, null=True, default=dict, help_text="JSON object representing the API results.") history = HistoricalRecords(cascade_delete_history=True) From 4ed9b007ade2e426f6d8b08b2dbe138d98e5690a Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 15 Mar 2022 17:04:30 -0500 Subject: [PATCH 009/115] Update csam mock.py --- integrations/csam/mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index 88446f826..125f42f02 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -62,7 +62,7 @@ def do_GET(self, method=None): data = {"message":"hello"} # Send the JSON response - self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + self.wfile.write(json.dumps(data, indent=4)) elif parsed_path.path == "/authenticate-test": """Test authentication""" From d15c88ca780cacd24304d219eb5da7bc2d5bc325 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Thu, 17 Mar 2022 16:15:43 -0500 Subject: [PATCH 010/115] Improve integrations --- integrations/csam/urls.py | 5 +++++ integrations/csam/views.py | 19 +++++++++++++++---- integrations/urls.py | 4 ---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/integrations/csam/urls.py b/integrations/csam/urls.py index e209e81b4..76a99d131 100644 --- a/integrations/csam/urls.py +++ b/integrations/csam/urls.py @@ -7,4 +7,9 @@ name='integration_endpoint'), # Ex: /integrations/csam/endpoint/system/111 url(r"^post_endpoint(?P.*)$", views.integration_endpoint_post, name='integration_endpoint'), + + url(r"^update_system_description_test/(?P.*)$", views.update_system_description_test, + name='integration_endpoint'), + url(r"^update_system_description$", views.update_system_description, + name='integration_endpoint'), ] diff --git a/integrations/csam/views.py b/integrations/csam/views.py index e116a00e9..a61bdf302 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -3,6 +3,7 @@ from django.http import HttpResponse, HttpResponseNotFound from integrations.models import Integration, Endpoint from .communicate import CSAMCommunication +from controls.models import System INTEGRATION_NAME = 'csam' try: @@ -65,15 +66,25 @@ def integration_endpoint_post(request, endpoint=None): f"
{json.dumps(data,indent=4)}
" f"") -def update_system_description(params={"src_obj_type": "system", "src_obj_id": 2}): +def update_system_description_test(request, system_id=2): + """Test updating system description in CSAM""" + + params={"src_obj_type": "system", "src_obj_id": system_id} + data = update_system_description(params) + return HttpResponse( + f"

Attempting to update CSAM description of System id {system_id}...' " + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def update_system_description(request, params={"src_obj_type": "system", "src_obj_id": 2}): """Update System description in CSAM""" - result = dict() - from controls.models import System system_id = params['src_obj_id'] system = System.objects.get(pk=system_id) + # TODO: Check user permission to update csam_system_id = system.info.get('csam_system_id', None) - print("10, ========== csam_system_id", csam_system_id) + # print("10, ========== csam_system_id", csam_system_id) if csam_system_id is not None: new_description = "This is the new system description." endpoint = f"/system/{csam_system_id}" diff --git a/integrations/urls.py b/integrations/urls.py index 8f3163ed2..51614838c 100644 --- a/integrations/urls.py +++ b/integrations/urls.py @@ -2,10 +2,6 @@ from . import views urlpatterns = [ - # url(r"^jsonplaceholder/users$", views.jsonplaceholders_users, name='jsonplaceholders_users'), - # url(r"^(?P.*)/identify$", views.integration_identify, name='integration_identify'), - # url(r"^(?P.*)/endpoint(?P.*)$", views.integration_endpoint, - # name='integration_endpoint'), # Ex: /integrations/csam/endpoint/system/111 url(r"^csam/", include("integrations.csam.urls")), url(r"^jsonplaceholder/", include("integrations.jsonplaceholder.urls")), ] \ No newline at end of file From af5a6b18a047f6626ff63d462bd24038e60e8a2d Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Thu, 31 Mar 2022 05:59:03 -0500 Subject: [PATCH 011/115] Improve CSAM integration: multiple get loop, better README. Various improvements to CSAM integration: - Refactor mock service for readability - add '/v1' prefix to endpoints in mock service - Enhance details in README for better documentation - Add view to get info on multiple systems in a loop to support batch update of cached information --- integrations/csam/README.md | 117 +++++++++++++++++++++++++--- integrations/csam/mock.py | 147 ++++++++++++++++-------------------- integrations/csam/urls.py | 8 +- integrations/csam/views.py | 66 +++++++++++++++- 4 files changed, 242 insertions(+), 96 deletions(-) diff --git a/integrations/csam/README.md b/integrations/csam/README.md index b2e80149f..988f9bfa9 100644 --- a/integrations/csam/README.md +++ b/integrations/csam/README.md @@ -1,29 +1,124 @@ # ABOUT CSAM Integreation -A simple Python webserver to generate mock CSAM API -results -Usage: - +## Configure +Create an Integration record in Django admin: + +- Name: csam +- Description: Integration to support CSAM version 4.10 +- Config: +```json +{ + "base_url": "http://csam-test.agency.gov/csam/api", + "personal_access_token": "" +} ``` -# Start mock service -python3 integrations/csam/mock.py +- Config schema: +```json +{} ``` +For local dev and testing, create an Integration record in Django admin for CSAM mock service: + +- Name: csam +- Description: Integration to support CSAM version 4.10 +- Config: +```json +{ + "base_url": "http://localhost:9002", + "personal_access_token": "FAD619", +} ``` -# Accessing mock service -curl localhost:9002/endpoint -curl localhost:9002/hello -curl localhost:9002/system/111 # requires authentication +- Config schema: +```json +{} ``` +## Details + +Data will be stored in endpoints record with the endpoint as the reference. + +TBD: How often is endpoint history clean up? Update everytime or only if information changes? +## [WIP] Field Mappings Notes +```python +system.description = get_object_or_404(Endpoint, integration=csam, endpoint_path=f'/system/{csam_system_id}').data['description'] +system.name = get_object_or_404(Endpoint, integration=csam, endpoint_path=f'/system/{csam_system_id}').data['name'] ``` +## Testing in Browser + +The following URLs will test the integration. Data will be returned from either the mock service or actual service based on integration configuration. + +- URL: http://localhost:8000/integrations/csam/identify +- Purpose: Identifies the integration. All integrations have this URL. +- Returns: +```text +Attempting to communicate with csam integration: This is csam version 0.1 +``` + +- URL: http://localhost:8000/integrations/csam/endpoint/system/111 +- Purpose: Get data from an endpoint +- Returns: +```text +Attempting to communicate with 'csam' integration: This is csam version 0.1 + +endpoint: /system/111 + +{ + "system_id": 111, + "name": "My IT System", + "description": "This is a simple test system" +} +``` + +## Pairing a System in GovReady-Q to system in CSAM + +To pair a system in GovReady-Q to a system in CSAM, add the following line to the GovReady-Q's `System.info` JSONfield replacing `` with the system's CSAM ID. + +```bash +{"csam_system_id": } +``` + +## [WIP] Updating Multiple Systems + +http://localhost:8000/integrations/csam/get_multiple_system_info +TODO: Pass in multiple systems parameters + +## Running the Mock Service (mock.py) + +The integration's mock service consists of a simple Python webserver to simulate CSAM's API. + +Start mock service in its own terminal window: + +```bash +docker exec -it govready-q-dev python3 integrations/csam/mock.py +``` + +In a separate terminal window, `exec` into the govready-q-dev container to interact via `curl` with mock service: + +```bash +docker exec -it govready-q-dev /bin/bash +``` + +The following `curl` commands can be run from within the govready-q-dev container to interact with the mock service: + +```bash +# Accessing mock service +curl localhost:9002/test/hello +curl localhost:9002/v1/system/111 # requires authentication + # Accessing mock service with authentication - curl -X 'GET' 'http://localhost:9002/system/111' \ + curl -X 'GET' 'http://localhost:9002/v1/system/111' \ -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ -H 'Authorization: Bearer FAD619' ``` +## Experimenting + +```python +# Fetch previously retrieved and stored endpoint data +get_object_or_404(Endpoint, integration=csam, endpoint_path=f"/v1/system/222").data["name"] + +``` diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index 125f42f02..c3c873254 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -12,7 +12,7 @@ # # Accessing: # curl localhost:9002/endpoint -# curl localhost:9002/hello +# curl -X 'GET' 'http://127.0.0.1:9002/system/111' # curl localhost:9002/system/111 # requires authentication # # Accessing with simple authentication: @@ -33,92 +33,88 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): - SYSTEM = { - "system_id": 111, - "name": "My IT System", - "description": "This is a simple test system" - } + # Mock data + SYSTEMS = {"111": {"system_id": 111, + "name": "SystemA", + "description": "This is a simple test system" + }, + "222": {"system_id": 222, + "name": "SystemB", + "description": "This is another simple test system" + } + } - def mk_csam_system_info_response(self): - return self.SYSTEM + def mk_csam_system_info_response(self, system_id): + return self.SYSTEMS[system_id] def do_GET(self, method=None): - + """Parse and route GET request""" # Parse path - parsed_path = urlparse(self.path) - params = parse_qs(parsed_path.query) - print("parsed_path.path:", parsed_path.path) - print("** params", params) + request = urlparse(self.path) + params = parse_qs(request.query) + print(f"request.path: {request.path}, params: {params}") # params are received as arrays, so get first element in array - system_id = params.get('system_id', [0])[0] + # system_id = params.get('system_id', [0])[0] - # Route and handle request - if parsed_path.path == "/hello": + # Route GET request + if request.path == "/test/hello": """Reply with 'hello'""" - self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() data = {"message":"hello"} + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - # Send the JSON response - self.wfile.write(json.dumps(data, indent=4)) - - elif parsed_path.path == "/authenticate-test": + elif request.path == "/test/authenticate-test": """Test authentication""" - # Test authentication by reading headers and looking for 'Authentication' self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - # Read headers - print("Authorization header:", self.headers['Authorization']) - - data = { - "reply": "yes", - "Authorization": self.headers['Authorization'], - "pat": self.headers['Authorization'].split("Bearer ")[-1] - } - # Send the JSON response + if 'Authorization' in self.headers: + print("Authorization header:", self.headers['Authorization']) + data = {"reply": "Success", + "Authorization": self.headers['Authorization'], + "pat": self.headers['Authorization'].split("Bearer ")[-1] + } + else: + data = {"reply": "Fail", + "Authorization": None, + "pat": None + } self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - elif parsed_path.path == "/system/111": + elif request.path == "/v1/system/111" or request.path == "/v1/system/222": """Reply with system information""" - - # Usage: - # # # authorized example - # curl -X 'GET' 'http://localhost:9002/system/111' \ + # curl -X 'GET' 'http://localhost:9002/v1/system/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ # -H 'Authorization: Bearer FAD619' # # # unauthorized example: - # curl -X 'GET' 'http://localhost:9002/system/111' \ + # curl -X 'GET' 'http://localhost:9002/v1/system/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' - # - pat = None if 'Authorization' in self.headers: pat = self.headers['Authorization'].split("Bearer ")[-1] - if pat is None or pat != "FAD619": - # Reply with unauthorized + # Authentication fails self.send_response(401) self.send_header('Content-Type', 'application/json') self.end_headers() - data = { - "message": "Unauthorized request", - "endpoint": parsed_path.path - } + data = {"message": "Unauthorized request", + "endpoint": request.path + } else: - # Request is authenticated + # Authentication succeeds self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - data = self.mk_csam_system_info_response() - - # Send the JSON response + if '111' in request.path: + data = self.mk_csam_system_info_response('111') + elif '222' in request.path: + data = self.mk_csam_system_info_response('222') self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) else: @@ -126,35 +122,28 @@ def do_GET(self, method=None): self.send_response(404) self.send_header('Content-Type', 'application/json') self.end_headers() - data = {"message":"Path not found"} # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) def do_POST(self): - - parsed_path = urlparse(self.path) - params = parse_qs(parsed_path.query) - print("parsed_path.path:", parsed_path.path) + """Parse and route POST request""" + request = urlparse(self.path) + params = parse_qs(request.query) + print("request.path:", request.path) print("** params", params) - # Route and handle request - if parsed_path.path == "/hello": + # Route POST request + if request.path == "/test/hello": """Reply with 'hello'""" - self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() data = {"message": "hello, POST"} - - # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - if parsed_path.path == "/system/111": + if request.path == "/v1/system/111" or request.path == "/v1/system/222": """Update system information""" - - # Usage: - # # # authorized example # curl -X 'POST' 'http://localhost:9002/system/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ @@ -163,46 +152,40 @@ def do_POST(self): # # unauthorized example: # curl -X 'POST' 'http://localhost:9002/system/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' - # - pat = None if 'Authorization' in self.headers: pat = self.headers['Authorization'].split("Bearer ")[-1] - if pat is None or pat != "FAD619": - # Reply with unauthorized + # Authorization failed self.send_response(401) self.send_header('Content-Type', 'application/json') self.end_headers() - data = { - "message": "Unauthorized request", - "endpoint": parsed_path.path - } + data = {"message": "Unauthorized request", + "endpoint": request.path + } else: - # Request is authenticated - read POST data and update system info + # Authorization succeeded - read POST data and update system info content_length = int(self.headers['Content-Length']) self.post_data = self.rfile.read(content_length) self.post_data_json = json.loads(self.post_data) - # post_data_decoded = post_data.decode('utf-8') - self.SYSTEM['name'] = self.post_data_json.get('name', self.SYSTEM['name']) - self.SYSTEM['description'] = self.post_data_json.get('description', self.SYSTEM['description']) - + if '111' in request.path: + system_id = '111' + elif '222' in request.path: + system_id = '222' + self.SYSTEMS[system_id]['name'] = self.post_data_json.get('name', self.SYSTEM['name']) + self.SYSTEM[system_id]['description'] = self.post_data_json.get('description', self.SYSTEM['description']) # Send response self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - - data = self.mk_csam_system_info_response() - # Send the JSON response + data = self.mk_csam_system_info_response(system_id) self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - else: - """Reply with Path not found""" + # Path not found self.send_response(404) self.send_header('Content-Type', 'application/json') self.end_headers() - data = {"message":"Path not found"} # Send the JSON response self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) diff --git a/integrations/csam/urls.py b/integrations/csam/urls.py index 76a99d131..d66a00292 100644 --- a/integrations/csam/urls.py +++ b/integrations/csam/urls.py @@ -8,8 +8,12 @@ url(r"^post_endpoint(?P.*)$", views.integration_endpoint_post, name='integration_endpoint'), + url(r"^get_system_info_test/(?P.*)$", views.get_system_info, name='get_system_info_test'), url(r"^update_system_description_test/(?P.*)$", views.update_system_description_test, - name='integration_endpoint'), + name='update_system_description_test'), url(r"^update_system_description$", views.update_system_description, - name='integration_endpoint'), + name='update_system_description'), + + url(r"^get_multiple_system_info$", views.get_multiple_system_info, name='get_multiple_system_info'), + ] diff --git a/integrations/csam/views.py b/integrations/csam/views.py index a61bdf302..d17a7a68d 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -1,4 +1,5 @@ import json +import time from django.shortcuts import get_object_or_404, redirect, render from django.http import HttpResponse, HttpResponseNotFound from integrations.models import Integration, Endpoint @@ -66,6 +67,69 @@ def integration_endpoint_post(request, endpoint=None): f"
{json.dumps(data,indent=4)}
" f"") +def get_system_info(request, system_id=2): + """Retrieve the system information from CSAM""" + + system = System.objects.get(pk=system_id) + # TODO: Check user permission to view + csam_system_id = system.info.get('csam_system_id', None) + if csam_system_id is None: + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

System '{system_id}' does not have an associated 'csam_system_id'.

" + f"") + + endpoint = f'/v1/system/{csam_system_id}' + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def get_multiple_system_info(request, system_id_list=[1,2]): + """Get and cach system info for multiple systems""" + systems_updated =[] + systems = System.objects.filter(pk__in=system_id_list) + for s in systems: + csam_system_id = s.info.get("csam_system_id", None) + if csam_system_id is None: + print(f"System id {s.id} has no csam_system_id") + else: + endpoint = f'/v1/system/{csam_system_id}' + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + msg = f"System id {s.id} info updated from csam system {csam_system_id}" + print(msg) + systems_updated.append(msg) + time.sleep(0.25) + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"get_multiple_system_info for system ids {system_id_list}

" + f"

Result:

" + f"
{systems_updated}
" + f"") + def update_system_description_test(request, system_id=2): """Test updating system description in CSAM""" @@ -87,7 +151,7 @@ def update_system_description(request, params={"src_obj_type": "system", "src_ob # print("10, ========== csam_system_id", csam_system_id) if csam_system_id is not None: new_description = "This is the new system description." - endpoint = f"/system/{csam_system_id}" + endpoint = f"/v1/system/{csam_system_id}" post_data = { "description": new_description } From 72c781f1904c8e105aca057293f35ac44aa0b626 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Thu, 31 Mar 2022 06:27:21 -0500 Subject: [PATCH 012/115] Pass system_id_list, dev views reurn date --- integrations/csam/urls.py | 2 +- integrations/csam/views.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/integrations/csam/urls.py b/integrations/csam/urls.py index d66a00292..c99cbdc2f 100644 --- a/integrations/csam/urls.py +++ b/integrations/csam/urls.py @@ -14,6 +14,6 @@ url(r"^update_system_description$", views.update_system_description, name='update_system_description'), - url(r"^get_multiple_system_info$", views.get_multiple_system_info, name='get_multiple_system_info'), + url(r"^get_multiple_system_info/(?P.*)$", views.get_multiple_system_info, name='get_multiple_system_info'), ] diff --git a/integrations/csam/views.py b/integrations/csam/views.py index d17a7a68d..2762adae1 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -1,5 +1,6 @@ import json import time +from datetime import datetime from django.shortcuts import get_object_or_404, redirect, render from django.http import HttpResponse, HttpResponseNotFound from integrations.models import Integration, Endpoint @@ -37,6 +38,7 @@ def integration_endpoint(request, endpoint=None): return HttpResponse( f"

Attempting to communicate with '{INTEGRATION_NAME}' " f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" f"

endpoint: {endpoint}

" f"

Returned data:

" f"
{json.dumps(data,indent=4)}
" @@ -62,6 +64,7 @@ def integration_endpoint_post(request, endpoint=None): return HttpResponse( f"

Attempting to communicate using POST with '{INTEGRATION_NAME}' " f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" f"

endpoint: {endpoint}.

" f"

Returned data:

" f"
{json.dumps(data,indent=4)}
" @@ -77,6 +80,7 @@ def get_system_info(request, system_id=2): return HttpResponse( f"

Attempting to communicate with '{INTEGRATION_NAME}' " f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" f"

System '{system_id}' does not have an associated 'csam_system_id'.

" f"") @@ -94,15 +98,16 @@ def get_system_info(request, system_id=2): return HttpResponse( f"

Attempting to communicate with '{INTEGRATION_NAME}' " f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" f"

endpoint: {endpoint}

" f"

Returned data:

" f"
{json.dumps(data,indent=4)}
" f"") -def get_multiple_system_info(request, system_id_list=[1,2]): +def get_multiple_system_info(request, system_id_list="1,2"): """Get and cach system info for multiple systems""" - systems_updated =[] - systems = System.objects.filter(pk__in=system_id_list) + systems_updated = [] + systems = System.objects.filter(pk__in=system_id_list.split(",")) for s in systems: csam_system_id = s.info.get("csam_system_id", None) if csam_system_id is None: @@ -126,6 +131,7 @@ def get_multiple_system_info(request, system_id_list=[1,2]): return HttpResponse( f"

Attempting to communicate with '{INTEGRATION_NAME}' " f"get_multiple_system_info for system ids {system_id_list}

" + f"

now: {datetime.now()}

" f"

Result:

" f"
{systems_updated}
" f"") @@ -137,6 +143,7 @@ def update_system_description_test(request, system_id=2): data = update_system_description(params) return HttpResponse( f"

Attempting to update CSAM description of System id {system_id}...' " + f"

now: {datetime.now()}

" f"

Returned data:

" f"
{json.dumps(data,indent=4)}
" f"") From ab0d8fe3df0eb10169067c28a1d1ac1c1653f557 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 1 Apr 2022 08:42:38 -0500 Subject: [PATCH 013/115] Improve CSAM integration templates --- integrations/csam/communicate.py | 2 +- integrations/csam/mock.py | 56 ++++++++++++++--- integrations/csam/templates/csam2/system.html | 1 + integrations/csam/urls.py | 2 + integrations/csam/views.py | 45 +++++++++++++ integrations/templates/csam/system.html | 63 +++++++++++++++++++ integrations/templatetags/integration_tags.py | 38 +++++++++++ 7 files changed, 199 insertions(+), 8 deletions(-) create mode 100644 integrations/csam/templates/csam2/system.html create mode 100644 integrations/templates/csam/system.html create mode 100644 integrations/templatetags/integration_tags.py diff --git a/integrations/csam/communicate.py b/integrations/csam/communicate.py index 28ebcaec4..d33ec1fda 100644 --- a/integrations/csam/communicate.py +++ b/integrations/csam/communicate.py @@ -12,7 +12,7 @@ class CSAMCommunication(Communication): DESCRIPTION = { "name": "csam", "description": "CSAM API Service", - "version": "0.1", + "version": "0.3", "integration_db_record": True, "mock": { "base_url": "http:/localhost:9002", diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index c3c873254..38112fb4d 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -34,14 +34,56 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): # Mock data - SYSTEMS = {"111": {"system_id": 111, - "name": "SystemA", - "description": "This is a simple test system" + SYSTEMS = { "111": {"id": 111, + "externalId": "string", + "name": "System A string", + "acronym": "string", + "organization": "string", + "subOrganization": "string", + "operationalStatus": "string", + "systemType": "string", + "financialSystem": "string", + "classification": "string", + "contractorSystem": True, + "fismaReportable": True, + "criticalInfrastructure": True, + "missionCritical": True, + "purpose": "string", + "ombExhibit": "string", + "uiiCode": "string", + "investmentName": "string", + "portfolio": "string", + "priorFyFunding": 0, + "currentFyFunding": 0, + "nextFyFunding": 0, + "categorization": "string", + "fundingImportStatus": "string" }, - "222": {"system_id": 222, - "name": "SystemB", - "description": "This is another simple test system" - } + "222": {"id": 222, + "name": "System B string", + "description": "This is another simple test system", + "acronym": "string", + "organization": "string", + "subOrganization": "string", + "operationalStatus": "string", + "systemType": "string", + "financialSystem": "string", + "classification": "string", + "contractorSystem": True, + "fismaReportable": True, + "criticalInfrastructure": True, + "missionCritical": True, + "purpose": "string", + "ombExhibit": "string", + "uiiCode": "string", + "investmentName": "string", + "portfolio": "string", + "priorFyFunding": 0, + "currentFyFunding": 0, + "nextFyFunding": 0, + "categorization": "string", + "fundingImportStatus": "string" + } } def mk_csam_system_info_response(self, system_id): diff --git a/integrations/csam/templates/csam2/system.html b/integrations/csam/templates/csam2/system.html new file mode 100644 index 000000000..1c48a5472 --- /dev/null +++ b/integrations/csam/templates/csam2/system.html @@ -0,0 +1 @@ +This is is integrations/csam/templates/system.html \ No newline at end of file diff --git a/integrations/csam/urls.py b/integrations/csam/urls.py index c99cbdc2f..c42ab0f01 100644 --- a/integrations/csam/urls.py +++ b/integrations/csam/urls.py @@ -15,5 +15,7 @@ name='update_system_description'), url(r"^get_multiple_system_info/(?P.*)$", views.get_multiple_system_info, name='get_multiple_system_info'), + + url(r"^system/(?P.*)$", views.system_info, name='csam_system_info'), ] diff --git a/integrations/csam/views.py b/integrations/csam/views.py index 2762adae1..1c15ffc2c 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -104,6 +104,50 @@ def get_system_info(request, system_id=2): f"
{json.dumps(data,indent=4)}
" f"") +def system_info(request, system_id=2): + """Retrieve the system information from CSAM""" + + system = System.objects.get(pk=system_id) + # TODO: Check user permission to view + csam_system_id = system.info.get('csam_system_id', None) + if csam_system_id is None: + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

System '{system_id}' does not have an associated 'csam_system_id'.

" + f"") + + communication = set_integration() + endpoint = f'/v1/system/{csam_system_id}' + + # is there local information? + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + if created: + # Cache not available + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep.data = data + ep.save() + else: + # Cache available + cached = True + pass + + context = { + "system": system, + "cached": True, + "communication": communication, + "ep": ep + } + from siteapp import settings + # settings.TEMPLATES[0]['DIRS'].append('/Users/gregelinadmin/Documents/workspace/govready-q-private/integrations/csam/templates/') + # print(2,"========= TEMPLATES", settings.TEMPLATES[0]['DIRS']) + return render(request, "csam/system.html", context) + def get_multiple_system_info(request, system_id_list="1,2"): """Get and cach system info for multiple systems""" systems_updated = [] @@ -167,3 +211,4 @@ def update_system_description(request, params={"src_obj_type": "system", "src_ob result = data return result + diff --git a/integrations/templates/csam/system.html b/integrations/templates/csam/system.html new file mode 100644 index 000000000..1ecb21dd7 --- /dev/null +++ b/integrations/templates/csam/system.html @@ -0,0 +1,63 @@ +{% extends "project-base.html" %} +{% load humanize %} +{% load guardian_tags %} +{% load static %} +{% load q %} +{% load integration_tags %} + +{% block title %} + {{title}} +{% endblock %} + +{% block head %} +{{block.super}} + +{% endblock %} + +{% block body_content %} + +
+
+ +

CSAM

+ +

The system "{{ system.root_element.name }}" (id: {{ system.id }}) is paired with CSAM system "{{ ep.data.name }}" (id: {{ ep.data.id }})

+ +

endpoint: {{ ep.endpoint_path }}

+ + {% if cached %} +

cached: {{ ep.updated|naturaltime|capfirst }}

+ {% else %} +

Returned data:

+ {% endif %} + + + + + + {% for key, value in ep.data.items %} + + {% endfor %} + + + + +{% endblock %} + +{% block modals %} +{% endblock %} + +{% block scripts %} + {{ block.super }} + +{% endblock %} + diff --git a/integrations/templatetags/integration_tags.py b/integrations/templatetags/integration_tags.py new file mode 100644 index 000000000..eefdf1dfb --- /dev/null +++ b/integrations/templatetags/integration_tags.py @@ -0,0 +1,38 @@ +from django import template +from django.template.defaultfilters import stringfilter +from django.utils.safestring import mark_safe + +register = template.Library() + +@register.filter(is_safe=True) +def json(value): + # Encode value as JSON for inclusion within a tag. + # Since we are not using |escapejs (which would only be valid within + # strings), we must instead ensure that the literal "" doesn't + # occur within the JSON content since that would break out of the script + # tag. This could occur both in string values and in the keys of objects. + # Since < and > can only occur within strings (i.e. they're not valid + # characters otherwise), we can JSON-escape them after serialization. + import json + value = json.dumps(value, sort_keys=True) + value = value.replace("<", r'\u003c') + value = value.replace(">", r'\u003e') # not necessary but for good measure + value = value.replace("&", r'\u0026') # not necessary but for good measure + return mark_safe(value) # nosec + +@register.filter(is_safe=True) +def get_item(dictionary, key): + return dictionary.get(key, None) + +@register.filter +def divide(value, arg): + if value is None: + value = 0 + try: + return float(value) / float(arg) + except (ValueError, ZeroDivisionError): + return None + +@register.filter +def multiply(value, arg): + return float(value) * float(arg) \ No newline at end of file From 6a19431cf30da9a58c01c8acc8b1005112b5dacf Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 1 Apr 2022 11:37:35 -0500 Subject: [PATCH 014/115] Improve CSAM integration templates --- integrations/csam/mock.py | 6 ++++-- integrations/templates/csam/system.html | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index 38112fb4d..ec0c90a68 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -36,7 +36,8 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): # Mock data SYSTEMS = { "111": {"id": 111, "externalId": "string", - "name": "System A string", + "name": "System A", + "description": "This is a simple test system", "acronym": "string", "organization": "string", "subOrganization": "string", @@ -60,7 +61,8 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): "fundingImportStatus": "string" }, "222": {"id": 222, - "name": "System B string", + "externalId": "string", + "name": "System B", "description": "This is another simple test system", "acronym": "string", "organization": "string", diff --git a/integrations/templates/csam/system.html b/integrations/templates/csam/system.html index 1ecb21dd7..278ccad15 100644 --- a/integrations/templates/csam/system.html +++ b/integrations/templates/csam/system.html @@ -26,7 +26,7 @@
-

CSAM

+

{{ system.root_element.name }} <==> CSAM {{ ep.data.name }}

The system "{{ system.root_element.name }}" (id: {{ system.id }}) is paired with CSAM system "{{ ep.data.name }}" (id: {{ ep.data.id }})

From 7c7609c87a660d1e90dbd7c74897b2b1168bb826 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 1 Apr 2022 13:42:28 -0500 Subject: [PATCH 015/115] Fix reference order to communicate in csam/views --- integrations/csam/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integrations/csam/views.py b/integrations/csam/views.py index 1c15ffc2c..aaf89fca6 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -109,6 +109,9 @@ def system_info(request, system_id=2): system = System.objects.get(pk=system_id) # TODO: Check user permission to view + + communication = set_integration() + endpoint = f'/v1/system/{csam_system_id}' csam_system_id = system.info.get('csam_system_id', None) if csam_system_id is None: return HttpResponse( @@ -118,9 +121,6 @@ def system_info(request, system_id=2): f"

System '{system_id}' does not have an associated 'csam_system_id'.

" f"") - communication = set_integration() - endpoint = f'/v1/system/{csam_system_id}' - # is there local information? ep, created = Endpoint.objects.get_or_create( integration=INTEGRATION, From 67f98f56d5396cce345500431f9742ce198d97e1 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 1 Apr 2022 13:55:57 -0500 Subject: [PATCH 016/115] Fix reference order to communicate in csam/views --- integrations/csam/views.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/integrations/csam/views.py b/integrations/csam/views.py index aaf89fca6..4ec089b2f 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -107,11 +107,19 @@ def get_system_info(request, system_id=2): def system_info(request, system_id=2): """Retrieve the system information from CSAM""" - system = System.objects.get(pk=system_id) - # TODO: Check user permission to view + system = get_object_or_404(System, pk=system_id) + try: + # system = System.objects.get(pk=system_id) + system = get_object_or_404(System, pk=system_id) + except: + return HttpResponse( + f"" + f"

now: {datetime.now()}

" + f"

System '{system_id}' does not exist.

" + f"") + # TODO: Check user permission to view communication = set_integration() - endpoint = f'/v1/system/{csam_system_id}' csam_system_id = system.info.get('csam_system_id', None) if csam_system_id is None: return HttpResponse( @@ -121,6 +129,7 @@ def system_info(request, system_id=2): f"

System '{system_id}' does not have an associated 'csam_system_id'.

" f"") + endpoint = f'/v1/system/{csam_system_id}' # is there local information? ep, created = Endpoint.objects.get_or_create( integration=INTEGRATION, From b6e77a021c72e81da3d1e9a5c8065dfa77339790 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 1 Apr 2022 14:23:56 -0500 Subject: [PATCH 017/115] Fix CSAM integration path to `systems` --- integrations/csam/mock.py | 8 ++++---- integrations/csam/views.py | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/integrations/csam/mock.py b/integrations/csam/mock.py index ec0c90a68..838f381fc 100644 --- a/integrations/csam/mock.py +++ b/integrations/csam/mock.py @@ -129,15 +129,15 @@ def do_GET(self, method=None): } self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - elif request.path == "/v1/system/111" or request.path == "/v1/system/222": + elif request.path == "/v1/systems/111" or request.path == "/v1/systems/222": """Reply with system information""" # # authorized example - # curl -X 'GET' 'http://localhost:9002/v1/system/111' \ + # curl -X 'GET' 'http://localhost:9002/v1/systems/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' \ # -H 'Authorization: Bearer FAD619' # # # unauthorized example: - # curl -X 'GET' 'http://localhost:9002/v1/system/111' \ + # curl -X 'GET' 'http://localhost:9002/v1/systems/111' \ # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' pat = None if 'Authorization' in self.headers: @@ -186,7 +186,7 @@ def do_POST(self): data = {"message": "hello, POST"} self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) - if request.path == "/v1/system/111" or request.path == "/v1/system/222": + if request.path == "/v1/systems/111" or request.path == "/v1/systems/222": """Update system information""" # # authorized example # curl -X 'POST' 'http://localhost:9002/system/111' \ diff --git a/integrations/csam/views.py b/integrations/csam/views.py index 4ec089b2f..8858c7648 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -73,7 +73,8 @@ def integration_endpoint_post(request, endpoint=None): def get_system_info(request, system_id=2): """Retrieve the system information from CSAM""" - system = System.objects.get(pk=system_id) + # system = System.objects.get(pk=system_id) + system = get_object_or_404(System, pk=system_id) # TODO: Check user permission to view csam_system_id = system.info.get('csam_system_id', None) if csam_system_id is None: @@ -84,7 +85,7 @@ def get_system_info(request, system_id=2): f"

System '{system_id}' does not have an associated 'csam_system_id'.

" f"") - endpoint = f'/v1/system/{csam_system_id}' + endpoint = f'/v1/systems/{csam_system_id}' communication = set_integration() data = communication.get_response(endpoint) # Cache remote data locally in database @@ -129,12 +130,13 @@ def system_info(request, system_id=2): f"

System '{system_id}' does not have an associated 'csam_system_id'.

" f"") - endpoint = f'/v1/system/{csam_system_id}' + endpoint = f'/v1/systems/{csam_system_id}' # is there local information? ep, created = Endpoint.objects.get_or_create( integration=INTEGRATION, endpoint_path=endpoint ) + # TODO: Refresh data if empty if created: # Cache not available data = communication.get_response(endpoint) @@ -166,7 +168,7 @@ def get_multiple_system_info(request, system_id_list="1,2"): if csam_system_id is None: print(f"System id {s.id} has no csam_system_id") else: - endpoint = f'/v1/system/{csam_system_id}' + endpoint = f'/v1/systems/{csam_system_id}' communication = set_integration() data = communication.get_response(endpoint) # Cache remote data locally in database @@ -211,7 +213,7 @@ def update_system_description(request, params={"src_obj_type": "system", "src_ob # print("10, ========== csam_system_id", csam_system_id) if csam_system_id is not None: new_description = "This is the new system description." - endpoint = f"/v1/system/{csam_system_id}" + endpoint = f"/v1/systems/{csam_system_id}" post_data = { "description": new_description } From f2a81dfa286b3c353b66f355a929dcdc1a0db88e Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Mon, 11 Apr 2022 10:37:34 -0500 Subject: [PATCH 018/115] Merge a second component statements to statements of first component method --- CHANGELOG.md | 3 ++- controls/models.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 912738160..664039716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,8 @@ v0.9.14-dev (March xx, 2022) * Added ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer for component (element) permissions. * Added ElementWithPermissionsViewSet for component (element) permissions. * Added more permission functions to element model: assigning a user specific permissions, removing all permissions from a user, and checking if a user is an owner of the element -* Updated User model to include search by 'username' and exclusion functionality to queryset +* Updated User model to include search by 'username' and exclusion functionality to queryset. +* Add to controls.models.Element.merge_component_implementation_statements method to Merge a second component statements to statements of first component. **Bug fixes** diff --git a/controls/models.py b/controls/models.py index f294170d7..d59d319d8 100644 --- a/controls/models.py +++ b/controls/models.py @@ -538,6 +538,35 @@ def copy(self, name=None): smt_copy.save() return e_copy + @transaction.atomic + def merge_component_statements(self, mrg_cmpt, strategy="KEEP_BOTH"): + """Merge a second component statements to statements of first component""" + + # print(f"strategy: {strategy}") + for smt_new in mrg_cmpt.statements(StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name): + src_cmpt_match_smts = self.statements_produced.filter(sid_class=smt_new.sid_class, sid=smt_new.sid, pid=smt_new.pid, statement_type=smt_new.statement_type) + + if len(src_cmpt_match_smts) == 0 or (len(src_cmpt_match_smts) == 1 and strategy == "KEEP_BOTH") or (len(src_cmpt_match_smts) > 1): + # Append smt if no matching statements, multiple matching statements, or 1 matching statement and strategy keep both + instance = deepcopy(smt_new) + instance.producer_element = self + instance.id = instance.created = instance.update = instance.import_record = None + instance.uuid = uuid.uuid4() + instance.save() + # print(f"Duplicate smt added {smt_new.sid}") + elif len(src_cmpt_match_smts) == 1 and strategy == "REPLACE": + # replace text + src_cmpt_smt = src_cmpt_match_smts[0] + # print(f"{src_cmpt_smt} src_cmpt_smt.body: {src_cmpt_smt.body}") + # print(f"smt_new.body: {smt_new.body}") + src_cmpt_smt.body = smt_new.body + # print(f"src_cmpt_smt: {src_cmpt_smt}") + src_cmpt_smt.save() + # print(f"Replaced smt body {smt_new.sid}") + else: + # print(f"Ignored smt {smt_new.sid}") + pass + @property def selected_controls_oscal_ctl_ids(self): """Return array of selected controls oscal ids""" From af23577e24e8366a54b663a5d0febc9bdf114186 Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Tue, 12 Apr 2022 11:27:48 -0500 Subject: [PATCH 019/115] wip request endpoint --- api/siteapp/serializers/request.py | 13 +++++++++++++ api/siteapp/urls.py | 4 ++-- api/siteapp/views/request.py | 15 +++++++++++++++ siteapp/models.py | 12 ++++++++++-- templates/components/element_form.html | 1 + 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 api/siteapp/serializers/request.py create mode 100644 api/siteapp/views/request.py diff --git a/api/siteapp/serializers/request.py b/api/siteapp/serializers/request.py new file mode 100644 index 000000000..b51db6eb2 --- /dev/null +++ b/api/siteapp/serializers/request.py @@ -0,0 +1,13 @@ +from api.base.serializers.types import ReadOnlySerializer, WriteOnlySerializer +from siteapp.models import Request + + +class SimpleRequestSerializer(ReadOnlySerializer): + class Meta: + model = Request + fields = ['user', 'system', 'element', 'req_comment', 'req_reject_comment', 'status', 'created'] + +class WriteRequestSerializer(WriteOnlySerializer): + class Meta: + model = Request + fields = ['user', 'system', 'element', 'req_comment', 'req_reject_comment', 'status'] diff --git a/api/siteapp/urls.py b/api/siteapp/urls.py index f4d9775ab..0ec8ff8bc 100644 --- a/api/siteapp/urls.py +++ b/api/siteapp/urls.py @@ -12,7 +12,7 @@ from api.siteapp.views.role import RoleViewSet from api.siteapp.views.party import PartyViewSet from api.siteapp.views.appointment import AppointmentViewSet - +from api.siteapp.views.request import RequestViewSet router = routers.DefaultRouter() router.register(r'organizations', OrganizationViewSet) @@ -25,7 +25,7 @@ router.register(r'roles', RoleViewSet) router.register(r'parties', PartyViewSet) router.register(r'appointments', AppointmentViewSet) - +router.register(r'requests', RequestViewSet) project_router = NestedSimpleRouter(router, r'projects', lookup='projects') diff --git a/api/siteapp/views/request.py b/api/siteapp/views/request.py new file mode 100644 index 000000000..480f24238 --- /dev/null +++ b/api/siteapp/views/request.py @@ -0,0 +1,15 @@ +from api.base.views.base import SerializerClasses +from api.base.views.viewsets import ReadWriteViewSet +from api.siteapp.serializers.request import SimpleRequestSerializer, WriteRequestSerializer +from siteapp.models import Request +# from api.siteapp.filters.requests import RequestFilter + +class RequestViewSet(ReadWriteViewSet): + queryset = Request.objects.all() + + serializer_classes = SerializerClasses(retrieve=SimpleRequestSerializer, + list=SimpleRequestSerializer, + create=WriteRequestSerializer, + update=WriteRequestSerializer, + destroy=WriteRequestSerializer) + # filter_class = RequestFilter \ No newline at end of file diff --git a/siteapp/models.py b/siteapp/models.py index 29670459a..5aa9c4104 100644 --- a/siteapp/models.py +++ b/siteapp/models.py @@ -1,6 +1,7 @@ from collections import ChainMap from itertools import chain import logging +from platform import system import structlog import uuid as uuid import auto_prefetch @@ -25,7 +26,7 @@ from siteapp.model_mixins.tags import TagModelMixin from siteapp.enums.assets import AssetTypeEnum from siteapp.utils.uploads import hash_file -from controls.models import ImportRecord +from controls.models import ImportRecord, System, Element from controls.utilities import * logging.basicConfig() @@ -1461,7 +1462,14 @@ def __repr__(self): def __str__(self): return f"{self.model_name} {self.role.title} - {self.party.name}" - +class Request(BaseModel): + user = models.ForeignKey(User, blank=True, null=True, related_name="request", on_delete=models.SET_NULL, + help_text="User creating the request.") + system = models.ForeignKey(System, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) + element = models.ForeignKey(Element, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) + req_comment = models.TextField(blank=True, null=True, help_text="Comments on this request.") + req_reject_comment = models.TextField(blank=True, null=True, help_text="Comment on request rejection.") + status = models.TextField(blank=True, null=True, help_text="Status of the request.") class Asset(BaseModel): UPLOAD_TO = None # Should be overriden when iheritted title = models.CharField(max_length=255, help_text="The title of this asset.") diff --git a/templates/components/element_form.html b/templates/components/element_form.html index b0a6d5fb9..2c81e9916 100644 --- a/templates/components/element_form.html +++ b/templates/components/element_form.html @@ -37,6 +37,7 @@

New Component (aka Element)

var slug = $(this).val(); slug = slug.toLowerCase().replace(/[^a-z0-9--]+/g, "-").replace(/^-+/, "").replace(/-+$/, ""); $('#id_slug').val(slug); + debugger; }) {% if request.method == "POST" %} From 90f722999e81c85d464b2d9ff1cd00d60e5db24d Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Thu, 14 Apr 2022 13:43:33 -0500 Subject: [PATCH 020/115] wip --- api/siteapp/serializers/request.py | 4 ++-- controls/models.py | 3 ++- controls/views.py | 4 ++++ frontend/src/components/requests/component.js | 17 +++++++++++++ frontend/src/components/requests/requests.js | 0 siteapp/model_mixins/requests.py | 24 +++++++++++++++++++ siteapp/models.py | 6 ++--- templates/components/element_detail_tabs.html | 17 ++----------- 8 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 frontend/src/components/requests/component.js create mode 100644 frontend/src/components/requests/requests.js create mode 100644 siteapp/model_mixins/requests.py diff --git a/api/siteapp/serializers/request.py b/api/siteapp/serializers/request.py index b51db6eb2..236750a99 100644 --- a/api/siteapp/serializers/request.py +++ b/api/siteapp/serializers/request.py @@ -5,9 +5,9 @@ class SimpleRequestSerializer(ReadOnlySerializer): class Meta: model = Request - fields = ['user', 'system', 'element', 'req_comment', 'req_reject_comment', 'status', 'created'] + fields = ['user', 'system', 'element', 'criteria_comment', 'criteria_reject_comment', 'status', 'created'] class WriteRequestSerializer(WriteOnlySerializer): class Meta: model = Request - fields = ['user', 'system', 'element', 'req_comment', 'req_reject_comment', 'status'] + fields = ['user', 'system', 'element', 'criteria_comment', 'criteria_reject_comment', 'status'] diff --git a/controls/models.py b/controls/models.py index 2ec53324a..9d88e7fad 100644 --- a/controls/models.py +++ b/controls/models.py @@ -17,6 +17,7 @@ from controls.enums.components import ComponentTypeEnum, ComponentStateEnum from siteapp.model_mixins.tags import TagModelMixin from siteapp.model_mixins.appointments import AppointmentModelMixin +from siteapp.model_mixins.requests import RequestsModelMixin from controls.enums.statements import StatementTypeEnum from controls.enums.remotes import RemoteTypeEnum from controls.oscal import Catalogs, Catalog, CatalogData @@ -261,7 +262,7 @@ class StatementRemote(auto_prefetch.Model): unique=False, blank=True, null=True, help_text="The Import Record which created this record.") -class Element(auto_prefetch.Model, TagModelMixin, AppointmentModelMixin): +class Element(auto_prefetch.Model, TagModelMixin, AppointmentModelMixin, RequestsModelMixin): name = models.CharField(max_length=250, help_text="Common name or acronym of the element", unique=True, blank=False, null=False) full_name =models.CharField(max_length=250, help_text="Full name of the element", unique=False, blank=True, null=True) description = models.TextField(default="Description needed", help_text="Description of the Element", unique=False, blank=False, null=False) diff --git a/controls/views.py b/controls/views.py index 3818ec075..2ae11f1b9 100644 --- a/controls/views.py +++ b/controls/views.py @@ -1360,6 +1360,9 @@ def component_library_component(request, element_id): listOfContacts.append(user) get_all_parties = element.appointments.all() + # import ipdb; ipdb.set_trace() + #TODO: Count element's requests - element.requests.count() + total_number_of_requests = 3 contacts = [] for poc in get_all_parties: @@ -1401,6 +1404,7 @@ def get_item(dictionary, key): "criteria": criteria_text, "listOfContacts": listOfContacts, "contacts": serializers.serialize('json', contacts), + "requestsTotal": total_number_of_requests, "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol, "form_source": "component_library" } diff --git a/frontend/src/components/requests/component.js b/frontend/src/components/requests/component.js new file mode 100644 index 000000000..26c6eadf7 --- /dev/null +++ b/frontend/src/components/requests/component.js @@ -0,0 +1,17 @@ +import React, {useState} from 'react'; +import ReactDOM from 'react-dom'; +import { PointOfContacts } from './point_of_contacts'; +import { Provider } from "react-redux"; +import store from "../../store"; + +window.pocTable = ( elementId, is_owner ) => { + $(window).on('load', function () { + $("#content").show(); + ReactDOM.render( + + + , + document.getElementById('poc-table') + ); + }); +}; \ No newline at end of file diff --git a/frontend/src/components/requests/requests.js b/frontend/src/components/requests/requests.js new file mode 100644 index 000000000..e69de29bb diff --git a/siteapp/model_mixins/requests.py b/siteapp/model_mixins/requests.py new file mode 100644 index 000000000..ff8d737b8 --- /dev/null +++ b/siteapp/model_mixins/requests.py @@ -0,0 +1,24 @@ +from django.db import models +from django.http import JsonResponse + +class RequestsModelMixin(models.Model): + requests = models.ManyToManyField("siteapp.Request", related_name="%(class)s") + + class Meta: + abstract = True + + def add_requests(self, requests): + if requests is None: + requests = [] + elif isinstance(requests, str): + requests = [requests] + assert isinstance(requests, list) + self.requests.add(*requests) + + def remove_requests(self, requests=None): + if requests is None: + requests = [] + elif isinstance(requests, str): + requests = [requests] + assert isinstance(requests, list) + self.requests.remove(*requests) \ No newline at end of file diff --git a/siteapp/models.py b/siteapp/models.py index 5aa9c4104..640881625 100644 --- a/siteapp/models.py +++ b/siteapp/models.py @@ -1466,9 +1466,9 @@ class Request(BaseModel): user = models.ForeignKey(User, blank=True, null=True, related_name="request", on_delete=models.SET_NULL, help_text="User creating the request.") system = models.ForeignKey(System, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) - element = models.ForeignKey(Element, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) - req_comment = models.TextField(blank=True, null=True, help_text="Comments on this request.") - req_reject_comment = models.TextField(blank=True, null=True, help_text="Comment on request rejection.") + requested_element = models.ForeignKey(Element, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) + criteria_comment = models.TextField(blank=True, null=True, help_text="Comments on this request.") + criteria_reject_comment = models.TextField(blank=True, null=True, help_text="Comment on request rejection.") status = models.TextField(blank=True, null=True, help_text="Status of the request.") class Asset(BaseModel): UPLOAD_TO = None # Should be overriden when iheritted diff --git a/templates/components/element_detail_tabs.html b/templates/components/element_detail_tabs.html index 3978bf294..1461cc6ac 100644 --- a/templates/components/element_detail_tabs.html +++ b/templates/components/element_detail_tabs.html @@ -399,22 +399,9 @@

TBD

-

Requested Usage

- {% comment %}
{{users_can_edit}}
{% endcomment %} - -

Approved Usage ({{users_with_permission | length}})

-
{{users_with_permission}}
-
-
-
Requested By
-
+

Requested Usage({{requestsTotal}})

- {% for user in users_with_permission %} -
-

{{ user.username }}

-
- {% endfor%} -
+
From 8c64fce592ad92cbc9406917ade4ae7a5ae7ef63 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 06:54:30 -0500 Subject: [PATCH 021/115] Debugging oidc --- siteapp/authentication/OIDCAuthentication.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 629b18b2f..273c6a883 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -15,6 +15,23 @@ class OIDCAuth(OIDCAuthenticationBackend): + # override get_user method to debug token + def get_userinfo(self, access_token, id_token, payload): + """Return user details dictionary. The id_token and payload are not used in + the default implementation, but may be used when overriding this method""" + + user_response = requests.get( + self.OIDC_OP_USER_ENDPOINT, + headers={ + 'Authorization': 'Bearer {0}'.format(access_token) + }, + verify=self.get_settings('OIDC_VERIFY_SSL', True), + timeout=self.get_settings('OIDC_TIMEOUT', None), + proxies=self.get_settings('OIDC_PROXY', None)) + user_response.raise_for_status() + LOGGER.warning(f"user info, {type(user_response.text)}, {user_response.text}") + return user_response.json() + def is_admin(self, groups): if settings.OIDC_ROLES_MAP["admin"] in groups: return True From 434fe97494764ae169b9d54ec027dfb5d3dc88d8 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 08:59:58 -0500 Subject: [PATCH 022/115] Debugging oidc 2 --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 273c6a883..bfef81fee 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -19,7 +19,7 @@ class OIDCAuth(OIDCAuthenticationBackend): def get_userinfo(self, access_token, id_token, payload): """Return user details dictionary. The id_token and payload are not used in the default implementation, but may be used when overriding this method""" - + import requests user_response = requests.get( self.OIDC_OP_USER_ENDPOINT, headers={ From 27f9fa7021d105748dbafe6c5a24e37f690ecb10 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 13:02:04 -0500 Subject: [PATCH 023/115] Debugging OIDC response 3 --- siteapp/authentication/OIDCAuthentication.py | 23 +++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index bfef81fee..7c6d569d4 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -1,5 +1,6 @@ import time from urllib.parse import urlencode +import json from django.conf import settings from django.core.exceptions import SuspiciousOperation @@ -9,10 +10,12 @@ from mozilla_django_oidc.auth import OIDCAuthenticationBackend, LOGGER from mozilla_django_oidc.middleware import SessionRefresh from mozilla_django_oidc.utils import absolutify, add_state_and_nonce_to_session +from base64 import urlsafe_b64encode, urlsafe_b64decode from siteapp.models import Portfolio + class OIDCAuth(OIDCAuthenticationBackend): # override get_user method to debug token @@ -30,7 +33,25 @@ def get_userinfo(self, access_token, id_token, payload): proxies=self.get_settings('OIDC_PROXY', None)) user_response.raise_for_status() LOGGER.warning(f"user info, {type(user_response.text)}, {user_response.text}") - return user_response.json() + # split on ".": Header.Payload.Signature + header, payload, signature = [parse_b64url(content) for content in user_response.text.split(".")] + header = json.loads(header.decode('UTF-8')) + payload = parse_b64url(payload)[:-1] if b'\x1b' in parse_b64url(payload) else parse_b64url(payload) + payload = json.loads(payload.decode('UTF-8)')) + LOGGER.warning(f"header: {header}, \npayload: {payload}, \nsignature: {signature}") + #return user_response.json() + return payload + + def parse_b64url(content): + """Return decoded base64url content""" + + try: + decoded = urlsafe_b64decode(content+str(b'=======')) + except: + decoded = urlsafe_b64decode(content) + return decoded + + def is_admin(self, groups): if settings.OIDC_ROLES_MAP["admin"] in groups: From 259f821adba1b9aa5ca4bbcd6231053e879b5d8b Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 13:13:09 -0500 Subject: [PATCH 024/115] Debugging OIDC response 4 --- siteapp/authentication/OIDCAuthentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 7c6d569d4..b4dcf84b3 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -34,9 +34,9 @@ def get_userinfo(self, access_token, id_token, payload): user_response.raise_for_status() LOGGER.warning(f"user info, {type(user_response.text)}, {user_response.text}") # split on ".": Header.Payload.Signature - header, payload, signature = [parse_b64url(content) for content in user_response.text.split(".")] + header, payload, signature = [self.parse_b64url(content) for content in user_response.text.split(".")] header = json.loads(header.decode('UTF-8')) - payload = parse_b64url(payload)[:-1] if b'\x1b' in parse_b64url(payload) else parse_b64url(payload) + payload = payload[:-1] if b'\x1b' in payload else payload payload = json.loads(payload.decode('UTF-8)')) LOGGER.warning(f"header: {header}, \npayload: {payload}, \nsignature: {signature}") #return user_response.json() From 12419b3660277ec964bb106f0ec05a75c42d87ac Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 13:38:46 -0500 Subject: [PATCH 025/115] Debugging OIDC response 5 --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index b4dcf84b3..d045aa92e 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -42,7 +42,7 @@ def get_userinfo(self, access_token, id_token, payload): #return user_response.json() return payload - def parse_b64url(content): + def parse_b64url(self, content): """Return decoded base64url content""" try: From 3ad2e857eacd118cd79e83df41faedd0a470fcbc Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 14:27:03 -0500 Subject: [PATCH 026/115] Debugging OIDC response 6 --- siteapp/authentication/OIDCAuthentication.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index d045aa92e..c580fe31d 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -59,11 +59,17 @@ def is_admin(self, groups): return False def create_user(self, claims): - data = {'email': claims[settings.OIDC_CLAIMS_MAP['email']], - 'first_name': claims[settings.OIDC_CLAIMS_MAP['first_name']], - 'last_name': claims[settings.OIDC_CLAIMS_MAP['last_name']], - 'username': claims[settings.OIDC_CLAIMS_MAP['username']], - 'is_staff': self.is_admin(claims[settings.OIDC_CLAIMS_MAP['groups']])} + # data = {'email': claims[settings.OIDC_CLAIMS_MAP['email']], + # 'first_name': claims[settings.OIDC_CLAIMS_MAP['first_name']], + # 'last_name': claims[settings.OIDC_CLAIMS_MAP['last_name']], + # 'username': claims[settings.OIDC_CLAIMS_MAP['username']], + # 'is_staff': self.is_admin(claims[settings.OIDC_CLAIMS_MAP['groups']])} + + data = {'email': claims.get(settings.OIDC_CLAIMS_MAP['email'], None), + 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['first_name'], None), + 'last_name': claims.get(settings.OIDC_CLAIMS_MAP['last_name'], None), + 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], None), + 'is_staff': self.is_admin(claims.get(settings.OIDC_CLAIMS_MAP['groups'], None))} user = self.UserModel.objects.create_user(**data) portfolio = Portfolio.objects.create(title=user.email.split('@')[0], description="Personal Portfolio") From 59c935d5c4449b5390117fbf55e7e034738d3b92 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 15:10:34 -0500 Subject: [PATCH 027/115] Debugging OIDC response 7 --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index c580fe31d..3bce951d8 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -69,7 +69,7 @@ def create_user(self, claims): 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['first_name'], None), 'last_name': claims.get(settings.OIDC_CLAIMS_MAP['last_name'], None), 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], None), - 'is_staff': self.is_admin(claims.get(settings.OIDC_CLAIMS_MAP['groups'], None))} + 'is_staff': False} user = self.UserModel.objects.create_user(**data) portfolio = Portfolio.objects.create(title=user.email.split('@')[0], description="Personal Portfolio") From 188897a6323251176e3fcc4d7417a4f42faee6fd Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 15:46:05 -0500 Subject: [PATCH 028/115] Debugging OIDC response 8 - potential data --- siteapp/authentication/OIDCAuthentication.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 3bce951d8..37a4c1703 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -65,10 +65,11 @@ def create_user(self, claims): # 'username': claims[settings.OIDC_CLAIMS_MAP['username']], # 'is_staff': self.is_admin(claims[settings.OIDC_CLAIMS_MAP['groups']])} - data = {'email': claims.get(settings.OIDC_CLAIMS_MAP['email'], None), - 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['first_name'], None), - 'last_name': claims.get(settings.OIDC_CLAIMS_MAP['last_name'], None), - 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], None), + data = {'email': claims.get(settings.OIDC_CLAIMS_MAP['email'], "email@example.com"), + 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['name'], "first_name"), + 'last_name': claims.get(settings.OIDC_CLAIMS_MAP['last_name'], "last_name"), + # 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], None), + 'username': claims.get(settings.OIDC_CLAIMS_MAP['name'], "name"), 'is_staff': False} user = self.UserModel.objects.create_user(**data) From 99105480c08e756101d0112f70f3aeb3fd26034c Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 15:56:44 -0500 Subject: [PATCH 029/115] Debugging OIDC response 9 - potential data --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 37a4c1703..9cbb962de 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -69,7 +69,7 @@ def create_user(self, claims): 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['name'], "first_name"), 'last_name': claims.get(settings.OIDC_CLAIMS_MAP['last_name'], "last_name"), # 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], None), - 'username': claims.get(settings.OIDC_CLAIMS_MAP['name'], "name"), + 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], "username01"), 'is_staff': False} user = self.UserModel.objects.create_user(**data) From 2b815ad4d14d2cf76dfca8bc6f0a7d2fd09eff76 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 15 Apr 2022 16:22:57 -0500 Subject: [PATCH 030/115] Debugging OIDC response 10 - potential data --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 9cbb962de..4be278ff9 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -66,7 +66,7 @@ def create_user(self, claims): # 'is_staff': self.is_admin(claims[settings.OIDC_CLAIMS_MAP['groups']])} data = {'email': claims.get(settings.OIDC_CLAIMS_MAP['email'], "email@example.com"), - 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['name'], "first_name"), + 'first_name': claims.get(settings.OIDC_CLAIMS_MAP['first_name'], "first_name"), 'last_name': claims.get(settings.OIDC_CLAIMS_MAP['last_name'], "last_name"), # 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], None), 'username': claims.get(settings.OIDC_CLAIMS_MAP['username'], "username01"), From 852d9efdabb53674b9a9874b4a9167707396edfc Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Tue, 19 Apr 2022 10:55:01 -0500 Subject: [PATCH 031/115] request-ui wip --- api/controls/serializers/element.py | 59 ++++- api/controls/views/element.py | 33 ++- api/siteapp/serializers/request.py | 4 +- api/siteapp/views/request.py | 20 +- .../migrations/0070_auto_20220418_1140.py | 29 +++ controls/views.py | 55 ++++- frontend/src/components/requests/component.js | 8 +- frontend/src/components/requests/requests.js | 212 ++++++++++++++++++ .../system-owner-approval/component.js | 17 ++ .../requireApprovalModal.js | 131 +++++++++++ frontend/src/index.js | 4 +- siteapp/migrations/0058_request.py | 33 +++ siteapp/migrations/0059_auto_20220418_1159.py | 32 +++ siteapp/migrations/0060_auto_20220418_1508.py | 31 +++ siteapp/models.py | 21 +- templates/components/element_detail_tabs.html | 4 +- templates/components/element_form.html | 1 - templates/systems/components_selected.html | 20 +- 18 files changed, 687 insertions(+), 27 deletions(-) create mode 100644 controls/migrations/0070_auto_20220418_1140.py create mode 100644 frontend/src/components/system-owner-approval/component.js create mode 100644 frontend/src/components/system-owner-approval/requireApprovalModal.js create mode 100644 siteapp/migrations/0058_request.py create mode 100644 siteapp/migrations/0059_auto_20220418_1159.py create mode 100644 siteapp/migrations/0060_auto_20220418_1508.py diff --git a/api/controls/serializers/element.py b/api/controls/serializers/element.py index 51291f50d..7d2a0b61e 100644 --- a/api/controls/serializers/element.py +++ b/api/controls/serializers/element.py @@ -6,7 +6,7 @@ from api.siteapp.serializers.tags import SimpleTagSerializer from api.siteapp.serializers.appointment import SimpleAppointmentSerializer from controls.models import Element, ElementRole, ElementControl -from siteapp.models import Appointment, Role, Party, Tag +from siteapp.models import Appointment, Party, Request, Role, Tag from guardian.shortcuts import (assign_perm, get_objects_for_user, get_perms_for_model, get_user_perms, get_users_with_perms, remove_perm) @@ -169,4 +169,59 @@ class CreateMultipleAppointmentsFromRoleIds(WriteOnlySerializer): role_ids = serializers.JSONField() class Meta: model = Element - fields = ['role_ids'] \ No newline at end of file + fields = ['role_ids'] + +class ElementRequestsSerializer(ReadOnlySerializer): + requested = serializers.SerializerMethodField('get_list_of_requested') + + def get_list_of_requested(self, element): + list_of_requests = [] + for request in element.requests.all(): + + list_of_system_PointOfContacts = [] + for user in request.system.root_element.appointments.filter(role__title="Point of Contact"): + list_of_system_PointOfContacts.append(user.party.name) + + list_of_requestedElements_PointOfContacts = [] + for user in request.requested_element.appointments.filter(role__title="Point of Contact"): + list_of_requestedElements_PointOfContacts.append(user.party.name) + + req = { + "id": request.id, + "user_name": request.user.name, + "user_email": request.user.email, + "user_phone_number": request.user.phone_number, + "system": { + "id": request.system.id, + "name": request.system.root_element.name, + "full_name": request.system.root_element.full_name, + "name": request.system.root_element.name, + "description": request.system.root_element.description, + "point_of_contact": list_of_system_PointOfContacts, + }, + "requested_element: ": { + "id": request.requested_element.id, + "name": request.requested_element.name, + "full_name": request.requested_element.full_name, + "name": request.requested_element.name, + "description": request.requested_element.description, + "private": request.requested_element.private, + "require_approval": request.requested_element.require_approval, + "point_of_contact": list_of_requestedElements_PointOfContacts, + }, + "criteria_comment": request.criteria_comment, + "criteria_reject_comment": request.criteria_reject_comment, + "status": request.status, + } + # import ipdb; ipdb.set_trace() + list_of_requests.append(req) + return list_of_requests + class Meta: + model = Element + fields = ['requested'] + +class ElementSetRequestsSerializer(WriteOnlySerializer): + requests_ids = PrimaryKeyRelatedField(source='request', many=True, queryset=Request.objects) + class Meta: + model = Element + fields = ['requests_ids'] \ No newline at end of file diff --git a/api/controls/views/element.py b/api/controls/views/element.py index d9600c587..4d309b28b 100644 --- a/api/controls/views/element.py +++ b/api/controls/views/element.py @@ -4,7 +4,7 @@ from api.base.views.base import SerializerClasses from api.base.views.viewsets import ReadOnlyViewSet, ReadWriteViewSet from api.controls.serializers.element import DetailedElementSerializer, SimpleElementSerializer, \ - WriteElementTagsSerializer, ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer, WriteElementAppointPartySerializer, ElementPartySerializer, DeletePartyAppointmentsFromElementSerializer, CreateMultipleAppointmentsFromRoleIds + WriteElementTagsSerializer, ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer, WriteElementAppointPartySerializer, ElementPartySerializer, DeletePartyAppointmentsFromElementSerializer, CreateMultipleAppointmentsFromRoleIds, ElementRequestsSerializer, ElementSetRequestsSerializer from controls.models import Element from siteapp.models import Appointment, Party, Role from siteapp.models import User @@ -18,7 +18,10 @@ class ElementViewSet(ReadOnlyViewSet): appointments=WriteElementAppointPartySerializer, removeAppointments=WriteElementAppointPartySerializer, removeAppointmentsByParty=DeletePartyAppointmentsFromElementSerializer, - CreateAndSet=CreateMultipleAppointmentsFromRoleIds + CreateAndSet=CreateMultipleAppointmentsFromRoleIds, + retrieveRequests=ElementRequestsSerializer, + setRequest=ElementSetRequestsSerializer, + ) @action(detail=True, url_path="tags", methods=["PUT"]) @@ -37,7 +40,7 @@ def retrieveParties(self, request, **kwargs): element, validated_data = self.validate_serializer_and_get_object(request) element.save() - # import ipdb; ipdb.set_trace() + serializer_class = self.get_serializer_class('retrieve') serializer = self.get_serializer(serializer_class, element) return Response(serializer.data) @@ -108,6 +111,30 @@ def CreateAndSet(self, request, **kwargs): serializer_class = self.get_serializer_class('retrieve') serializer = self.get_serializer(serializer_class, element) return Response(serializer.data) + + @action(detail=True, url_path="retrieveRequests", methods=["GET"]) + def retrieveRequests(self, request, **kwargs): + element, validated_data = self.validate_serializer_and_get_object(request) + element.save() + + serializer_class = self.get_serializer_class('retrieveRequests') + serializer = self.get_serializer(serializer_class, element) + return Response(serializer.data) + + @action(detail=True, url_path="setRequest", methods=["PUT"]) + def setRequest(self, request, **kwargs): + element, validated_data = self.validate_serializer_and_get_object(request) + # import ipdb; ipdb.set_trace() + for key, value in validated_data.items(): + element.add_requests(value) + element.save() + + serializer_class = self.get_serializer_class('retrieve') + serializer = self.get_serializer(serializer_class, element) + return Response(serializer.data) + + + class ElementWithPermissionsViewSet(ReadWriteViewSet): # NESTED_ROUTER_PKS = [{'pk': 'modules_pk', 'model_field': 'module', 'model': Module}] queryset = Element.objects.all() diff --git a/api/siteapp/serializers/request.py b/api/siteapp/serializers/request.py index 236750a99..f2d980379 100644 --- a/api/siteapp/serializers/request.py +++ b/api/siteapp/serializers/request.py @@ -5,9 +5,9 @@ class SimpleRequestSerializer(ReadOnlySerializer): class Meta: model = Request - fields = ['user', 'system', 'element', 'criteria_comment', 'criteria_reject_comment', 'status', 'created'] + fields = ['user', 'system', 'requested_element', 'criteria_comment', 'criteria_reject_comment', 'status', 'created'] class WriteRequestSerializer(WriteOnlySerializer): class Meta: model = Request - fields = ['user', 'system', 'element', 'criteria_comment', 'criteria_reject_comment', 'status'] + fields = ['user', 'system', 'requested_element', 'criteria_comment', 'criteria_reject_comment', 'status'] diff --git a/api/siteapp/views/request.py b/api/siteapp/views/request.py index 480f24238..8db9c653a 100644 --- a/api/siteapp/views/request.py +++ b/api/siteapp/views/request.py @@ -1,3 +1,6 @@ +from rest_framework.decorators import action +from rest_framework.response import Response + from api.base.views.base import SerializerClasses from api.base.views.viewsets import ReadWriteViewSet from api.siteapp.serializers.request import SimpleRequestSerializer, WriteRequestSerializer @@ -11,5 +14,18 @@ class RequestViewSet(ReadWriteViewSet): list=SimpleRequestSerializer, create=WriteRequestSerializer, update=WriteRequestSerializer, - destroy=WriteRequestSerializer) - # filter_class = RequestFilter \ No newline at end of file + destroy=WriteRequestSerializer,) + # requestList=WriteRequestSerializer) + # filter_class = RequestFilter + + # @action(detail=True, url_path="requestList", methods=["PUT"]) + # def appointments(self, request, **kwargs): + # req, validated_data = self.validate_serializer_and_get_object(request) + + # for key, value in validated_data.items(): + # req.add_appointments(value) + # req.save() + + # serializer_class = self.get_serializer_class('retrieve') + # serializer = self.get_serializer(serializer_class, req) + # return Response(serializer.data) diff --git a/controls/migrations/0070_auto_20220418_1140.py b/controls/migrations/0070_auto_20220418_1140.py new file mode 100644 index 000000000..098e99487 --- /dev/null +++ b/controls/migrations/0070_auto_20220418_1140.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.12 on 2022-04-18 11:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('siteapp', '0058_request'), + ('controls', '0069_alter_element_require_approval'), + ] + + operations = [ + migrations.AddField( + model_name='element', + name='requests', + field=models.ManyToManyField(related_name='element', to='siteapp.Request'), + ), + migrations.AlterField( + model_name='historicalstatement', + name='statement_type', + field=models.CharField(blank=True, choices=[('CONTROL_IMPLEMENTATION', 'control_implementation'), ('CONTROL_IMPLEMENTATION_LEGACY', 'control_implementation_legacy'), ('CONTROL_IMPLEMENTATION_PROTOTYPE', 'control_implementation_prototype'), ('ASSESSMENT_RESULT', 'assessment_result'), ('POAM', 'POAM'), ('SECURITY_SENSITIVITY_LEVEL', 'security_sensitivity_level'), ('SECURITY_IMPACT_LEVEL', 'security_impact_level'), ('COMPONENT_APPROVAL_CRITERIA', 'component_approval_criteria')], help_text='Statement type.', max_length=150, null=True), + ), + migrations.AlterField( + model_name='statement', + name='statement_type', + field=models.CharField(blank=True, choices=[('CONTROL_IMPLEMENTATION', 'control_implementation'), ('CONTROL_IMPLEMENTATION_LEGACY', 'control_implementation_legacy'), ('CONTROL_IMPLEMENTATION_PROTOTYPE', 'control_implementation_prototype'), ('ASSESSMENT_RESULT', 'assessment_result'), ('POAM', 'POAM'), ('SECURITY_SENSITIVITY_LEVEL', 'security_sensitivity_level'), ('SECURITY_IMPACT_LEVEL', 'security_impact_level'), ('COMPONENT_APPROVAL_CRITERIA', 'component_approval_criteria')], help_text='Statement type.', max_length=150, null=True), + ), + ] diff --git a/controls/views.py b/controls/views.py index 2ae11f1b9..1bee3d873 100644 --- a/controls/views.py +++ b/controls/views.py @@ -49,7 +49,7 @@ from .models import * from .utilities import * from siteapp.utils.views_helper import project_context -from siteapp.models import Role, Party, Appointment +from siteapp.models import Role, Party, Appointment, Request logging.basicConfig() import structlog @@ -1324,7 +1324,7 @@ def component_library_component(request, element_id): # Retrieve element element = Element.objects.get(id=element_id) - + govSystem = System.objects.filter(root_element__name="GovReady-Q Sample System") # Check permissions if element.private == True and 'view_element' not in get_user_perms(request.user, element): logger.warning( @@ -1363,7 +1363,12 @@ def component_library_component(request, element_id): # import ipdb; ipdb.set_trace() #TODO: Count element's requests - element.requests.count() total_number_of_requests = 3 + # status=RequestStatusEnum.PENDING.name + # req_instance = Request.objects.create(user=user, element=element, status="pending") + # req_instance.system.set(system) + # req_instance.save() + # import ipdb; ipdb.set_trace() contacts = [] for poc in get_all_parties: contacts.append(poc.party) @@ -1458,7 +1463,7 @@ def get_item(dictionary, key): "is_owner": is_owner, "can_edit": hasPermissionToEdit, "users_with_permissions": usersWithPermission, - "criteria_text": criteria_text.first().body, + "criteria": criteria_text, "contacts": serializers.serialize('json', contacts), "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol, "enable_experimental_oscal": SystemSettings.enable_experimental_oscal, @@ -2461,10 +2466,53 @@ def add_system_component(request, system_id): elements_selected = system.producer_elements elements_selected_ids = [e.id for e in elements_selected] + + # Serg TODO: CHECK IF ELEMENT WE ARE ABOUT TO ADD REQUIRES A REQUEST + # 1. IF REQUEST IS REQUIERED -> FIRE OFF A MODAL + + + element = Element.objects.get(pk=form_values['producer_element_id']) + + if element.require_approval == True: + # 1.a Return Element's criteria and Point of Contact information + # 1.b Create Request object, attach to both System and Element component + + # import ipdb; ipdb.set_trace() + criteria_results = element.statements_produced.filter(statement_type=StatementTypeEnum.COMPONENT_APPROVAL_CRITERIA.name) + if len(criteria_results) > 0: + criteria_text = criteria_results.first().body + else: + criteria_text = "" + + newRequest = Request.objects.create( + user=request.user, + system=system, + requested_element=element, + criteria_comment=criteria_text, + criteria_reject_comment="", + status="PENDING", + ) + newRequest.save() + element.add_requests([newRequest.id]) + element.save() + # system.add_requests([newRequest]) + # import ipdb; ipdb.set_trace(); + context = { + "requested_elementId": element.id, + "requested_element_name": element.name, + "requested_element_require_approval": element.require_approval, + "criteria_text": criteria_text, + } + return render(request, "systems/components_selected.html", context) + # 2. IF REQUEST IS NOT REQUIERED -> ADD ELEMENT AND STATEMENTS TO SYSTEM + # Add element to system's selected components # Look up the element rto add + producer_element = Element.objects.get(pk=form_values['producer_element_id']) + + # TODO: various use cases # - component previously added but element has statements not yet added to system # this issue may be best addressed elsewhere. @@ -2515,6 +2563,7 @@ def add_system_component(request, system_id): else: return HttpResponseRedirect("/systems/{}/components/selected".format(system_id)) + @login_required def search_system_component(request): """Add an existing element and its statements to a system""" diff --git a/frontend/src/components/requests/component.js b/frontend/src/components/requests/component.js index 26c6eadf7..89f465a7f 100644 --- a/frontend/src/components/requests/component.js +++ b/frontend/src/components/requests/component.js @@ -1,17 +1,17 @@ import React, {useState} from 'react'; import ReactDOM from 'react-dom'; -import { PointOfContacts } from './point_of_contacts'; +import { RequestsTable } from './requests'; import { Provider } from "react-redux"; import store from "../../store"; -window.pocTable = ( elementId, is_owner ) => { +window.requestsTable = ( elementId, is_owner ) => { $(window).on('load', function () { $("#content").show(); ReactDOM.render( - + , - document.getElementById('poc-table') + document.getElementById('requests-table') ); }); }; \ No newline at end of file diff --git a/frontend/src/components/requests/requests.js b/frontend/src/components/requests/requests.js index e69de29bb..945917a6c 100644 --- a/frontend/src/components/requests/requests.js +++ b/frontend/src/components/requests/requests.js @@ -0,0 +1,212 @@ +import React, { useEffect, useState } from 'react'; +import { useSelector, useDispatch } from 'react-redux'; +import { DataTable } from '../shared/table'; +import axios from 'axios'; +import moment from 'moment'; +import { DataGrid } from '@mui/x-data-grid'; +import { v4 as uuid_v4 } from "uuid"; +import { + Chip, + Grid, + Stack, +} from '@mui/material'; +import { makeStyles } from '@mui/styles'; +import { + Tooltip, + Button, + Glyphicon, + OverlayTrigger, + Col, + ControlLabel, + Form, + FormControl, + FormGroup, + Row, + Modal +} from 'react-bootstrap'; +import { AsyncPagination } from "../shared/asyncTypeahead"; +import { red, green } from '@mui/material/colors'; +import { ReactModal } from '../shared/modal'; +import { hide, show } from '../shared/modalSlice'; + +const datagridStyles = makeStyles({ + root: { + "& .MuiDataGrid-renderingZone": { + maxHeight: "none !important" + }, + "& .MuiDataGrid-cell": { + lineHeight: "unset !important", + maxHeight: "none !important", + whiteSpace: "normal" + }, + "& .MuiDataGrid-row": { + maxHeight: "none !important" + }, + "& .MuiDataGrid-main":{ + height: "30rem !important", + }, + "& .MuiDataGrid-virtualScroller":{ + height: "30rem !important", + } + } +}); + + +const useStyles = makeStyles({ + root: { + fontweight: 900, + }, + header: { + '& .MuiDataGrid-columnHeaderTitleContainer':{ + flexFlow: 'row-reverse', + }, + } +}); + +export const RequestsTable = ({ elementId, isOwner }) => { + const dispatch = useDispatch(); + + const classes = useStyles(); + const dgClasses = datagridStyles(); + const [data, setData] = useState([]); + const [sortby, setSortBy] = useState(["name", "asc"]); + + const endpoint = (querystrings) => { + return axios.get(`/api/v2/requests/`, { params: querystrings }); + }; + + useEffect(() => { + axios(`/api/v2/elements/${elementId}/retrieveRequests/`).then(response => { + // debugger; + setData(response.data.requested); + }); + }, []) + + const [columnsForEditor, setColumnsForEditor] = useState([ + { + field: 'user', + headerName: 'User', + width: 150, + editable: false, + valueGetter: (params) => console.log(params), + }, + { + field: 'system', + headerName: 'Requested by', + width: 150, + editable: false, + valueGetter: (params) => params.row.system.name, + }, + { + field: 'point_of_contact', + headerName: 'Point of Contact', + width: 300, + editable: false, + renderCell: (params) => ( +
+ {params.row.user_name} {(params.row.user_phone_number) ? `(${params.row.user_phone_number})` : ''} +
+ ), + }, + // { + // field: 'point_of_contact', + // headerName: 'Point of Contact', + // width: 300, + // editable: false, + // valueGetter: (params) => params.row.system.point_of_contact, + // }, + // { + // field: 'req_poc', + // headerName: 'RequestedPoint of Contact', + // width: 300, + + // editable: false, + // valueGetter: (params) => params.row.requested_element.point_of_contact[0], + // }, + { + field: 'status', + headerName: 'Status', + width: 150, + editable: false, + valueGetter: (params) => params.row.status, + }, + { + field: 'action', + headerName: 'Action', + width: 300, + editable: false, + renderCell: (params) => ( +
+ Action bar and submit button +
+ ), + }, + + ]); + + const [columns, setColumns] = useState([ + { + field: 'user', + headerName: 'User', + width: 150, + editable: false, + valueGetter: (params) => params.row.system.user.full_name, + }, + { + field: 'system', + headerName: 'Requested by', + width: 150, + editable: false, + valueGetter: (params) => params.row.system.root_element.full_name, + }, + // { + // field: 'email', + // headerName: 'Point of Contact', + // width: 300, + // editable: false, + // valueGetter: (params) => params.row.email, + // }, + { + field: 'status', + headerName: 'Status', + width: 150, + editable: false, + valueGetter: (params) => params.row.status, + }, + ]); + + + + + return ( +
+ +
+
+ { + // console.log(selectionModel, details); + // }} + // disableSelectionOnClick + sx={{ + fontSize: '14px', + '& .MuiDataGrid-columnHeaderTitle':{ + fontWeight: 600, + }, + }} + /> +
+
+ +
+ ) +} \ No newline at end of file diff --git a/frontend/src/components/system-owner-approval/component.js b/frontend/src/components/system-owner-approval/component.js new file mode 100644 index 000000000..73f36b0f6 --- /dev/null +++ b/frontend/src/components/system-owner-approval/component.js @@ -0,0 +1,17 @@ +import React, {useState} from 'react'; +import ReactDOM from 'react-dom'; +import { RequireApprovalModal } from './requireApprovalModal'; +import { Provider } from "react-redux"; +import store from "../../store"; + +window.requireApprovalModal = ( systemId, is_owner ) => { + $(window).on('load', function () { + $("#content").show(); + ReactDOM.render( + + + , + document.getElementById('private-component-modal') + ); + }); +}; \ No newline at end of file diff --git a/frontend/src/components/system-owner-approval/requireApprovalModal.js b/frontend/src/components/system-owner-approval/requireApprovalModal.js new file mode 100644 index 000000000..2f4dce298 --- /dev/null +++ b/frontend/src/components/system-owner-approval/requireApprovalModal.js @@ -0,0 +1,131 @@ +import React, { useEffect, useState } from 'react'; +import { useSelector, useDispatch } from 'react-redux'; +import { DataTable } from '../shared/table'; +import axios from 'axios'; +import moment from 'moment'; +import { DataGrid } from '@mui/x-data-grid'; +import { v4 as uuid_v4 } from "uuid"; +import { + Chip, + Grid, + Stack, +} from '@mui/material'; +import { makeStyles } from '@mui/styles'; +import { + Tooltip, + Button, + Glyphicon, + OverlayTrigger, + Col, + ControlLabel, + Form, + FormControl, + FormGroup, + Row, + Modal +} from 'react-bootstrap'; +import { AsyncPagination } from "../shared/asyncTypeahead"; +import { red, green } from '@mui/material/colors'; +import { ReactModal } from '../shared/modal'; +import { hide, show } from '../shared/modalSlice'; +import { element } from 'prop-types'; + +const datagridStyles = makeStyles({ + root: { + "& .MuiDataGrid-renderingZone": { + maxHeight: "none !important" + }, + "& .MuiDataGrid-cell": { + lineHeight: "unset !important", + maxHeight: "none !important", + whiteSpace: "normal" + }, + "& .MuiDataGrid-row": { + maxHeight: "none !important" + }, + "& .MuiDataGrid-main":{ + height: "30rem !important", + }, + "& .MuiDataGrid-virtualScroller":{ + height: "30rem !important", + } + } +}); + + +const useStyles = makeStyles({ + root: { + fontweight: 900, + }, + header: { + '& .MuiDataGrid-columnHeaderTitleContainer':{ + flexFlow: 'row-reverse', + }, + } +}); + +export const RequireApprovalModal = ({ systemId, isOwner }) => { + const dispatch = useDispatch(); + + const classes = useStyles(); + const dgClasses = datagridStyles(); + const [data, setData] = useState([]); + const [openRequireApprovalModal, setOpenRequireApprovalModal] = useState(false); + + const endpoint = (querystrings) => { + return axios.get(`/api/v2/roles/`, { params: querystrings }); + }; + + + useEffect(() => { + axios(`/api/v2/elements/${elementId}/`).then(response => { + setData(response.data.parties); + }); + }, []) + + const handleSubmit = async (event) => { + console.log('handleSubmit!') + } + + return ( +
+ setOpenRequireApprovalModal(false)} + header={ +
+ <> + +
+

You have selected a "protected" common control component.

+ + + + + } + body={ + + +
+

The {element.full_name} common control set required approval/whitelist.

+ +
+ + + + + + + + } + /> + + ) +} \ No newline at end of file diff --git a/frontend/src/index.js b/frontend/src/index.js index b9cfed8a8..3f72eaea1 100644 --- a/frontend/src/index.js +++ b/frontend/src/index.js @@ -7,4 +7,6 @@ import './components/permissions/component'; import './components/permissions/permissions'; import './components/point_of_contacts/component'; -import './components/point_of_contacts/point_of_contacts'; \ No newline at end of file +import './components/point_of_contacts/point_of_contacts'; +import './components/requests/component'; +import './components/requests/requests'; \ No newline at end of file diff --git a/siteapp/migrations/0058_request.py b/siteapp/migrations/0058_request.py new file mode 100644 index 000000000..dbde970d8 --- /dev/null +++ b/siteapp/migrations/0058_request.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.12 on 2022-04-18 11:40 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0069_alter_element_require_approval'), + ('siteapp', '0057_auto_20220331_1650'), + ] + + operations = [ + migrations.CreateModel( + name='Request', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated', models.DateTimeField(auto_now=True, db_index=True, null=True)), + ('criteria_comment', models.TextField(blank=True, help_text='Comments on this request.', null=True)), + ('criteria_reject_comment', models.TextField(blank=True, help_text='Comment on request rejection.', null=True)), + ('status', models.TextField(blank=True, help_text='Status of the request.', null=True)), + ('requested_element', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='request', to='controls.element')), + ('system', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='request', to='controls.system')), + ('user', models.ForeignKey(blank=True, help_text='User creating the request.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='request', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/siteapp/migrations/0059_auto_20220418_1159.py b/siteapp/migrations/0059_auto_20220418_1159.py new file mode 100644 index 000000000..7936572a0 --- /dev/null +++ b/siteapp/migrations/0059_auto_20220418_1159.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.12 on 2022-04-18 11:59 + +import auto_prefetch +from django.conf import settings +from django.db import migrations +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0070_auto_20220418_1140'), + ('siteapp', '0058_request'), + ] + + operations = [ + migrations.AlterField( + model_name='request', + name='requested_element', + field=auto_prefetch.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='request', to='controls.element'), + ), + migrations.AlterField( + model_name='request', + name='system', + field=auto_prefetch.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='request', to='controls.system'), + ), + migrations.AlterField( + model_name='request', + name='user', + field=auto_prefetch.ForeignKey(blank=True, help_text='User creating the request.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='request', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/siteapp/migrations/0060_auto_20220418_1508.py b/siteapp/migrations/0060_auto_20220418_1508.py new file mode 100644 index 000000000..b4b54eee7 --- /dev/null +++ b/siteapp/migrations/0060_auto_20220418_1508.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.12 on 2022-04-18 15:08 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0070_auto_20220418_1140'), + ('siteapp', '0059_auto_20220418_1159'), + ] + + operations = [ + migrations.AlterField( + model_name='request', + name='requested_element', + field=models.ForeignKey(blank=True, help_text='Element being requested.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='request', to='controls.element'), + ), + migrations.AlterField( + model_name='request', + name='system', + field=models.ForeignKey(blank=True, help_text='System making the request.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='request', to='controls.system'), + ), + migrations.AlterField( + model_name='request', + name='user', + field=models.ForeignKey(blank=True, help_text='User creating the request.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='request', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/siteapp/models.py b/siteapp/models.py index 640881625..3901e9a61 100644 --- a/siteapp/models.py +++ b/siteapp/models.py @@ -1463,13 +1463,26 @@ def __str__(self): return f"{self.model_name} {self.role.title} - {self.party.name}" class Request(BaseModel): - user = models.ForeignKey(User, blank=True, null=True, related_name="request", on_delete=models.SET_NULL, - help_text="User creating the request.") - system = models.ForeignKey(System, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) - requested_element = models.ForeignKey(Element, blank=True, null=True, related_name="request", on_delete=models.SET_NULL,) + user = models.ForeignKey(User, blank=True, null=True, related_name="request", on_delete=models.CASCADE, help_text="User creating the request.") + system = models.ForeignKey(System, blank=True, null=True, related_name="request", on_delete=models.CASCADE, help_text="System making the request.") + requested_element = models.ForeignKey(Element, blank=True, null=True, related_name="request", on_delete=models.CASCADE, help_text="Element being requested.") criteria_comment = models.TextField(blank=True, null=True, help_text="Comments on this request.") criteria_reject_comment = models.TextField(blank=True, null=True, help_text="Comment on request rejection.") status = models.TextField(blank=True, null=True, help_text="Status of the request.") + + # user = models.ForeignKey(User, on_delete=models.CASCADE, help_text="User creating the request.") + # system = models.ForeignKey(System, on_delete=models.CASCADE, help_text="System making the request.") + # requested_element = models.ForeignKey(Element, on_delete=models.CASCADE, help_text="Element being requested.") + + def __repr__(self): + return f"{self.system} -> {self.requested_element} - {self.status}" + + def __str__(self): + return f"{self.system} -> {self.requested_element} - {self.status}" + + def serialize(self): + return {"system": self.system, "requested_element": self.requested_element, "id": self.id} + class Asset(BaseModel): UPLOAD_TO = None # Should be overriden when iheritted title = models.CharField(max_length=255, help_text="The title of this asset.") diff --git a/templates/components/element_detail_tabs.html b/templates/components/element_detail_tabs.html index 9b63928a4..d3a6ebd62 100644 --- a/templates/components/element_detail_tabs.html +++ b/templates/components/element_detail_tabs.html @@ -405,7 +405,7 @@

TBD

Requested Usage({{requestsTotal}})

-
+
@@ -424,6 +424,7 @@

Requested Usage({{requestsTotal}})

window.pocTable( {{ element.id }}, {{ contacts | safe }}, "{{is_owner}}" === 'True' ? true : false ); window.permissionsTable( {{ element.id }}, "{{is_owner}}" === 'True' ? true : false ); window.renderElementTags({{ element.id }}, {{ tags | safe }}); + window.requestsTable( {{ element.id }}, "{{is_owner}}" === 'True' ? true : false ); From 3e7e8909c3de61a4f810f0b01a695d12862203aa Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Thu, 21 Apr 2022 11:12:52 -0500 Subject: [PATCH 033/115] React Modal for System Owner Requesting --- api/controls/serializers/element.py | 13 +- api/controls/views/element.py | 28 ++++- controls/views.py | 43 +------ frontend/src/components/requests/requests.js | 11 +- .../system-owner-approval/component.js | 4 +- .../requireApprovalModal.js | 112 ++++++++++-------- templates/systems/components_selected.html | 26 +--- 7 files changed, 110 insertions(+), 127 deletions(-) diff --git a/api/controls/serializers/element.py b/api/controls/serializers/element.py index e2fab7c21..b27736c44 100644 --- a/api/controls/serializers/element.py +++ b/api/controls/serializers/element.py @@ -233,4 +233,15 @@ class ElementSetRequestsSerializer(WriteOnlySerializer): requests_ids = PrimaryKeyRelatedField(source='request', many=True, queryset=Request.objects) class Meta: model = Element - fields = ['requests_ids'] \ No newline at end of file + fields = ['requests_ids'] + +class ElementCreateAndSetRequestSerializer(WriteOnlySerializer): + # request = serializers.JSONField() + userId = serializers.IntegerField(max_value=None, min_value=None) + systemId = serializers.IntegerField(max_value=None, min_value=None) + criteria_comment = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True) + criteria_reject_comment = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True) + status = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True) + class Meta: + model = Element + fields = ['userId', 'systemId', 'criteria_comment', 'criteria_reject_comment', 'status'] \ No newline at end of file diff --git a/api/controls/views/element.py b/api/controls/views/element.py index 4d309b28b..742362bbd 100644 --- a/api/controls/views/element.py +++ b/api/controls/views/element.py @@ -1,13 +1,13 @@ +import collections from rest_framework.decorators import action from rest_framework.response import Response from api.base.views.base import SerializerClasses from api.base.views.viewsets import ReadOnlyViewSet, ReadWriteViewSet from api.controls.serializers.element import DetailedElementSerializer, SimpleElementSerializer, \ - WriteElementTagsSerializer, ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer, WriteElementAppointPartySerializer, ElementPartySerializer, DeletePartyAppointmentsFromElementSerializer, CreateMultipleAppointmentsFromRoleIds, ElementRequestsSerializer, ElementSetRequestsSerializer -from controls.models import Element -from siteapp.models import Appointment, Party, Role -from siteapp.models import User + WriteElementTagsSerializer, ElementPermissionSerializer, UpdateElementPermissionSerializer, RemoveUserPermissionFromElementSerializer, WriteElementAppointPartySerializer, ElementPartySerializer, DeletePartyAppointmentsFromElementSerializer, CreateMultipleAppointmentsFromRoleIds, ElementRequestsSerializer, ElementSetRequestsSerializer, ElementCreateAndSetRequestSerializer +from controls.models import Element, System +from siteapp.models import Appointment, Party, Role, Request, User class ElementViewSet(ReadOnlyViewSet): queryset = Element.objects.all() @@ -21,7 +21,7 @@ class ElementViewSet(ReadOnlyViewSet): CreateAndSet=CreateMultipleAppointmentsFromRoleIds, retrieveRequests=ElementRequestsSerializer, setRequest=ElementSetRequestsSerializer, - + CreateAndSetRequest=ElementCreateAndSetRequestSerializer, ) @action(detail=True, url_path="tags", methods=["PUT"]) @@ -133,6 +133,24 @@ def setRequest(self, request, **kwargs): serializer = self.get_serializer(serializer_class, element) return Response(serializer.data) + @action(detail=True, url_path="CreateAndSetRequest", methods=["POST"]) + def CreateAndSetRequest(self, request, **kwargs): + element, validated_data = self.validate_serializer_and_get_object(request) + newRequest = Request.objects.create( + user=User.objects.get(id=validated_data['userId']), + system=System.objects.get(id=validated_data['systemId']), + requested_element=element, + criteria_comment=validated_data['criteria_comment'], + criteria_reject_comment=validated_data['criteria_reject_comment'], + status=validated_data['status'], + ) + newRequest.save() + element.add_requests([newRequest.id]) + element.save() + + serializer_class = self.get_serializer_class('retrieve') + serializer = self.get_serializer(serializer_class, element) + return Response(serializer.data) class ElementWithPermissionsViewSet(ReadWriteViewSet): diff --git a/controls/views.py b/controls/views.py index 4d8e09ca2..a31560932 100644 --- a/controls/views.py +++ b/controls/views.py @@ -2469,46 +2469,6 @@ def add_system_component(request, system_id): elements_selected = system.producer_elements elements_selected_ids = [e.id for e in elements_selected] - - # Serg TODO: CHECK IF ELEMENT WE ARE ABOUT TO ADD REQUIRES A REQUEST - # 1. IF REQUEST IS REQUIERED -> FIRE OFF A MODAL - - - element = Element.objects.get(pk=form_values['producer_element_id']) - - if element.require_approval == True: - # 1.a Return Element's criteria and Point of Contact information - # 1.b Create Request object, attach to both System and Element component - - # import ipdb; ipdb.set_trace() - criteria_results = element.statements_produced.filter(statement_type=StatementTypeEnum.COMPONENT_APPROVAL_CRITERIA.name) - if len(criteria_results) > 0: - criteria_text = criteria_results.first().body - else: - criteria_text = "" - - newRequest = Request.objects.create( - user=request.user, - system=system, - requested_element=element, - criteria_comment=criteria_text, - criteria_reject_comment="", - status="PENDING", - ) - newRequest.save() - element.add_requests([newRequest.id]) - element.save() - # system.add_requests([newRequest]) - # import ipdb; ipdb.set_trace(); - context = { - "requested_elementId": element.id, - "requested_element_name": element.name, - "requested_element_require_approval": element.require_approval, - "criteria_text": criteria_text, - } - return render(request, "systems/components_selected.html", context) - # 2. IF REQUEST IS NOT REQUIERED -> ADD ELEMENT AND STATEMENTS TO SYSTEM - # Add element to system's selected components # Look up the element rto add @@ -2521,6 +2481,7 @@ def add_system_component(request, system_id): # this issue may be best addressed elsewhere. # Component already added to system. Do not add the component (element) to the system again. + if producer_element.id in elements_selected_ids: messages.add_message(request, messages.ERROR, f'Component "{producer_element.name}" already exists in selected components.') @@ -2529,7 +2490,7 @@ def add_system_component(request, system_id): return HttpResponseRedirect(form_values['redirect_url']) else: return HttpResponseRedirect("/systems/{}/components/selected".format(system_id)) - + smts = Statement.objects.filter(producer_element_id = producer_element.id, statement_type=StatementTypeEnum.CONTROL_IMPLEMENTATION_PROTOTYPE.name) # Component does not have any statements of type control_implementation_prototype to diff --git a/frontend/src/components/requests/requests.js b/frontend/src/components/requests/requests.js index 945917a6c..b6a95aec6 100644 --- a/frontend/src/components/requests/requests.js +++ b/frontend/src/components/requests/requests.js @@ -71,15 +71,10 @@ export const RequestsTable = ({ elementId, isOwner }) => { const [data, setData] = useState([]); const [sortby, setSortBy] = useState(["name", "asc"]); - const endpoint = (querystrings) => { - return axios.get(`/api/v2/requests/`, { params: querystrings }); - }; - useEffect(() => { - axios(`/api/v2/elements/${elementId}/retrieveRequests/`).then(response => { - // debugger; - setData(response.data.requested); - }); + axios(`/api/v2/elements/${elementId}/retrieveRequests/`).then(response => { + setData(response.data.requested); + }); }, []) const [columnsForEditor, setColumnsForEditor] = useState([ diff --git a/frontend/src/components/system-owner-approval/component.js b/frontend/src/components/system-owner-approval/component.js index 273469561..dabd01ed1 100644 --- a/frontend/src/components/system-owner-approval/component.js +++ b/frontend/src/components/system-owner-approval/component.js @@ -5,11 +5,11 @@ import { RequireApprovalModal } from './requireApprovalModal'; import { Provider } from "react-redux"; import store from "../../store"; -window.requireApprovalModal = ( systemId, elementId, require_approval ) => { +window.requireApprovalModal = ( userId, systemId, elementId, require_approval ) => { const uuid = uuid_v4(); ReactDOM.render( - + , document.getElementById('private-component-modal') ); diff --git a/frontend/src/components/system-owner-approval/requireApprovalModal.js b/frontend/src/components/system-owner-approval/requireApprovalModal.js index 04600d79e..f05d98e1b 100644 --- a/frontend/src/components/system-owner-approval/requireApprovalModal.js +++ b/frontend/src/components/system-owner-approval/requireApprovalModal.js @@ -64,29 +64,48 @@ const useStyles = makeStyles({ } }); -export const RequireApprovalModal = ({ systemId, elementId, require_approval, uuid }) => { +export const RequireApprovalModal = ({ userId, systemId, elementId, require_approval, uuid }) => { const classes = useStyles(); const dgClasses = datagridStyles(); const [data, setData] = useState([]); - const [openRequireApprovalModal, setOpenRequireApprovalModal] = useState(require_approval); - -// const endpoint = (querystrings) => { -// return axios.get(`/api/v2/roles/`, { params: querystrings }); -// }; - + const [openRequireApprovalModal, setOpenRequireApprovalModal] = useState(false); useEffect(() => { - axios(`/api/v2/elements/${elementId}/`).then(response => { - setData(response.data); - }); - }, [elementId]) + axios(`/api/v2/elements/${elementId}/`).then(response => { + setData(response.data); + if(require_approval || response.data.criteria.length > 0){ + setOpenRequireApprovalModal(true); + } + if(!require_approval && response.data.criteria === ""){ + debugger; + console.log('No approval requirement and no criteria set') + /* add_component form can be found in systems/component_selected.html */ + document.add_component.submit(); + } + + }); + }, [elementId, uuid]) const clearModal = async (event) => { console.log('clearModal!') } const handleSubmit = async (event) => { console.log('handleSubmit!') + + const newReq = { + userId: userId, + systemId: systemId, + criteria_comment: data.criteria, + criteria_reject_comment: "", + status: "pending" + } /* Create a request and assign it to element and system */ + const newRequestResponse = await axios.post(`/api/v2/elements/${elementId}/CreateAndSetRequest/`, newReq); + if(newRequestResponse.status === 200){ + handleClose(); + } else { + console.error("Something went wrong in creating and setting new request to element") + } } const handleClose = async (event) => { console.log('handleClose!') @@ -96,43 +115,42 @@ export const RequireApprovalModal = ({ systemId, elementId, require_approval, uu console.log('data: ', data, require_approval, uuid, openRequireApprovalModal); return (
- {data.criteria !== "" && setOpenRequireApprovalModal(false)} - header={ -
- <> - -
-

You have selected a "protected" common control component.

- - - - - } - body={ - - -
-

The {data.full_name} common control set required approval/whitelist.

- {data.criteria} -
+ setOpenRequireApprovalModal(false)} + header={ + + <> + + +

You have selected a "protected" common control component.

+ - - - - - - } - /> - } + + + } + body={ + + +
+ {require_approval ?

The {data.full_name} common control has set an approval/whitelist requirement.

:

The {data.full_name} common control has not set required approval/whitelist, but has criteria that must be met.

} + {data.criteria ? {data.criteria} : No criteria set} +
+ + + + + + + } + /> ) } \ No newline at end of file diff --git a/templates/systems/components_selected.html b/templates/systems/components_selected.html index dcc94a128..a30cb4bb6 100644 --- a/templates/systems/components_selected.html +++ b/templates/systems/components_selected.html @@ -20,7 +20,7 @@
Selected components
 
-
+ {% csrf_token %}
   @@ -78,33 +78,13 @@ {% block scripts %} {{ block.super }} + +{% endblock %} From 21210902dfb59ca5a455315493950affc1fd16d9 Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Fri, 22 Apr 2022 13:22:23 -0500 Subject: [PATCH 037/115] wip --- frontend/src/components/requests/requests.js | 64 +++++++++++++++++--- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/requests/requests.js b/frontend/src/components/requests/requests.js index 1dfbd93c8..81ba516c3 100644 --- a/frontend/src/components/requests/requests.js +++ b/frontend/src/components/requests/requests.js @@ -3,11 +3,13 @@ import { useSelector, useDispatch } from 'react-redux'; import { DataTable } from '../shared/table'; import axios from 'axios'; import moment from 'moment'; -import { DataGrid } from '@mui/x-data-grid'; +import { DataGrid, useGridApiContext } from '@mui/x-data-grid'; import { v4 as uuid_v4 } from "uuid"; +import PropTypes from 'prop-types'; import { Chip, Grid, + Select, Stack, } from '@mui/material'; import { makeStyles } from '@mui/styles'; @@ -77,6 +79,52 @@ export const RequestsTable = ({ elementId, isOwner }) => { }); }, []) + function SelectEditInputCell(props) { + const { id, value, field } = props; + const apiRef = useGridApiContext(); + + const handleChange = async (event) => { + debugger; + await apiRef.current.setEditCellValue({ id, field: 'status', value: event.target.value }); + apiRef.current.stopCellEditMode({ id, field: 'status' }); + apiRef.current.stopCellEditMode({ id, field }); + }; + + return ( + + ); + } + + SelectEditInputCell.propTypes = { + /** + * The column field of the cell that triggered the event. + */ + field: PropTypes.string.isRequired, + /** + * The grid row id. + */ + id: PropTypes.oneOfType([PropTypes.number, PropTypes.string]).isRequired, + /** + * The cell value, but if the column has valueGetter, use getValue. + */ + value: PropTypes.any, + }; + + const renderSelectEditInputCell = (params) => { + return ; + }; const statuses = ["pending", "incomplete", "complete", "Approval to Proceed", "Enabled", "Implemented", "Rejected"] const [columnsForEditor, setColumnsForEditor] = useState([ { @@ -130,12 +178,13 @@ export const RequestsTable = ({ elementId, isOwner }) => { field: 'action', headerName: 'Action', width: 300, - editable: false, - renderCell: (params) => ( -
- Action bar and submit button -
- ), + editable: true, + renderEditCell: renderSelectEditInputCell, + // renderCell: (params) => ( + //
+ // Action bar and submit button + //
+ // ), }, ]); @@ -184,6 +233,7 @@ export const RequestsTable = ({ elementId, isOwner }) => { pageSize={25} rowsPerPageOptions={[25]} rowHeight={50} + experimentalFeatures={{ newEditingApi: true}} checkboxSelection // onSelectionModelChange={(selectionModel, details) => { // console.log(selectionModel, details); From 3468ebe17e3aa300799677f30dc7ddb563f21da7 Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Fri, 22 Apr 2022 13:44:09 -0500 Subject: [PATCH 038/115] wip --- templates/systems/system_summary_1.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 628286442..52079e1b7 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -20,7 +20,7 @@ {% block body_content %}

System Summary 1

- +
{% endblock %} @@ -28,6 +28,7 @@

System Summary 1

{% block scripts %} {{ block.super }} {% endblock %} From dd1422d8f2e1a580eb341bb9677c5f4f0be0278c Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 22 Apr 2022 15:35:35 -0400 Subject: [PATCH 039/115] rough scaffold --- templates/systems/system_summary_1.html | 118 +++++++++++++++++++++++- 1 file changed, 115 insertions(+), 3 deletions(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 628286442..1811df0cd 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -18,10 +18,122 @@ {% block body_content %} -
-

System Summary 1

- + + + +
+
+

Moderate Impact

+

Dairy Billing Inspection Grading System

+

AKA "DBIGS"

+
+ +
Systems-wide Initiative Status +
+
CSAM KeyCAM Value
{{key}}{{value}}
+ + + +
 
800-53 rev5 migration
 
Log4J remediation solution (updated 4/21/22)
 
Q3 Audit
+
+ + + +
+ + +
+

Overview

+ +
+

System Details

+
+ + + + + + +
System ID 23756263482-23
Operational Status Operational
System Type General Support System
System Created 743 days ago
Hosting Facility DISC
+ +
+
+ + + + + + +
Next Scheduled Audit June 21, 2022 (59 days)
Next Scheduled Scan 05/01/22
Security Scan Last known scan - 04/20/22 @ 1:23pm
Penetration Test Scheduled for 05/05/22
Auto-Config Scan Last known scan - 01/17/20 @ 4:32pm
+ +
+ + +
+

Recent System Related Events

+
TEST
+
Penetration test scheduled - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do...
+ +
SCAN
+
Security scan scheduled - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do...
+ +
SYSTEM
+
Isso appointed - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod...
+ + +
+ + +
+ + +
+

System Summary

+

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

+

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

+ +
+

Ownership

+
+
organization:
+

Forest Service

+
System Owner +

Lorem ipsum

+

555-888-1212

+

contact@dfafsdf.com

+ +
+
Technical Contact +

Department of Forgotten Systems

+

252-314-1414

+

admin@sdfsdqfqfqf.com

+ +
+
+ +
+ + +
+ + {% endblock %} From 17364dbf127c6c4bf7f0d6558fa01ecdf524e77a Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 22 Apr 2022 16:45:21 -0400 Subject: [PATCH 040/115] revising --- templates/systems/system_summary_1.html | 62 ++++++++++++++++--------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 1811df0cd..90d747d9a 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -20,6 +20,10 @@ {% block body_content %}
-
+

Moderate Impact

Dairy Billing Inspection Grading System

-

AKA "DBIGS"

+

A.K.A. "DBIGS"

-
Systems-wide Initiative Status -
- - - - +
Systems-wide Initiative Status
+
+
 
800-53 rev5 migration
 
Log4J remediation solution (updated 4/21/22)
 
Q3 Audit
+ + +
 
800-53 rev5 migration
 
Log4J remediation solution (updated 4/21/22)
 
Q3 Audit
@@ -59,11 +68,12 @@

AKA "DBIGS"

-

Overview

+

Overview

-
+

System Details

-
+
+
@@ -73,7 +83,8 @@

System Details

System ID 23756263482-23
Operational Status Operational
-
+ +
@@ -83,6 +94,7 @@

System Details

Next Scheduled Audit June 21, 2022 (59 days)
Next Scheduled Scan 05/01/22
+
@@ -103,24 +115,24 @@

Recent System Related Events

-
+

System Summary

-

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

-

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

+

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

+

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

+
+

Ownership

-

Ownership

-
-
organization:
-

Forest Service

-
System Owner +
Organization
+

Forest Service, Office of CIO (Customer Experience Center)

+
System Owner

Lorem ipsum

555-888-1212

contact@dfafsdf.com

-
Technical Contact +
Technical Contact

Department of Forgotten Systems

252-314-1414

admin@sdfsdqfqfqf.com

@@ -134,6 +146,10 @@
organization:
+
+

Risk Assessment

+ +
{% endblock %} From 3f71f6426a14218ab3e2b7719a77038a435d2f76 Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 22 Apr 2022 17:57:06 -0400 Subject: [PATCH 041/115] further revisions --- templates/systems/system_summary_1.html | 144 +++++++++++++----------- 1 file changed, 77 insertions(+), 67 deletions(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 90d747d9a..544b68ccf 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -21,92 +21,104 @@
-
+

Moderate Impact

Dairy Billing Inspection Grading System

A.K.A. "DBIGS"

-
+
Systems-wide Initiative Status
-
- +
+
-
 
800-53 rev5 migration
 
Log4J remediation solution (updated 4/21/22)
 
Q3 Audit
-
- - - + +
-

Overview

- -
-

System Details

-
-
- - - - - - -
System ID 23756263482-23
Operational Status Operational
System Type General Support System
System Created 743 days ago
Hosting Facility DISC
- -
+

Overview

+ +
+

System Details

+
+
+ + + + + + +
System ID: 23756263482-23
Status: Operational
System Type: General Support System
System Created: 743 days ago
Hosting Facility: DISC
+
+
+ + + + + + +
Next Audit: June 21, 2022 (59 days)
Next Scan: 05/01/22
Security Scan: Last known-04/20/22 @ 1:23pm
Penetration Test: Scheduled for 05/05/22
Auto-Config Scan: Last known-01/17/20 @ 4:32pm
+
+
+
-
- - - - - - -
Next Scheduled Audit June 21, 2022 (59 days)
Next Scheduled Scan 05/01/22
Security Scan Last known scan - 04/20/22 @ 1:23pm
Penetration Test Scheduled for 05/05/22
Auto-Config Scan Last known scan - 01/17/20 @ 4:32pm
-
-
+

Recent System Related Events

+ + + + + + + -
-

Recent System Related Events

-
TEST
-
Penetration test scheduled - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do...
+
+ + -
SCAN
-
Security scan scheduled - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do...
-
SYSTEM
-
Isso appointed - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod...
+
TEST
Penetration test scheduled - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do...
SCAN
Security scan scheduled - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do...
SYS
Isso appointed - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod...
@@ -115,42 +127,40 @@

Recent System Related Events

-
+

System Summary

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.

Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

+
+

Ownership

-

Ownership

-
-
Organization
+
Organization

Forest Service, Office of CIO (Customer Experience Center)

-
System Owner -

Lorem ipsum

-

555-888-1212

-

contact@dfafsdf.com

- +
System Owner +

Lorem ipsum
+555-888-1212
+contact@dfafsdf.com

-
Technical Contact -

Department of Forgotten Systems

-

252-314-1414

-

admin@sdfsdqfqfqf.com

+
Technical Contact +

Department of Forgotten Systems
+252-314-1414
+admin@sdfsdqfqfqf.com

-
- -
+

Risk Assessment

+ {% endblock %} {% block scripts %} From 52d9d743e6c9ebe8f8afafbff9f93cc13b4ef1b5 Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 22 Apr 2022 18:06:37 -0400 Subject: [PATCH 042/115] further revisions --- templates/systems/system_summary_1.html | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 544b68ccf..6e58f845e 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -21,7 +21,7 @@
@@ -144,8 +153,45 @@
Organization
-
+

Risk Assessment

+ + Vulnerabilities     (as of 4:24pm, 04/26/22) + +
+ + + + + + +

12

New in last 30 days

12

Resolved in last 30 days

12

Ongoing (90+ days)
+
+
+ +
+

Vulnerability Overview

+

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Consequat interdum varius sit amet. Auctor elit sed vulputate mi sit amet mauris commodo quis. Vestibulum sed arcu non odio euismod. Adipiscing at in tellus integer feugiat scelerisque varius.

+
+
+
+ + +

8.7

Risk Score
+
9.9Lorem
+
9.2Ipsum
+
8.4Consectetur
+
8.8Adipiscing
+
7.2Aliqua
+ + +
+ + +
+
+ Plan of Actions & Milestones     ( 16 open / 72 resolved ) +
From 63451a9e7c9d447be7b929199428d5dd6df24089 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 26 Apr 2022 11:49:01 -0400 Subject: [PATCH 060/115] tweak --- templates/systems/system_summary_1.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index cb99c7d81..0a53fe826 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -171,7 +171,7 @@

Risk Assessment

Vulnerability Overview

-

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Consequat interdum varius sit amet. Auctor elit sed vulputate mi sit amet mauris commodo quis. Vestibulum sed arcu non odio euismod. Adipiscing at in tellus integer feugiat scelerisque varius.

+

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Consequat interdum varius sit amet. Auctor elit sed vulputate mi sit amet mauris commodo quis. Vestibulum sed arcu non odio euismod. Adipiscing at in tellus integer feugiat scelerisque varius.


From 621da30cd0c86becc776eb124a3caf63680bd05e Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 26 Apr 2022 11:50:21 -0400 Subject: [PATCH 061/115] tweak --- templates/systems/system_summary_1.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 0a53fe826..ecf9b31a7 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -57,6 +57,8 @@ .vuln-header { font-weight:bold; font-size: 1.2em; font-family:Helvetica Neue,"Helvetica","Roboto","Arial",sans-serif;} +.greybg { background-color:#F2F2F2;} + #sys-vuln-table {width:100%;} .vuln-scores { border:1px solid #ccc; margin-right:3px; padding:1em; border-radius: 5px; height:8em;} @@ -177,7 +179,7 @@

Vulnerability Overview

-

8.7

Risk Score
+

8.7

Risk Score
9.9Lorem
9.2Ipsum
8.4Consectetur
From c9c1f2e63f9b2d5679cfbfec8c86c19515fae951 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 26 Apr 2022 11:53:06 -0400 Subject: [PATCH 062/115] tweak --- templates/systems/system_summary_1.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index ecf9b31a7..7ab2927cf 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -59,7 +59,7 @@ .greybg { background-color:#F2F2F2;} -#sys-vuln-table {width:100%;} +#sys-vuln-table {width:100%; font-family:Helvetica Neue,"Helvetica","Roboto","Arial",sans-serif;} .vuln-scores { border:1px solid #ccc; margin-right:3px; padding:1em; border-radius: 5px; height:8em;} .vuln-scores-cat-tot { font-weight: bold; display: block; padding-bottom:2em; font-size:1.2em; font-family:Helvetica Neue,"Helvetica","Roboto","Arial",sans-serif; color:#666666;} @@ -158,7 +158,7 @@
Organization

Risk Assessment

- Vulnerabilities     (as of 4:24pm, 04/26/22) + Vulnerabilities     (as of 4:24pm, 04/26/22)
@@ -192,7 +192,7 @@

Vulnerability Overview

- Plan of Actions & Milestones     ( 16 open / 72 resolved ) + Plan of Action & Milestones     ( 16 open / 72 resolved )
From d4d43487e675df4c69246b28fc0a020571c6ca21 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 26 Apr 2022 11:35:49 -0500 Subject: [PATCH 063/115] Remove stray character from system_summary_1.html --- templates/systems/system_summary_1.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/systems/system_summary_1.html b/templates/systems/system_summary_1.html index 7ab2927cf..ecd4a49b6 100644 --- a/templates/systems/system_summary_1.html +++ b/templates/systems/system_summary_1.html @@ -103,7 +103,7 @@

System Details

- +
Next Audit: {{ system.next_audit }}
Next Scan: {{ system.next_scan|naturaltime }}
Security Scan: {{ system.security_scan }} d
Security Scan: {{ system.security_scan }}
Penetration Test: {{ system.pen_test|naturaltime }}
Auto-Config Scan: {{ system.config_scan }}
From f702d77bd1dd81353b0105e4a545e92ddde59402 Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Tue, 26 Apr 2022 16:21:56 -0500 Subject: [PATCH 064/115] Message outputted for successfully submitting a new request along with a message outputted for an already requested component being requested again --- controls/urls.py | 1 + controls/views.py | 22 +++++++++++++++++++ .../system-owner-approval/component.js | 4 ++-- .../requireApprovalModal.js | 22 +++++++++++++++---- templates/systems/components_selected.html | 8 ++++++- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/controls/urls.py b/controls/urls.py index be18015c2..f55667970 100644 --- a/controls/urls.py +++ b/controls/urls.py @@ -34,6 +34,7 @@ url(r'^editor_autocomplete/', views.EditorAutocomplete.as_view(), name="search_system_component"), url(r'^related_system_components/', views.RelatedComponentStatements.as_view(), name="related_system_components"), url(r'^(?P.*)/components/add_system_component$', views.add_system_component, name="add_system_component"), + url(r'^(?P.*)/components/request_message$', views.request_message, name="request_message"), url(r'^(?P.*)/components/editor_autocomplete$', views.EditorAutocomplete.as_view(), name="editor_autocomplete"), url(r'^(?P.)/profile/oscal/json', views.system_profile_oscal_json, name="profile_oscal_json"), url(r'^statement_history/(?P.*)/$', views.statement_history, name="statement_history"), diff --git a/controls/views.py b/controls/views.py index eea248e3f..9854ee578 100644 --- a/controls/views.py +++ b/controls/views.py @@ -2446,6 +2446,27 @@ def delete_smt(request): # Components +def request_message(request, system_id): + """ Send a global message to indicate a request has been successful """ + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + form_dict = dict(request.POST) + form_values = {} + for key in form_dict.keys(): + form_values[key] = form_dict[key][0] + messageType = form_values['req_message_type'] + message = form_values['req_message'] + + if(messageType == "INFO"): + messages.add_message(request, messages.INFO, f'{message}') + elif(messageType == "WARNING"): + messages.add_message(request, messages.WARNING, f'{message}') + else: + messages.add_message(request, messages.ERROR, f'{message}') + + return HttpResponseRedirect("/systems/{}/components/selected".format(system_id)) + @login_required def add_system_component(request, system_id): """Add an existing element and its statements to a system""" @@ -2463,6 +2484,7 @@ def add_system_component(request, system_id): # Does user have permission to add element? # Check user permissions + # import ipdb; ipdb.set_trace() system = System.objects.get(pk=system_id) if not request.user.has_perm('change_system', system): # User does not have write permissions diff --git a/frontend/src/components/system-owner-approval/component.js b/frontend/src/components/system-owner-approval/component.js index dabd01ed1..e3bd10d39 100644 --- a/frontend/src/components/system-owner-approval/component.js +++ b/frontend/src/components/system-owner-approval/component.js @@ -5,11 +5,11 @@ import { RequireApprovalModal } from './requireApprovalModal'; import { Provider } from "react-redux"; import store from "../../store"; -window.requireApprovalModal = ( userId, systemId, elementId, require_approval ) => { +window.requireApprovalModal = ( userId, systemId, systemName, elementId, require_approval ) => { const uuid = uuid_v4(); ReactDOM.render( - + , document.getElementById('private-component-modal') ); diff --git a/frontend/src/components/system-owner-approval/requireApprovalModal.js b/frontend/src/components/system-owner-approval/requireApprovalModal.js index be4eb0ef2..70de5c84f 100644 --- a/frontend/src/components/system-owner-approval/requireApprovalModal.js +++ b/frontend/src/components/system-owner-approval/requireApprovalModal.js @@ -28,7 +28,6 @@ import { AsyncPagination } from "../shared/asyncTypeahead"; import { red, green } from '@mui/material/colors'; import { ReactModal } from '../shared/modal'; import { hide, show } from '../shared/modalSlice'; -import { element } from 'prop-types'; const datagridStyles = makeStyles({ root: { @@ -64,7 +63,7 @@ const useStyles = makeStyles({ } }); -export const RequireApprovalModal = ({ userId, systemId, elementId, require_approval, uuid }) => { +export const RequireApprovalModal = ({ userId, systemId, systemName, elementId, require_approval, uuid }) => { const classes = useStyles(); const dgClasses = datagridStyles(); const [data, setData] = useState([]); @@ -83,6 +82,20 @@ export const RequireApprovalModal = ({ userId, systemId, elementId, require_appr }); }, [elementId, uuid]) + const successful_request_message = () => { + const message = `System ${systemName} has requested ${data.name}`; + document.getElementById("req_message_type").value = "INFO"; + document.getElementById("req_message").value = message; + document.send_request_message.submit() + } + + const send_alreadyRequested_message = () => { + const message = `System ${systemName} has already requested ${data.name}.`; + document.getElementById("req_message_type").value = "WARNING"; + document.getElementById("req_message").value = message; + document.send_request_message.submit() + } + const handleAddComponent = () => { handleClose(); document.add_component.submit(); @@ -112,12 +125,13 @@ export const RequireApprovalModal = ({ userId, systemId, elementId, require_appr const newRequestResponse = await axios.post(`/api/v2/elements/${elementId}/CreateAndSetRequest/`, newReq); if(newRequestResponse.status === 200){ handleClose(); + successful_request_message(); } else { console.error("Something went wrong in creating and setting new request to element"); } } else { handleClose(); - alert('ALREADY REQUESTED!'); + send_alreadyRequested_message(); } } else { console.error("Something went wrong with checking element"); @@ -126,7 +140,7 @@ export const RequireApprovalModal = ({ userId, systemId, elementId, require_appr const handleClose = async (event) => { setOpenRequireApprovalModal(false); } - + console.log('data: ', data); return (
{data !== null &&
+ + {% csrf_token %} + + +
{% for component in system_elements %} {# Each "component" is a Element model object. #} @@ -84,7 +89,8 @@ const [component_id, require_approval] = value.split(','); const userId = {{request.user.id}}; const systemId = {{system.id}}; - window.requireApprovalModal(userId, systemId, component_id, require_approval === 'True' ? true : false); + const systemName = `{{system.root_element.name}}`; + window.requireApprovalModal(userId, systemId, systemName, component_id, require_approval === 'True' ? true : false); } // Convert select field for Add Component to jQuery Select2 box From 8a1e3f5a4767ee34873b538d4d2799f894b99f05 Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Wed, 27 Apr 2022 10:37:06 -0500 Subject: [PATCH 065/115] User can change status of request in action column of request datagrid --- api/controls/serializers/element.py | 6 +- controls/views.py | 4 +- frontend/src/components/requests/requests.js | 336 ++++++++++++------ .../requireApprovalModal.js | 2 +- 4 files changed, 224 insertions(+), 124 deletions(-) diff --git a/api/controls/serializers/element.py b/api/controls/serializers/element.py index d9ffec7f0..6a2d4acfe 100644 --- a/api/controls/serializers/element.py +++ b/api/controls/serializers/element.py @@ -183,6 +183,7 @@ class ElementRequestsSerializer(ReadOnlySerializer): def get_list_of_requested(self, element): list_of_requests = [] + counter = 1 for request in element.requests.all(): list_of_system_PointOfContacts = [] @@ -194,7 +195,8 @@ def get_list_of_requested(self, element): list_of_requestedElements_PointOfContacts.append(user.party.name) req = { - "id": request.id, + "id": counter, + "requestId": request.id, "userId": request.user.id, "user_name": request.user.name, "user_email": request.user.email, @@ -221,7 +223,7 @@ def get_list_of_requested(self, element): "criteria_reject_comment": request.criteria_reject_comment, "status": request.status, } - + counter += 1 list_of_requests.append(req) return list_of_requests class Meta: diff --git a/controls/views.py b/controls/views.py index 9854ee578..1d4ab53ce 100644 --- a/controls/views.py +++ b/controls/views.py @@ -1373,9 +1373,7 @@ def component_library_component(request, element_id): listOfContacts.append(user) get_all_parties = element.appointments.all() - # import ipdb; ipdb.set_trace() - #TODO: Count element's requests - element.requests.count() - total_number_of_requests = 3 + total_number_of_requests = element.requests.count() # status=RequestStatusEnum.PENDING.name # req_instance = Request.objects.create(user=user, element=element, status="pending") # req_instance.system.set(system) diff --git a/frontend/src/components/requests/requests.js b/frontend/src/components/requests/requests.js index 3ea9967c6..a7da1e54b 100644 --- a/frontend/src/components/requests/requests.js +++ b/frontend/src/components/requests/requests.js @@ -1,18 +1,144 @@ import React, { useEffect, useState } from 'react'; -import { useSelector, useDispatch } from 'react-redux'; -import { DataTable } from '../shared/table'; +import { useDispatch } from 'react-redux'; import axios from 'axios'; -import moment from 'moment'; -import { DataGrid, useGridApiContext } from '@mui/x-data-grid'; -import { v4 as uuid_v4 } from "uuid"; -import PropTypes from 'prop-types'; -import { - Chip, +import { DataGrid } from '@mui/x-data-grid'; +import { + Button, Grid, - Select, - Stack, } from '@mui/material'; import { makeStyles } from '@mui/styles'; +import SelectUnstyled, { selectUnstyledClasses } from '@mui/base/SelectUnstyled'; +import OptionUnstyled, { optionUnstyledClasses } from '@mui/base/OptionUnstyled'; +import PopperUnstyled from '@mui/base/PopperUnstyled'; +import { styled } from '@mui/system'; + +const blue = { + 100: '#DAECFF', + 200: '#99CCF3', + 400: '#3399FF', + 500: '#007FFF', + 600: '#0072E5', + 900: '#003A75', +}; + +const grey = { + 100: '#E7EBF0', + 200: '#E0E3E7', + 300: '#CDD2D7', + 400: '#B2BAC2', + 500: '#A0AAB4', + 600: '#6F7E8C', + 700: '#3E5060', + 800: '#2D3843', + 900: '#1A2027', +}; + +const StyledButton = styled('button')( + ({ theme }) => ` + font-family: IBM Plex Sans, sans-serif; + font-size: 0.875rem; + box-sizing: border-box; + min-height: calc(1.5em + 22px); + min-width: 130px; + background: ${theme.palette.mode === 'dark' ? grey[900] : '#fff'}; + border: 1px solid ${theme.palette.mode === 'dark' ? grey[800] : grey[300]}; + border-radius: 0.75em; + margin: 0.5em; + padding: 10px; + text-align: left; + line-height: 1.5; + color: ${theme.palette.mode === 'dark' ? grey[300] : grey[900]}; + + &:hover { + background: ${theme.palette.mode === 'dark' ? '' : grey[100]}; + border-color: ${theme.palette.mode === 'dark' ? grey[700] : grey[400]}; + } + + &.${selectUnstyledClasses.focusVisible} { + outline: 3px solid ${theme.palette.mode === 'dark' ? blue[600] : blue[100]}; + } + + &.${selectUnstyledClasses.expanded} { + &::after { + content: 'â–´'; + } + } + + &::after { + content: 'â–¾'; + float: right; + } + `, +); + +const StyledListbox = styled('ul')( + ({ theme }) => ` + font-family: IBM Plex Sans, sans-serif; + font-size: 0.875rem; + box-sizing: border-box; + padding: 5px; + margin: 10px 0; + min-width: 320px; + background: ${theme.palette.mode === 'dark' ? grey[900] : '#fff'}; + border: 1px solid ${theme.palette.mode === 'dark' ? grey[800] : grey[300]}; + border-radius: 0.75em; + color: ${theme.palette.mode === 'dark' ? grey[300] : grey[900]}; + overflow: auto; + outline: 0px; + `, +); + +const StyledOption = styled(OptionUnstyled)( + ({ theme }) => ` + list-style: none; + padding: 8px; + border-radius: 0.45em; + cursor: default; + + &:last-of-type { + border-bottom: none; + } + + &.${optionUnstyledClasses.selected} { + background-color: ${theme.palette.mode === 'dark' ? blue[900] : blue[100]}; + color: ${theme.palette.mode === 'dark' ? blue[100] : blue[900]}; + } + + &.${optionUnstyledClasses.highlighted} { + background-color: ${theme.palette.mode === 'dark' ? grey[800] : grey[100]}; + color: ${theme.palette.mode === 'dark' ? grey[300] : grey[900]}; + } + + &.${optionUnstyledClasses.highlighted}.${optionUnstyledClasses.selected} { + background-color: ${theme.palette.mode === 'dark' ? blue[900] : blue[100]}; + color: ${theme.palette.mode === 'dark' ? blue[100] : blue[900]}; + } + + &.${optionUnstyledClasses.disabled} { + color: ${theme.palette.mode === 'dark' ? grey[700] : grey[400]}; + } + + &:hover:not(.${optionUnstyledClasses.disabled}) { + background-color: ${theme.palette.mode === 'dark' ? grey[800] : grey[100]}; + color: ${theme.palette.mode === 'dark' ? grey[300] : grey[900]}; + } + `, +); + +const StyledPopper = styled(PopperUnstyled)` + z-index: 1; +`; + +const CustomSelect = React.forwardRef(function CustomSelect(props, ref) { + const components = { + Root: StyledButton, + Listbox: StyledListbox, + Popper: StyledPopper, + ...props.components, + }; + + return ; +}); const datagridStyles = makeStyles({ root: { @@ -49,116 +175,10 @@ const useStyles = makeStyles({ }); export const RequestsTable = ({ elementId, isOwner }) => { - const dispatch = useDispatch(); - - const classes = useStyles(); const dgClasses = datagridStyles(); const [data, setData] = useState([]); const [sortby, setSortBy] = useState(["name", "asc"]); - - useEffect(() => { - axios(`/api/v2/elements/${elementId}/retrieveRequests/`).then(response => { - setData(response.data.requested); - }); - }, []) - - const handleSubmit = (params) => { - console.log('handleSubmit'); - // { - // "user": 0, - // "system": 0, - // "requested_element": 0, - // "criteria_comment": "string", - // "criteria_reject_comment": "string", - // "status": "string" - // } - // const updatedRequest = { - // user: currentRequest.userId, - // system: currentRequest.system.id, - // requested_element: currentRequest.element.id, - // criteria_comment: currentRequest.criteria_comment, - // criteria_reject_comment: currentRequest.criteria_reject_comment, - // status: currentRequest.status, - // } - // const editRequestResponse = await axios.put(`/api/v2/elements/${elementId}/CreateAndSetRequest/`, updatedRequest); - // if(editRequestResponse.status === 200){ - // handleClose(); - // } else { - // console.error("Something went wrong in creating and setting new request to element"); - // } - } - - const [columnsForEditor, setColumnsForEditor] = useState([ - { - field: 'system', - headerName: 'Requested by', - width: 150, - editable: false, - valueGetter: (params) => params.row.system.name, - }, - { - field: 'point_of_contact', - headerName: 'Point of Contact', - width: 300, - editable: false, - renderCell: (params) => ( -
- {params.row.user_name} {(params.row.user_phone_number) ? `(${params.row.user_phone_number})` : ''} -
- ), - }, - // { - // field: 'point_of_contact', - // headerName: 'Point of Contact', - // width: 300, - // editable: false, - // valueGetter: (params) => params.row.system.point_of_contact, - // }, - // { - // field: 'req_poc', - // headerName: 'RequestedPoint of Contact', - // width: 300, - // editable: false, - // valueGetter: (params) => params.row.requested_element.point_of_contact[0], - // }, - { - field: 'status', - headerName: 'Status', - width: 150, - editable: false, - valueGetter: (params) => params.row.status, - }, - // { - // field: 'action', - // headerName: 'Action', - // width: 300, - // editable: true, - // type: 'text', - // renderCell: (params) => ( - // - // - // - // - // - // - // - // - // ), - // }, - ]); - + const [columnsForEditor, setColumnsForEditor] = useState([]); const [columns, setColumns] = useState([ { field: 'user', @@ -182,10 +202,89 @@ export const RequestsTable = ({ elementId, isOwner }) => { valueGetter: (params) => params.row.status, }, ]); + const handleChange = (event, params, data) => { + const updatedData = [...data]; + updatedData[params.row.id-1].status = event; + setData(updatedData); + } + const handleSubmit = async (params) => { + const updatedRequest = { + user: params.row.userId, + system: params.row.system.id, + requested_element: params.row.requested_element.id, + criteria_comment: params.row.criteria_comment, + criteria_reject_comment: params.row.criteria_reject_comment, + status: params.row.status, + } + const updateRequestResponse = await axios.put(`/api/v2/requests/${params.row.requestId}/`, updatedRequest); + if(updateRequestResponse.status === 200){ + + } else { + console.error("Something went wrong in creating and setting new request to element"); + } + } + useEffect(() => { + axios(`/api/v2/elements/${elementId}/retrieveRequests/`).then(response => { + setData(response.data.requested); + }); + }, []); + + useEffect(() => { + setColumnsForEditor([ + { + field: 'system', + headerName: 'Requested by', + width: 150, + editable: false, + valueGetter: (params) => params.row.system.name, + }, + { + field: 'point_of_contact', + headerName: 'Point of Contact', + width: 300, + editable: false, + renderCell: (params) => ( +
+ {params.row.user_name} {(params.row.user_phone_number) ? `(${params.row.user_phone_number})` : ''} +
+ ), + }, + { + field: 'status', + headerName: 'Status', + width: 150, + editable: false, + valueGetter: (params) => params.row.status, + }, + { + field: 'action', + headerName: 'Action', + width: 300, + editable: true, + type: 'text', + renderCell: (params) => ( + + {data.length !== 0 && + handleChange(event, params, data)}> + Started + Pending + In Progress + Complete + + } + + + + + + ), + }, + ]); + }, [data]) return (
- + {data !== null && columnsForEditor.length !== 0 &&
{
+ }
) } \ No newline at end of file diff --git a/frontend/src/components/system-owner-approval/requireApprovalModal.js b/frontend/src/components/system-owner-approval/requireApprovalModal.js index 70de5c84f..0cae778da 100644 --- a/frontend/src/components/system-owner-approval/requireApprovalModal.js +++ b/frontend/src/components/system-owner-approval/requireApprovalModal.js @@ -109,7 +109,7 @@ export const RequireApprovalModal = ({ userId, systemId, systemName, elementId, systemId: systemId, criteria_comment: data.criteria, criteria_reject_comment: "", - status: "pending" + status: "Pending" } const checkElement = await axios.get(`/api/v2/elements/${elementId}/retrieveRequests/`); From 219c116edcd94a048e1e5e5aa82b41d51f809c84 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Fri, 29 Apr 2022 07:26:14 -0500 Subject: [PATCH 066/115] Fix: Avoid creating duplicate roles during first_run --- siteapp/management/commands/first_run.py | 47 +++++++++++++----------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/siteapp/management/commands/first_run.py b/siteapp/management/commands/first_run.py index 67aca15ee..4cbcee95b 100644 --- a/siteapp/management/commands/first_run.py +++ b/siteapp/management/commands/first_run.py @@ -188,26 +188,29 @@ def handle(self, *args, **options): oscal_component_json = f.read() result = ComponentImporter().import_components_as_json(import_name, oscal_component_json) - Role.objects.create( - role_id="ao", title="Authorizing Official", short_name="AO", - description="Senior federal official or executive with the authority to formally assume responsibility for operating an information system at an acceptable level of risk to organizational operations, other organizations, and the Nation." - ) - Role.objects.create( - role_id="co", title="Component Owner", short_name="CO", description="Business Owner of a Component" - ) - Role.objects.create( - role_id="ccp", title="Common Control Provider", short_name="CCP", description="Business owner of a Common Control" - ) - Role.objects.create( - role_id="iso", title="Information System Owner", short_name="ISO", description="Business Owner of a System" - ) - Role.objects.create( - role_id="isso", title="Information System Security Officer", short_name="ISSO", description="Leads effort to secure a System" - ) - Role.objects.create( - role_id="isse", title="Information System Security Engineer", short_name="ISSE", description="Supports technical engineering to secure a System" - ) - Role.objects.create( - role_id="poc", title="Point of Contact", short_name="PoC", description="Contact for request assistance" - ) + # Create initial roles only once + # TODO: Probably need a field to indicate if first_run has been run to avoid recreating roles that + # installation intentionally deleted. + roles_desired = [ + {"role_id": "ao", "title": "Authorizing Official", "short_name": "AO", "description": "Senior federal official or executive with the authority to formally assume responsibility for operating an information system at an acceptable level of risk to organizational operations, other organizations, and the Nation."}, + {"role_id":"co", "title": "Component Owner", "short_name": "CO", "description": "Business Owner of a Component"}, + {"role_id": "ccp", "title": "Common Control Provider", "short_name": "CCP", "description": "Business owner of a Common Control"}, + {"role_id": "iso", "title": "Information System Owner", "short_name": "ISO", "description": "Business Owner of a System"}, + {"role_id": "isso", "title": "Information System Security Officer", "short_name": "ISSO", "description": "Leads effort to secure a System"}, + {"role_id": "isse", "title": "Information System Security Engineer", "short_name": "ISSE", "description": "Supports technical engineering to secure a System"}, + {"role_id": "poc", "title": "Point of Contact", "short_name": "PoC", "description": "Contact for request assistance"} + ] + roles_to_create = [] + for r in roles_desired: + if not Role.objects.filter(title=r['title']).exists(): + new_role = Role( + role_id=r['role_id'], + title=r['title'], + short_name=r['short_name'], + description=r['description'] + ) + roles_to_create.append(new_role) + if len(roles_to_create) > 0: + roles_created = Role.objects.bulk_create(roles_to_create) + print("GovReady-Q configuration complete.") From 3c9765f68a243af246a28e7c373fc0e50a6d8ef0 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 03:00:24 -0500 Subject: [PATCH 067/115] Fix: Use create_default_portfolio_if_none_exists during oidc auth --- siteapp/authentication/OIDCAuthentication.py | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 4be278ff9..0c26a256d 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -51,8 +51,6 @@ def parse_b64url(self, content): decoded = urlsafe_b64decode(content) return decoded - - def is_admin(self, groups): if settings.OIDC_ROLES_MAP["admin"] in groups: return True @@ -73,8 +71,24 @@ def create_user(self, claims): 'is_staff': False} user = self.UserModel.objects.create_user(**data) - portfolio = Portfolio.objects.create(title=user.email.split('@')[0], description="Personal Portfolio") - portfolio.assign_owner_permissions(user) + if user.default_portfolio is None: + # portfolio = Portfolio.objects.create(title=user.email.split('@')[0], description="Personal Portfolio") + portfolio = user.create_default_portfolio_if_missing() + # portfolio.assign_owner_permissions(user) + # if portfolio: + # # Send a message to site administrators. + # from django.core.mail import mail_admins + # def subvars(s): + # return s.format( + # portfolio=portfolio.title, + # username=user.username, + # email=user.email, + # ) + + # mail_admins( + # subvars("New portfolio: {portfolio} (created by {email})"), + # subvars( + # "A new portfolio has been registered!\n\nPortfolio\n------------\nName: {portfolio}\nRegistering User\n----------------\nUsername: {username}\nEmail: {email}")) return user def update_user(self, user, claims): From eeaeec66769f2610453107e293775315454d2c99 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 03:42:20 -0500 Subject: [PATCH 068/115] Fixing duplicate OIDC users --- siteapp/authentication/OIDCAuthentication.py | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 0c26a256d..0ae30c5fa 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -56,6 +56,39 @@ def is_admin(self, groups): return True return False + # override get_or_create_user method + def get_or_create_user(self, access_token, id_token, payload): + """Returns a User instance if 1 user is found. Creates a user if not found + and configured to do so. Returns nothing if multiple users are matched.""" + + user_info = self.get_userinfo(access_token, id_token, payload) + LOGGER.warning("\n DEBUG user_info:", user_info) + + claims_verified = self.verify_claims(user_info) + if not claims_verified: + msg = 'Claims verification failed' + raise SuspiciousOperation(msg) + + # email based filtering + #users = self.filter_users_by_claims(user_info) + users = User.objects.filter(username=user_info.get('preferred_username', None)) + + if len(users) == 1: + return self.update_user(users[0], user_info) + elif len(users) > 1: + # In the rare case that two user accounts have the same email address, + # bail. Randomly selecting one seems really wrong. + msg = 'Multiple users returned' + raise SuspiciousOperation(msg) + elif self.get_settings('OIDC_CREATE_USER', True): + user = self.create_user(user_info) + return user + else: + LOGGER.debug('Login failed: No user with %s found, and ' + 'OIDC_CREATE_USER is False', + self.describe_user_by_claims(user_info)) + return None + def create_user(self, claims): # data = {'email': claims[settings.OIDC_CLAIMS_MAP['email']], # 'first_name': claims[settings.OIDC_CLAIMS_MAP['first_name']], From d7ad2cca750992ab7d72bededeb95f1ecd1cbcee Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 04:18:21 -0500 Subject: [PATCH 069/115] Fixing duplicate OIDC users. Import User model. --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 0ae30c5fa..9d0e48642 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -12,7 +12,7 @@ from mozilla_django_oidc.utils import absolutify, add_state_and_nonce_to_session from base64 import urlsafe_b64encode, urlsafe_b64decode -from siteapp.models import Portfolio +from siteapp.models import Portfolio, User From 6d743aa2fd0a400c47878a4fdcad443efdaa5460 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 04:32:08 -0500 Subject: [PATCH 070/115] Fixing duplicate OIDC users. More robust claims ref lookup. --- siteapp/authentication/OIDCAuthentication.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 9d0e48642..732e6ce30 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -127,11 +127,11 @@ def create_user(self, claims): def update_user(self, user, claims): original_values = [getattr(user, x.name) for x in user._meta.get_fields() if hasattr(user, x.name)] - user.email = claims[settings.OIDC_CLAIMS_MAP['email']] - user.first_name = claims[settings.OIDC_CLAIMS_MAP['first_name']] - user.last_name = claims[settings.OIDC_CLAIMS_MAP['last_name']] - user.username = claims[settings.OIDC_CLAIMS_MAP['username']] - groups = claims[settings.OIDC_CLAIMS_MAP['groups']] + user.email = claims.get(settings.OIDC_CLAIMS_MAP['email'], None) + user.first_name = claims.get(settings.OIDC_CLAIMS_MAP['first_name'], None) + user.last_name = claims.get(settings.OIDC_CLAIMS_MAP['last_name'], None) + user.username = claims.get(settings.OIDC_CLAIMS_MAP['username'], None) + groups = claims.get(settings.OIDC_CLAIMS_MAP['groups'], None) user.is_staff = self.is_admin(groups) user.is_superuser = user.is_staff From 0a3d0d893fcba7de83496190db5ce65c96b42931 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 04:42:26 -0500 Subject: [PATCH 071/115] Fixing duplicate OIDC users. More robust claims ref lookup 2. --- siteapp/authentication/OIDCAuthentication.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 732e6ce30..96619d230 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -127,11 +127,11 @@ def create_user(self, claims): def update_user(self, user, claims): original_values = [getattr(user, x.name) for x in user._meta.get_fields() if hasattr(user, x.name)] - user.email = claims.get(settings.OIDC_CLAIMS_MAP['email'], None) - user.first_name = claims.get(settings.OIDC_CLAIMS_MAP['first_name'], None) - user.last_name = claims.get(settings.OIDC_CLAIMS_MAP['last_name'], None) - user.username = claims.get(settings.OIDC_CLAIMS_MAP['username'], None) - groups = claims.get(settings.OIDC_CLAIMS_MAP['groups'], None) + user.email = claims.get(settings.OIDC_CLAIMS_MAP['email'], "missing@example.com") + user.first_name = claims.get(settings.OIDC_CLAIMS_MAP['first_name'], "missing first_name") + user.last_name = claims.get(settings.OIDC_CLAIMS_MAP['last_name'], "missing last_name") + user.username = claims.get(settings.OIDC_CLAIMS_MAP['username'], "missing username") + groups = claims.get(settings.OIDC_CLAIMS_MAP['groups'], "missing groups") user.is_staff = self.is_admin(groups) user.is_superuser = user.is_staff From b0423db022e7323bfa4ed4fc564d5ad721029e0a Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 04:58:30 -0500 Subject: [PATCH 072/115] Examining claims info. --- siteapp/authentication/OIDCAuthentication.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 96619d230..1adc05ffb 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -62,17 +62,21 @@ def get_or_create_user(self, access_token, id_token, payload): and configured to do so. Returns nothing if multiple users are matched.""" user_info = self.get_userinfo(access_token, id_token, payload) - LOGGER.warning("\n DEBUG user_info:", user_info) + LOGGER.warning("\n DEBUG user_info (1):", user_info) claims_verified = self.verify_claims(user_info) if not claims_verified: msg = 'Claims verification failed' raise SuspiciousOperation(msg) + LOGGER.warning("\n DEBUG user_info (2):", user_info) + # email based filtering #users = self.filter_users_by_claims(user_info) users = User.objects.filter(username=user_info.get('preferred_username', None)) + LOGGER.warning("\n DEBUG user (3):", users) + if len(users) == 1: return self.update_user(users[0], user_info) elif len(users) > 1: From f815f096b86e7993d0fcbd4c2d4734795531d4fe Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 05:06:10 -0500 Subject: [PATCH 073/115] Examining claims info 2. --- siteapp/authentication/OIDCAuthentication.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 1adc05ffb..5f90df0bb 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -56,6 +56,23 @@ def is_admin(self, groups): return True return False + # override verify_claims to address custom OIDC_RP_SCOPES defined + def verify_claims(self, claims): + """Verify the provided claims to decide if authentication should be allowed.""" + + # Verify claims required by default configuration + scopes = self.get_settings('OIDC_RP_SCOPES', 'openid email') + if 'email' in scopes.split(): + return 'email' in claims + + LOGGER.warning('Custom OIDC_RP_SCOPES defined. ' + 'You need to override `verify_claims` for custom claims verification.') + + # Custom examination of OIDC_RP_SCOPES + LOGGER.warning(f"\n DEBUG custom OIDC_RP_SCOPES (1):", OIDC_RP_SCOPES) + + return True + # override get_or_create_user method def get_or_create_user(self, access_token, id_token, payload): """Returns a User instance if 1 user is found. Creates a user if not found From 0e0ced878836afe3eda48b444b43ece8d7b53964 Mon Sep 17 00:00:00 2001 From: Greg Elin Date: Tue, 3 May 2022 05:16:47 -0500 Subject: [PATCH 074/115] Examining claims info 3. --- siteapp/authentication/OIDCAuthentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteapp/authentication/OIDCAuthentication.py b/siteapp/authentication/OIDCAuthentication.py index 5f90df0bb..78f3dbffe 100644 --- a/siteapp/authentication/OIDCAuthentication.py +++ b/siteapp/authentication/OIDCAuthentication.py @@ -69,7 +69,7 @@ def verify_claims(self, claims): 'You need to override `verify_claims` for custom claims verification.') # Custom examination of OIDC_RP_SCOPES - LOGGER.warning(f"\n DEBUG custom OIDC_RP_SCOPES (1):", OIDC_RP_SCOPES) + # LOGGER.warning(f"\n DEBUG custom OIDC_RP_SCOPES (1):", OIDC_RP_SCOPES) return True From a33f8dff7d2ed92c3beb3e560763e796a674eff0 Mon Sep 17 00:00:00 2001 From: Sergio Falcon Date: Wed, 4 May 2022 10:43:29 -0500 Subject: [PATCH 075/115] System Proposal, System Owner Component Steps of a proposal to request a component --- api/controls/serializers/system.py | 33 +++ api/controls/views/system.py | 40 ++- api/siteapp/serializers/proposal.py | 13 + api/siteapp/urls.py | 2 + api/siteapp/views/proposal.py | 20 ++ controls/migrations/0071_system_proposals.py | 19 ++ controls/models.py | 3 +- controls/views.py | 111 +++++--- .../system-owner-approval/component.js | 10 + .../system-owner-approval/proposal-steps.js | 245 ++++++++++++++++++ .../requireApprovalModal.js | 56 ++-- frontend/src/index.js | 1 + siteapp/migrations/0061_proposal.py | 32 +++ .../migrations/0062_remove_proposal_system.py | 17 ++ siteapp/migrations/0063_auto_20220502_1334.py | 23 ++ siteapp/model_mixins/proposals.py | 24 ++ siteapp/models.py | 23 +- templates/systems/components_selected.html | 47 ++++ templates/systems/element_detail_tabs.html | 13 +- 19 files changed, 654 insertions(+), 78 deletions(-) create mode 100644 api/siteapp/serializers/proposal.py create mode 100644 api/siteapp/views/proposal.py create mode 100644 controls/migrations/0071_system_proposals.py create mode 100644 frontend/src/components/system-owner-approval/proposal-steps.js create mode 100644 siteapp/migrations/0061_proposal.py create mode 100644 siteapp/migrations/0062_remove_proposal_system.py create mode 100644 siteapp/migrations/0063_auto_20220502_1334.py create mode 100644 siteapp/model_mixins/proposals.py diff --git a/api/controls/serializers/system.py b/api/controls/serializers/system.py index 276be8729..b77956def 100644 --- a/api/controls/serializers/system.py +++ b/api/controls/serializers/system.py @@ -1,3 +1,4 @@ +from rest_framework import serializers from rest_framework.relations import PrimaryKeyRelatedField from api.base.serializers.types import ReadOnlySerializer, WriteOnlySerializer @@ -29,3 +30,35 @@ class WriteElementTagsSerializer(WriteOnlySerializer): class Meta: model = System fields = ['tag_ids'] + +class SystemCreateAndSetProposalSerializer(WriteOnlySerializer): + userId = serializers.IntegerField(max_value=None, min_value=None) + elementId = serializers.IntegerField(max_value=None, min_value=None) + criteria_comment = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True) + status = serializers.CharField(min_length=None, max_length=None, allow_blank=True, trim_whitespace=True) + class Meta: + model = System + fields = ['userId', 'elementId', 'criteria_comment','status'] + + +class SystemRetrieveProposalsSerializer(ReadOnlySerializer): + proposals = serializers.SerializerMethodField('get_proposals') + + def get_proposals(self, system): + list_of_proposals = [] + counter = 1 + for proposal in system.proposals.all(): + list_of_proposals.append({ + 'id': counter, + 'proposal_id': proposal.id, + 'user': proposal.user.username, + 'elementId': proposal.requested_element.id, + 'element_name': proposal.requested_element.name, + 'criteria_comment': proposal.criteria_comment, + 'status': proposal.status, + }) + counter += 1 + return list_of_proposals + class Meta: + model = System + fields = ['proposals'] \ No newline at end of file diff --git a/api/controls/views/system.py b/api/controls/views/system.py index e602d20c3..5e650f940 100644 --- a/api/controls/views/system.py +++ b/api/controls/views/system.py @@ -1,20 +1,50 @@ +from rest_framework.decorators import action +from rest_framework.response import Response + from api.base.views.base import SerializerClasses from api.base.views.viewsets import ReadOnlyViewSet from api.controls.serializers.element import SimpleElementControlSerializer, DetailedElementControlSerializer from api.controls.serializers.poam import DetailedPoamSerializer, SimplePoamSerializer -from api.controls.serializers.system import DetailedSystemSerializer, SimpleSystemSerializer +from api.controls.serializers.system import DetailedSystemSerializer, SimpleSystemSerializer, SystemCreateAndSetProposalSerializer, SystemRetrieveProposalsSerializer from api.controls.serializers.system_assement_results import DetailedSystemAssessmentResultSerializer, \ SimpleSystemAssessmentResultSerializer -from controls.models import System, ElementControl, SystemAssessmentResult, Poam +from controls.models import System, Element, ElementControl, SystemAssessmentResult, Poam +from siteapp.models import Proposal, User class SystemViewSet(ReadOnlyViewSet): queryset = System.objects.all() serializer_classes = SerializerClasses(retrieve=DetailedSystemSerializer, - list=SimpleSystemSerializer) - - + list=SimpleSystemSerializer, + CreateAndSetProposal=SystemCreateAndSetProposalSerializer, + retrieveProposals=SystemRetrieveProposalsSerializer,) + + @action(detail=True, url_path="CreateAndSetProposal", methods=["POST"]) + def CreateAndSetProposal(self, request, **kwargs): + system, validated_data = self.validate_serializer_and_get_object(request) + newProposal = Proposal.objects.create( + user=User.objects.get(id=validated_data['userId']), + requested_element=Element.objects.get(id=validated_data['elementId']), + criteria_comment=validated_data['criteria_comment'], + status=validated_data['status'], + ) + newProposal.save() + system.add_proposals([newProposal.id]) + system.save() + serializer_class = self.get_serializer_class('retrieve') + serializer = self.get_serializer(serializer_class, system) + return Response(serializer.data) + + @action(detail=True, url_path="retrieveProposals", methods=["GET"]) + def retrieveProposals(self, request, **kwargs): + system, validated_data = self.validate_serializer_and_get_object(request) + system.save() + + serializer_class = self.get_serializer_class('retrieveProposals') + serializer = self.get_serializer(serializer_class, system) + return Response(serializer.data) + class SystemControlsViewSet(ReadOnlyViewSet): queryset = ElementControl.objects.all() diff --git a/api/siteapp/serializers/proposal.py b/api/siteapp/serializers/proposal.py new file mode 100644 index 000000000..e141ccce8 --- /dev/null +++ b/api/siteapp/serializers/proposal.py @@ -0,0 +1,13 @@ +from api.base.serializers.types import ReadOnlySerializer, WriteOnlySerializer +from siteapp.models import Proposal + + +class SimpleProposalSerializer(ReadOnlySerializer): + class Meta: + model = Proposal + fields = ['user', 'requested_element', 'criteria_comment', 'status'] + +class WriteProposalSerializer(WriteOnlySerializer): + class Meta: + model = Proposal + fields = ['user', 'requested_element', 'criteria_comment', 'status'] diff --git a/api/siteapp/urls.py b/api/siteapp/urls.py index 0ec8ff8bc..33943b468 100644 --- a/api/siteapp/urls.py +++ b/api/siteapp/urls.py @@ -13,6 +13,7 @@ from api.siteapp.views.party import PartyViewSet from api.siteapp.views.appointment import AppointmentViewSet from api.siteapp.views.request import RequestViewSet +from api.siteapp.views.proposal import ProposalViewSet router = routers.DefaultRouter() router.register(r'organizations', OrganizationViewSet) @@ -26,6 +27,7 @@ router.register(r'parties', PartyViewSet) router.register(r'appointments', AppointmentViewSet) router.register(r'requests', RequestViewSet) +router.register(r'proposals', ProposalViewSet) project_router = NestedSimpleRouter(router, r'projects', lookup='projects') diff --git a/api/siteapp/views/proposal.py b/api/siteapp/views/proposal.py new file mode 100644 index 000000000..7025f91b9 --- /dev/null +++ b/api/siteapp/views/proposal.py @@ -0,0 +1,20 @@ +from rest_framework.decorators import action +from rest_framework.response import Response + +from api.base.views.base import SerializerClasses +from api.base.views.viewsets import ReadWriteViewSet +from api.siteapp.serializers.proposal import SimpleProposalSerializer, WriteProposalSerializer +from siteapp.models import Proposal +# from api.siteapp.filters.proposal import ProposalFilter + +class ProposalViewSet(ReadWriteViewSet): + queryset = Proposal.objects.all() + + serializer_classes = SerializerClasses(retrieve=SimpleProposalSerializer, + list=SimpleProposalSerializer, + create=WriteProposalSerializer, + update=WriteProposalSerializer, + destroy=WriteProposalSerializer, + ) + # filter_class = ProposalFilter + diff --git a/controls/migrations/0071_system_proposals.py b/controls/migrations/0071_system_proposals.py new file mode 100644 index 000000000..529abadf0 --- /dev/null +++ b/controls/migrations/0071_system_proposals.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.13 on 2022-04-29 14:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('siteapp', '0062_remove_proposal_system'), + ('controls', '0070_auto_20220418_1140'), + ] + + operations = [ + migrations.AddField( + model_name='system', + name='proposals', + field=models.ManyToManyField(related_name='system', to='siteapp.Proposal'), + ), + ] diff --git a/controls/models.py b/controls/models.py index 9d88e7fad..b8e1318f6 100644 --- a/controls/models.py +++ b/controls/models.py @@ -18,6 +18,7 @@ from siteapp.model_mixins.tags import TagModelMixin from siteapp.model_mixins.appointments import AppointmentModelMixin from siteapp.model_mixins.requests import RequestsModelMixin +from siteapp.model_mixins.proposals import ProposalModelMixin from controls.enums.statements import StatementTypeEnum from controls.enums.remotes import RemoteTypeEnum from controls.oscal import Catalogs, Catalog, CatalogData @@ -649,7 +650,7 @@ def __repr__(self): return "'%s id=%d'" % (self.role, self.id) -class System(auto_prefetch.Model, TagModelMixin): +class System(auto_prefetch.Model, TagModelMixin, ProposalModelMixin): root_element = auto_prefetch.ForeignKey(Element, related_name="system", on_delete=models.CASCADE, help_text="The Element that is this System. Element must be type [Application, General Support System]") fisma_id = models.CharField(max_length=40, help_text="The FISMA Id of the system", unique=False, blank=True, diff --git a/controls/views.py b/controls/views.py index 1d4ab53ce..4881c2932 100644 --- a/controls/views.py +++ b/controls/views.py @@ -49,7 +49,7 @@ from .models import * from .utilities import * from siteapp.utils.views_helper import project_context -from siteapp.models import Role, Party, Appointment, Request +from siteapp.models import Role, Party, Appointment, Request, Proposal logging.basicConfig() import structlog @@ -389,8 +389,12 @@ def get_queryset(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) # Retrieve identified System + system = System.objects.get(id=self.kwargs['system_id']) + system_proposals = [] + for proposal in system.proposals.all(): + system_proposals.append(proposal) # Retrieve related selected controls if user has permission on system if self.request.user.has_perm('view_system', system): # Retrieve primary system Project @@ -398,6 +402,7 @@ def get_context_data(self, **kwargs): project = system.projects.first() context['project'] = project context['system'] = system + context['system_proposals'] = system_proposals context['elements'] = Element.objects.all().exclude(element_type='system') context["display_urls"] = project_context(project) return context @@ -1121,35 +1126,81 @@ def system_element(request, system_id, element_id): # Retrieve impl_smts produced by element and consumed by system # Get the impl_smts contributed by this component to system + # if this is a proposal then we wont have any impl_smts, so we need to check if there is a proposal + # and if impl_smts is empty + # import ipdb; ipdb.set_trace() + impl_smts = element.statements_produced.filter(consumer_element=system.root_element) - - # Retrieve used catalog_key - catalog_key = impl_smts[0].sid_class - - # Retrieve control ids - catalog_controls = Catalog.GetInstance(catalog_key=catalog_key).get_controls_all() - - # Build OSCAL and OpenControl - oscal_string = OSCALComponentSerializer(element, impl_smts).as_json() - opencontrol_string = OpenControlComponentSerializer(element, impl_smts).as_yaml() - states = [choice_tup[1] for choice_tup in ComponentStateEnum.choices()] - types = [choice_tup[1] for choice_tup in ComponentTypeEnum.choices()] - # Return the system's element information - context = { - "states": states, - "types": types, - "system": system, - "project": project, - "element": element, - "impl_smts": impl_smts, - "catalog_controls": catalog_controls, - "catalog_key": catalog_key, - "oscal": oscal_string, - "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol, - "opencontrol": opencontrol_string, - "display_urls": project_context(project) - } - return render(request, "systems/element_detail_tabs.html", context) + # import ipdb; ipdb.set_trace() + + if(impl_smts.exists()): + # Retrieve used catalog_key + catalog_key = impl_smts[0].sid_class + + # Retrieve control ids + catalog_controls = Catalog.GetInstance(catalog_key=catalog_key).get_controls_all() + + # Build OSCAL and OpenControl + oscal_string = OSCALComponentSerializer(element, impl_smts).as_json() + opencontrol_string = OpenControlComponentSerializer(element, impl_smts).as_yaml() + states = [choice_tup[1] for choice_tup in ComponentStateEnum.choices()] + types = [choice_tup[1] for choice_tup in ComponentTypeEnum.choices()] + # Return the system's element information + + context = { + "states": states, + "types": types, + "system": system, + "project": project, + "element": element, + "impl_smts": impl_smts, + "catalog_controls": catalog_controls, + "catalog_key": catalog_key, + "oscal": oscal_string, + "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol, + "opencontrol": opencontrol_string, + "display_urls": project_context(project) + } + return render(request, "systems/element_detail_tabs.html", context) + else: + proposal = system.proposals.get(requested_element__id=element_id) + # TODO:FALCON + #get all statements that are not component_approval_criteria + impl_smts = element.statements_produced.filter(~Q(statement_type='COMPONENT_APPROVAL_CRITERIA')) + # import ipdb; ipdb.set_trace() + # Retrieve used catalog_key + catalog_key = impl_smts[0].sid_class + + # Retrieve control ids + catalog_controls = Catalog.GetInstance(catalog_key=catalog_key).get_controls_all() + # import ipdb; ipdb.set_trace() + # Build OSCAL and OpenControl + oscal_string = OSCALComponentSerializer(element, impl_smts).as_json() + opencontrol_string = OpenControlComponentSerializer(element, impl_smts).as_yaml() + states = [choice_tup[1] for choice_tup in ComponentStateEnum.choices()] + types = [choice_tup[1] for choice_tup in ComponentTypeEnum.choices()] + # Return the system's element information + + requests = Request.objects.filter(system=system, requested_element=element) + hasSentRequest = requests.exists() + # import ipdb; ipdb.set_trace() + context = { + "states": states, + "types": types, + "system": system, + "project": project, + "element": element, + "proposal": proposal, + "hasSentRequest": hasSentRequest, + "impl_smts": impl_smts, + "catalog_controls": catalog_controls, + "catalog_key": catalog_key, + "oscal": oscal_string, + "enable_experimental_opencontrol": SystemSettings.enable_experimental_opencontrol, + "opencontrol": opencontrol_string, + "display_urls": project_context(project) + } + return render(request, "systems/element_detail_tabs.html", context) @login_required def system_element_control(request, system_id, element_id, catalog_key, control_id): @@ -2482,8 +2533,8 @@ def add_system_component(request, system_id): # Does user have permission to add element? # Check user permissions - # import ipdb; ipdb.set_trace() system = System.objects.get(pk=system_id) + if not request.user.has_perm('change_system', system): # User does not have write permissions # Log permission to save answer denied diff --git a/frontend/src/components/system-owner-approval/component.js b/frontend/src/components/system-owner-approval/component.js index e3bd10d39..fc1582ada 100644 --- a/frontend/src/components/system-owner-approval/component.js +++ b/frontend/src/components/system-owner-approval/component.js @@ -2,6 +2,7 @@ import React, {useState} from 'react'; import ReactDOM from 'react-dom'; import {v4 as uuid_v4} from "uuid"; import { RequireApprovalModal } from './requireApprovalModal'; +import { ProposalSteps } from './proposal-steps'; import { Provider } from "react-redux"; import store from "../../store"; @@ -13,4 +14,13 @@ window.requireApprovalModal = ( userId, systemId, systemName, elementId, require , document.getElementById('private-component-modal') ); +}; + +window.proposalSteps = ( userId, systemId, elementId, systemName, elementName, proposalStatus, proposalCriteria, hasSentRequest ) => { + ReactDOM.render( + + + , + document.getElementById('system-owner-proposal-steps') + ); }; \ No newline at end of file diff --git a/frontend/src/components/system-owner-approval/proposal-steps.js b/frontend/src/components/system-owner-approval/proposal-steps.js new file mode 100644 index 000000000..242a8624e --- /dev/null +++ b/frontend/src/components/system-owner-approval/proposal-steps.js @@ -0,0 +1,245 @@ +import React, { useEffect, useState } from 'react'; +import { useSelector, useDispatch } from 'react-redux'; +import { DataTable } from '../shared/table'; +import axios from 'axios'; +import moment from 'moment'; +import { DataGrid } from '@mui/x-data-grid'; +import { v4 as uuid_v4 } from "uuid"; +import { + Button, + Chip, + Grid, + Stack, +} from '@mui/material'; +import { makeStyles } from '@mui/styles'; +import { ListGroup, ListGroupItem } from 'react-bootstrap'; + +const useStyles = makeStyles({ + root: { + fontweight: 900, + }, + completed: { + backgroundColor: '#dae0d2', + '& .dot': { + width: '75px', + height: '75px', + backgroundColor: '#06b30d', + borderRadius: '50%', + display: 'inline-block' + } + }, + current: { + backgroundColor: '#ffffe3', + '& .dot': { + width: '75px', + height: '75px', + backgroundColor: '#ffb404', + borderRadius: '50%', + display: 'inline-block' + } + }, + notStarted: { + backgroundColor: 'white', + '& .dot': { + width: '75px', + height: '75px', + backgroundColor: '#717171', + borderRadius: '50%', + display: 'inline-block' + } + }, +}); + +export const ProposalSteps = ({ userId, systemId, elementId, systemName, elementName, proposalStatus, proposalCriteria, hasSentRequest }) => { + + const [status, setStatus] = useState({ + open: proposalStatus.toLowerCase() === 'open' ? true : false, + planning: proposalStatus.toLowerCase() === 'planning' ? true : false, + request: proposalStatus.toLowerCase() === 'request' ? true : false, + approval: proposalStatus.toLowerCase() === 'approval' ? true : false, + additionalSteps: proposalStatus.toLowerCase() === 'additionalSteps' ? true : false, + }); + const classes = useStyles(status); + + useEffect(() => { + switch(proposalStatus) { + case 'open': + setStatus((prev) => ({ + ...prev, + open: true, + })); + break; + case 'planning': + setStatus((prev) => ({ + ...prev, + planning: true, + })); + break; + case 'request': + setStatus((prev) => ({ + ...prev, + request: true, + })); + break; + case 'approval': + setStatus((prev) => ({ + ...prev, + approval: true, + })); + break; + case 'additionalSteps': + setStatus((prev) => ({ + ...prev, + additionalSteps: true, + })); + break; + default: + setStatus((prev) => ({ + ...prev, + planning: false, + request: false, + approval: false, + additionalSteps: false, + })); + } + }, [proposalStatus]); + const getStatusLevel = (status) => { + console.log('status: ', status) + switch (status.toLowerCase()) { + case 'open': + return 1; + case 'planning': + return 2; + case 'request': + return 3; + case 'approval': + return 4; + case 'additionalSteps': + return 5; + case 'closed': + return 6; + default: + return 0; + } + } + const successful_proposal_message = (systemName, elementName) => { + const message = `System ${systemName} has proposed ${elementName}`; + document.getElementById("req_message_type").value = "INFO"; + document.getElementById("req_message").value = message; + document.send_request_message.submit() + } + + const send_alreadyProposed_message = (systemName, elementName) => { + const message = `System ${systemName} has already proposed ${elementName}.`; + document.getElementById("req_message_type").value = "WARNING"; + document.getElementById("req_message").value = message; + document.send_request_message.submit() + } + + const submitRequest = async () => { + console.log('submitRequest!'); + const newReq = { + userId: userId, + systemId: systemId, + criteria_comment: proposalCriteria, + criteria_reject_comment: "", + status: "open", + } + + const checkElement = await axios.get(`/api/v2/elements/${elementId}/retrieveRequests/`); + if(checkElement.status === 200){ + let alreadyRequested = false; + checkElement.data.requested.map((req) => { + if((req.userId === userId) && (req.requested_element.id === parseInt(elementId)) && (req.system.id === newReq.systemId)){ + alreadyRequested = true; + } + }); + if(!alreadyRequested){ + /* Create a request and assign it to element and system */ + const newRequestResponse = await axios.post(`/api/v2/elements/${elementId}/CreateAndSetRequest/`, newReq); + if(newRequestResponse.status === 200){ + successful_proposal_message(systemName, elementName); + } else { + console.error("Something went wrong in creating and setting new request to element"); + } + } else { + send_alreadyProposed_message(systemName, elementName); + } + } else { + console.error("Something went wrong with checking element"); + } + } + + return ( +
+ + 1 ? classes.completed : classes.notStarted}> + + + + + +

Open

+

Proposal has been opened.

+
{proposalCriteria === '' ? "Criteria has not been set" : proposalCriteria}
+ + + + 2 ? classes.completed : classes.notStarted}> + + + + + +

Planning

+

List

+
{proposalCriteria === '' ? "Criteria has not been set" : proposalCriteria}
+ + + + 3 ? classes.completed : classes.notStarted}> + + + + + +

Request

+
You have requested access to the {elementName} and its related controls.
+
+ {/* + Request button will only appear if + 1. user is not the owner of the element + 2. request has not been previously made + 3. proposal is not currently at the request status stage + */} + {getStatusLevel(proposalStatus) === 3 && hasSentRequest !== true && } +
+ + + + 4 ? classes.completed : classes.notStarted}> + + + + + +

Approval

+
The confirmation fo system using {elementName} from component owner. System can proceed to use the component.
+ + + + 5 ? classes.completed : classes.notStarted}> + + + + + +

Additional Steps

+
Technical team will need to understack various activities to configure your {elementName} (Paas-Server Service).
+ + + + +
+ ); +}; \ No newline at end of file diff --git a/frontend/src/components/system-owner-approval/requireApprovalModal.js b/frontend/src/components/system-owner-approval/requireApprovalModal.js index 0cae778da..fbec3de9c 100644 --- a/frontend/src/components/system-owner-approval/requireApprovalModal.js +++ b/frontend/src/components/system-owner-approval/requireApprovalModal.js @@ -1,15 +1,5 @@ import React, { useEffect, useState } from 'react'; -import { useSelector, useDispatch } from 'react-redux'; -import { DataTable } from '../shared/table'; import axios from 'axios'; -import moment from 'moment'; -import { DataGrid } from '@mui/x-data-grid'; -import { v4 as uuid_v4 } from "uuid"; -import { - Chip, - Grid, - Stack, -} from '@mui/material'; import { makeStyles } from '@mui/styles'; import { Tooltip, @@ -24,10 +14,7 @@ import { Row, Modal } from 'react-bootstrap'; -import { AsyncPagination } from "../shared/asyncTypeahead"; -import { red, green } from '@mui/material/colors'; import { ReactModal } from '../shared/modal'; -import { hide, show } from '../shared/modalSlice'; const datagridStyles = makeStyles({ root: { @@ -82,15 +69,15 @@ export const RequireApprovalModal = ({ userId, systemId, systemName, elementId, }); }, [elementId, uuid]) - const successful_request_message = () => { - const message = `System ${systemName} has requested ${data.name}`; + const successful_proposal_message = () => { + const message = `System ${systemName} has proposed ${data.name}`; document.getElementById("req_message_type").value = "INFO"; document.getElementById("req_message").value = message; document.send_request_message.submit() } - const send_alreadyRequested_message = () => { - const message = `System ${systemName} has already requested ${data.name}.`; + const send_alreadyProposed_message = () => { + const message = `System ${systemName} has already proposed ${data.name}.`; document.getElementById("req_message_type").value = "WARNING"; document.getElementById("req_message").value = message; document.send_request_message.submit() @@ -103,35 +90,34 @@ export const RequireApprovalModal = ({ userId, systemId, systemName, elementId, const handleSubmit = async (event) => { event.preventDefault(); - - const newReq = { + + const newProposal = { userId: userId, - systemId: systemId, + elementId: elementId, criteria_comment: data.criteria, - criteria_reject_comment: "", - status: "Pending" + status: "Open" } - const checkElement = await axios.get(`/api/v2/elements/${elementId}/retrieveRequests/`); - if(checkElement.status === 200){ - let alreadyRequested = false; - checkElement.data.requested.map((req) => { - if((req.userId === userId) && (req.requested_element.id === parseInt(elementId)) && (req.system.id === newReq.systemId)){ - alreadyRequested = true; + const checkSystem = await axios.get(`/api/v2/systems/${systemId}/retrieveProposals/`); + if(checkSystem.status === 200){ + let alreadyProposed = false; + checkSystem.data.proposals.map((req) => { + if(req.elementId === parseInt(elementId)){ + alreadyProposed = true; } }); - if(!alreadyRequested){ + if(!alreadyProposed){ /* Create a request and assign it to element and system */ - const newRequestResponse = await axios.post(`/api/v2/elements/${elementId}/CreateAndSetRequest/`, newReq); + const newRequestResponse = await axios.post(`/api/v2/systems/${systemId}/CreateAndSetProposal/`, newProposal); if(newRequestResponse.status === 200){ handleClose(); - successful_request_message(); + successful_proposal_message(); } else { - console.error("Something went wrong in creating and setting new request to element"); + console.error("Something went wrong in creating and setting new proposal to element"); } } else { handleClose(); - send_alreadyRequested_message(); + send_alreadyProposed_message(); } } else { console.error("Something went wrong with checking element"); @@ -140,7 +126,7 @@ export const RequireApprovalModal = ({ userId, systemId, systemName, elementId, const handleClose = async (event) => { setOpenRequireApprovalModal(false); } - console.log('data: ', data); + return (
{data !== null && Add Component : - + } diff --git a/frontend/src/index.js b/frontend/src/index.js index c86948321..acffed7c5 100644 --- a/frontend/src/index.js +++ b/frontend/src/index.js @@ -11,6 +11,7 @@ import './components/requests/requests'; import './components/system-owner-approval/component'; import './components/system-owner-approval/requireApprovalModal'; +import './components/system-owner-approval/proposal-steps'; import './components/system-summary/component'; import './components/system-summary/system_summary'; diff --git a/siteapp/migrations/0061_proposal.py b/siteapp/migrations/0061_proposal.py new file mode 100644 index 000000000..132868504 --- /dev/null +++ b/siteapp/migrations/0061_proposal.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.13 on 2022-04-29 13:24 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('controls', '0070_auto_20220418_1140'), + ('siteapp', '0060_auto_20220418_1508'), + ] + + operations = [ + migrations.CreateModel( + name='Proposal', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated', models.DateTimeField(auto_now=True, db_index=True, null=True)), + ('criteria_comment', models.TextField(blank=True, help_text='Comments on this request.', null=True)), + ('status', models.TextField(blank=True, help_text='Status of the request.', null=True)), + ('requested_element', models.ForeignKey(blank=True, help_text='Element being proposed for request.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='propose', to='controls.element')), + ('system', models.ForeignKey(blank=True, help_text='System making the request proposal.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='propose', to='controls.system')), + ('user', models.ForeignKey(blank=True, help_text='User creating the request proposal.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='propose', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/siteapp/migrations/0062_remove_proposal_system.py b/siteapp/migrations/0062_remove_proposal_system.py new file mode 100644 index 000000000..2c7f600bc --- /dev/null +++ b/siteapp/migrations/0062_remove_proposal_system.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.13 on 2022-04-29 14:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('siteapp', '0061_proposal'), + ] + + operations = [ + migrations.RemoveField( + model_name='proposal', + name='system', + ), + ] diff --git a/siteapp/migrations/0063_auto_20220502_1334.py b/siteapp/migrations/0063_auto_20220502_1334.py new file mode 100644 index 000000000..66b83e675 --- /dev/null +++ b/siteapp/migrations/0063_auto_20220502_1334.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-05-02 13:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('siteapp', '0062_remove_proposal_system'), + ] + + operations = [ + migrations.AlterField( + model_name='proposal', + name='criteria_comment', + field=models.TextField(blank=True, help_text='Comments on this proposal.', null=True), + ), + migrations.AlterField( + model_name='proposal', + name='status', + field=models.TextField(blank=True, help_text='Status of the proposal.', null=True), + ), + ] diff --git a/siteapp/model_mixins/proposals.py b/siteapp/model_mixins/proposals.py new file mode 100644 index 000000000..9493bc33d --- /dev/null +++ b/siteapp/model_mixins/proposals.py @@ -0,0 +1,24 @@ +from django.db import models +from django.http import JsonResponse + +class ProposalModelMixin(models.Model): + proposals = models.ManyToManyField("siteapp.Proposal", related_name="%(class)s") + + class Meta: + abstract = True + + def add_proposals(self, proposals): + if proposals is None: + proposals = [] + elif isinstance(proposals, str): + proposals = [proposals] + assert isinstance(proposals, list) + self.proposals.add(*proposals) + + def remove_proposals(self, proposals=None): + if proposals is None: + proposals = [] + elif isinstance(proposals, str): + proposals = [proposals] + assert isinstance(proposals, list) + self.proposals.remove(*proposals) \ No newline at end of file diff --git a/siteapp/models.py b/siteapp/models.py index 3901e9a61..6a38eb292 100644 --- a/siteapp/models.py +++ b/siteapp/models.py @@ -1470,19 +1470,30 @@ class Request(BaseModel): criteria_reject_comment = models.TextField(blank=True, null=True, help_text="Comment on request rejection.") status = models.TextField(blank=True, null=True, help_text="Status of the request.") - # user = models.ForeignKey(User, on_delete=models.CASCADE, help_text="User creating the request.") - # system = models.ForeignKey(System, on_delete=models.CASCADE, help_text="System making the request.") - # requested_element = models.ForeignKey(Element, on_delete=models.CASCADE, help_text="Element being requested.") - def __repr__(self): - return f"{self.system} -> {self.requested_element} - {self.status}" + return f"{self.system} requesting -> {self.requested_element} - {self.status}" def __str__(self): - return f"{self.system} -> {self.requested_element} - {self.status}" + return f"{self.system} requesting -> {self.requested_element} - {self.status}" def serialize(self): return {"system": self.system, "requested_element": self.requested_element, "id": self.id} +class Proposal(BaseModel): + user = models.ForeignKey(User, blank=True, null=True, related_name="propose", on_delete=models.CASCADE, help_text="User creating the request proposal.") + requested_element = models.ForeignKey(Element, blank=True, null=True, related_name="propose", on_delete=models.CASCADE, help_text="Element being proposed for request.") + criteria_comment = models.TextField(blank=True, null=True, help_text="Comments on this proposal.") + status = models.TextField(blank=True, null=True, help_text="Status of the proposal.") + + def __repr__(self): + return f"Proposing request -> {self.requested_element} - {self.status}" + + def __str__(self): + return f"Proposing request -> {self.requested_element} - {self.status}" + + def serialize(self): + return {"requested_element": self.requested_element, "id": self.id} + class Asset(BaseModel): UPLOAD_TO = None # Should be overriden when iheritted title = models.CharField(max_length=255, help_text="The title of this asset.") diff --git a/templates/systems/components_selected.html b/templates/systems/components_selected.html index db17344b0..d18e41087 100644 --- a/templates/systems/components_selected.html +++ b/templates/systems/components_selected.html @@ -42,7 +42,9 @@ +
+ {% for component in system_elements %} {# Each "component" is a Element model object. #}
@@ -74,7 +76,52 @@ {% endif %}
{% endfor %} + +
+ +
+ +
+
+
Proposed components
+
 
+
 
+
+ {% for proposal in system_proposals %} + {# Each "component" is a Element model object. #} +
+
+ {{ proposal.requested_element.name }} +
+
+ {{ proposal.requested_element.component_type }} + {{ proposal.requested_element.component_state }} +
+
+ {% if proposal.requested_element.description %}{{ proposal.requested_element.description }}{% else %}No description provided.{% endif %} +
{% for tag in proposal.requested_element.tags.all %}{{ tag.label }} {% endfor %}
+
+ {% with ctl_count=proposal.requested_element.get_control_impl_smts_prototype_count %} +
+ {% if ctl_count %}{{ ctl_count }} control{{ ctl_count|pluralize }}{% else %}None{% endif %} +
+ {% endwith %} + {% get_obj_perms request.user for system as "system_perms" %} + {% if "change_system" in system_perms %} + + {% endif %} +
+ + {% endfor %} +
+
{% include 'components/paginate_comp.html' with system_elements=page_obj %}
diff --git a/templates/systems/element_detail_tabs.html b/templates/systems/element_detail_tabs.html index 5e15fdc25..cb2b20d01 100644 --- a/templates/systems/element_detail_tabs.html +++ b/templates/systems/element_detail_tabs.html @@ -87,6 +87,9 @@ {% block body_content %}
+
+ Access Required ({{proposal.created}}) | Current status: {{proposal.status}} stage +

{{ element.name }} System Component @@ -127,6 +130,14 @@

About {{ element.name }}

Read more about {{ element.name }} in component library
+ +
+ +

{{ element.name }} contributes {{ impl_smts|length }} statements to controls

@@ -309,7 +320,7 @@

{{ element.name }} contributes {{ impl_smts|length }} statements to controls {% block scripts %} {{block.super}} +