diff --git a/CHANGELOG.md b/CHANGELOG.md index 0343f9c1a..c790f0ad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Release adds support for private components and integrations with third party se * Display the control framework along side of controls in component control listing page. * Remove icons from project listing. * Add Component search filter to filter results to components owned by user. +* Display project control text partially on page with modal for full text. **Developer changes** @@ -49,6 +50,7 @@ Release adds support for private components and integrations with third party se * Assign owners to components imported via OSCAL. If no user is identified during component (element creation) assign first Superuser (administrator) as component owner. * Support navigating to specific tab on component library component page using URL hash (#) reference. * Protype integrations System Summary page. +* Add Django admin command to create shell integration directory. * Refactor and OIDC authentication for proper testing of admin and not admin roles. * Create a new system via name given by a string in URL. * Add a large set of sample components (150+) generated from STIGs. diff --git a/controls/management/commands/importcomponents.py b/controls/management/commands/importcomponents.py index 23a0a98b4..77960e3d0 100644 --- a/controls/management/commands/importcomponents.py +++ b/controls/management/commands/importcomponents.py @@ -1,24 +1,15 @@ -import sys import os.path - from django.core.management import call_command -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction, models -from django.db.utils import OperationalError -from django.conf import settings +from django.core.management.base import BaseCommand from pathlib import Path -from pathlib import PurePath -from django.utils.text import slugify # from siteapp.models import User, Organization, Portfolio -from controls.models import Element, Statement # from controls.views import system_element_download_oscal_json -from controls.views import OSCALComponentSerializer, ComponentImporter - -import fs, fs.errors +from controls.views import ComponentImporter class Command(BaseCommand): + """Import directory of component files""" help = 'Import directory of component files.' def add_arguments(self, parser): @@ -28,7 +19,6 @@ def add_arguments(self, parser): parser.add_argument('--stopinvalid', default=True, action='store_true') parser.add_argument('--no-stopinvalid', dest='stopinvalid', action='store_false') - def handle(self, *args, **options): # Configure diff --git a/controls/views.py b/controls/views.py index de6b25ad8..818db7c35 100644 --- a/controls/views.py +++ b/controls/views.py @@ -1,6 +1,5 @@ import functools import logging -import operator import pathlib import random import shutil @@ -54,6 +53,7 @@ from .utilities import * from siteapp.utils.views_helper import project_context from integrations.models import Integration +from integrations.utils.integration import get_control_data_enhancements logging.basicConfig() import structlog @@ -2298,7 +2298,8 @@ def editor(request, system_id, catalog_key, cl_id): # Define status options impl_statuses = ["Not implemented", "Planned", "Partially implemented", "Implemented", "Unknown"] - # Only elements for the given control id, sid, and statement type + # Only elements for the given control id, sid, and statement type + control_matrix = get_control_data_enhancements(request, catalog_key, cl_id) elements = Element.objects.all().exclude(element_type='system') @@ -2307,6 +2308,7 @@ def editor(request, system_id, catalog_key, cl_id): "project": project, "catalog": catalog, "control": cg_flat[cl_id.lower()], + "control_matrix": control_matrix, "impl_smts": impl_smts, "impl_statuses": impl_statuses, "impl_smts_legacy": impl_smts_legacy, @@ -4072,7 +4074,7 @@ def system_summary_1_aspen(request, system_id): context = { "system": system_summary, #"project": project, - "project": projects, + "projects": projects, "system_events": system_events, # "deployments": deployments, "display_urls": project_context(project) diff --git a/integrations/controlmatrix/README.md b/integrations/controlmatrix/README.md new file mode 100644 index 000000000..844a55326 --- /dev/null +++ b/integrations/controlmatrix/README.md @@ -0,0 +1,52 @@ +# ABOUT Integration Example + +## Configure + +### Step 1: Create an Integration record in Django admin: + +- Name: controlmatrix +- Description: Integration to support Example service +- Config: +```json +{ + "base_url": "http://controlmatrix-test.agency.gov/controlmatrix/api", + "personal_access_token": "" +} +``` +- Config schema: +```json +{} +``` + +For local dev and testing, create an Integration record in Django admin for CSAM mock service: + +- Name: controlmatrix +- Description: Integration to support Example service +- Config: +```json +{ + "base_url": "http://localhost:9009", + "personal_access_token": "FAD619" +} +``` +- Config schema: +```json +{} +``` + +### Step 2: Add route to `integration.urls.py` + +```python +url(r"^controlmatrix/", include("integrations.controlmatrix.urls")), +``` + +## Testing with Mock Service + +The Example integration includes a mock Example service you can launch in the terminal to test your integration. + +To launch the mock service do the following in a separate terminal from the root directory of GovReady-Q: + +```python +pip install click +python integrations/controlmatrix/mock.py +``` \ No newline at end of file diff --git a/integrations/controlmatrix/__init__.py b/integrations/controlmatrix/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/controlmatrix/assets/data/controls_matrix.xlsx b/integrations/controlmatrix/assets/data/controls_matrix.xlsx new file mode 100644 index 000000000..d11a87997 Binary files /dev/null and b/integrations/controlmatrix/assets/data/controls_matrix.xlsx differ diff --git a/integrations/controlmatrix/communicate.py b/integrations/controlmatrix/communicate.py new file mode 100644 index 000000000..1f9532370 --- /dev/null +++ b/integrations/controlmatrix/communicate.py @@ -0,0 +1,125 @@ +import requests +import json +from base64 import b64encode +from urllib.parse import urlparse +from integrations.utils.integration import Communication +from integrations.models import Integration, Endpoint +import pandas +import pathlib + + +class ControlmatrixCommunication(Communication): + + DESCRIPTION = { + "name": "Controlmatrix", + "description": "Controlmatrix Service", + "version": "0.1", + "integration_db_record": False, + "mock": { + "base_url": "http:/localhost:9009", + "personal_access_token": None + } + } + + def __init__(self, **kwargs): + assert self.DESCRIPTION, "Developer must assign a description dict" + self.__is_authenticated = False + self.error_msg = {} + self.auth_dict = {} + self.data = None + self.base_url = "https://controlmatrix.com/api" + + def identify(self): + """Identify which Communication subclass""" + identity_str = f"This is {self.DESCRIPTION['name']} version {self.DESCRIPTION['version']}" + print(identity_str) + return identity_str + + def setup(self, **kwargs): + pass + + def get_response(self, endpoint, headers=None, verify=False): + response = requests.get(f"{self.base_url}{endpoint}") + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + + def authenticate(self, user=None, passwd=None): + """Authenticate with service""" + pass + + @property + def is_authenticated(self): + return self.__is_authenticated + + @is_authenticated.setter + def is_authenticated(self, value): + self.__is_authenticated = value + + def extract_data(self, authentication, identifiers): + """Extract data""" + try: + data = [] + rows_list = [] + # TODO: Fix file path + fn = "integrations/controlmatrix/assets/data/controls_matrix.xlsx" + if pathlib.Path(fn).is_file(): + try: + df_dict = pandas.read_excel(fn, 'controls_foundation', header=0) + for index, row in df_dict.iterrows(): + row_dict = { + "CONTROL_STATUS": row.get('CONTROL_STATUS', ""), + "MONITORED_STATUS": row.get('MONITORED_STATUS', ""), + "TIER": row.get('TIER', ""), + "REVIEW_FREQUENCY": row.get('REVIEW_FREQUENCY', ""), + "CONTROL_TYPE": row.get('CONTROL_TYPE', ""), + "AUTOMATED": row.get('AUTOMATED', ""), + "COMMON_CONTROL": row.get('COMMON_CONTROL', ""), + "PRIORITY": row.get('PRIORITY', ""), + "BASELINE_IMPACT": row.get('BASELINE_IMPACT', ""), + "RMF_CONTROL_FAMILY": row.get('RMF_CONTROL_FAMILY', ""), + "RMF_CONTROL": row.get('RMF_CONTROL', ""), + "RMF_CONTROL_NAME": row.get('RMF_CONTROL_NAME', ""), + "CSF_FUNCTION": row.get('CSF_FUNCTION', ""), + "CSF_CATEGORY": row.get('CSF_CATEGORY', ""), + "CSF_SUBCATEGORY": row.get('CSF_SUBCATEGORY', ""), + "CYBERSECURITY_PROGRAM_DOMAIN": row.get('CYBERSECURITY_PROGRAM_DOMAIN', ""), + "ORTB": row.get('ORTB', ""), + "FSA": row.get('FSA', ""), + "HVA": row.get('HVA', ""), + "FPKI": row.get('FPKI', ""), + "PRIVACY": row.get('PRIVACY', ""), + "CLOUD_SHARED": row.get('CLOUD_SHARED', ""), + "FAST_TRACK": row.get('FAST_TRACK', "") + } + rows_list.append(row_dict) + except FileNotFoundError as e: + print(f"Error reading file {fn}: {e}") + # logger.error(f"Error reading file {fn}: {e}") + except Exception as e: + # logger.error(f"Other error reading file {fn}: {e}") + print(f"Other error reading file {fn}: {e}") + else: + print(f"Failed to find or open file '{fn}'.") + + print(f"identifiers: ", identifiers) + # for ctl in rows_list: + # print(ctl['RMF_CONTROL'].lower()) + # if ctl['RMF_CONTROL'].lower() in identifiers: + # data.append(ctl) + control_matrix = next((ctl for ctl in rows_list if ctl['RMF_CONTROL'].lower() in identifiers), None) + data.append(control_matrix) + except: + data = [] + return data + + def transform_data(self, data, system_id=None, title=None, description=None, deployment_uuid=None): + pass + + def load_data(self, data): + pass diff --git a/integrations/controlmatrix/mock.py b/integrations/controlmatrix/mock.py new file mode 100644 index 000000000..8330c2e73 --- /dev/null +++ b/integrations/controlmatrix/mock.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +############################################################## +# +# A simple Python webserver to generate mock Integration API results +# +# Requirements: +# +# pip install click +# +# Usage: +# +# # Start mock service +# python3 integrations/controlmatrix/mock.py +# +# Accessing: +# +# curl localhost:9009/endpoint +# curl -X 'GET' 'http://127.0.0.1:9009/v1/system/111' +# curl localhost:9002/v1/system/111 # requires authentication +# +############################################################## + +# Parse command-line arguments +import click + +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse, parse_qs +import json +from abc import ABC +#from urllib.parse import urlparse +from django.utils import timezone +from time import time + + +PORT = 9002 +MOCK_SRVC = "CSAM" + +class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + + # Mock data + SYSTEMS = { "111": {"id": 111, + "someKey": "Some Value" + }, + "222": {"id": 222, + "someKey": "Some Value" + } + } + + def mk_system_info_response(self, system_id): + return self.SYSTEMS[system_id] + + def do_GET(self, method=None): + """Parse and route GET request""" + # Parse path + request = urlparse(self.path) + params = parse_qs(request.query) + print(f"request.path: {request.path}, params: {params}") + # params are received as arrays, so get first element in array + # system_id = params.get('system_id', [0])[0] + + # Route GET request + if request.path == "/v1/test/hello": + """Reply with 'hello'""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message":"hello"} + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif request.path == "/v1/test/authenticate-test": + """Test authentication""" + # Test authentication by reading headers and looking for 'Authentication' + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + # Read headers + if 'Authorization' in self.headers: + print("Authorization header:", self.headers['Authorization']) + data = {"reply": "Success", + "Authorization": self.headers['Authorization'], + "pat": self.headers['Authorization'].split("Bearer ")[-1] + } + else: + data = {"reply": "Fail", + "Authorization": None, + "pat": None + } + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif request.path == "/v1/systems/111" or request.path == "/v1/systems/222": + """Reply with system information""" + + system_id = re.search(r"/v1/systems/([0-9]{1,8})", request.path).group(1).strip() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + if '111' in request.path: + data = self.mk_system_info_response('111') + elif '222' in request.path: + data = self.mk_system_info_response('222') + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + else: + """Reply with Path not found""" + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message":"Path not found"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + def do_POST(self): + """Parse and route POST request""" + request = urlparse(self.path) + params = parse_qs(request.query) + print("request.path:", request.path) + print("** params", params) + + # Route POST request + if request.path == "/v1/test/hello": + """Reply with 'hello'""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message": "hello, POST"} + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + if request.path == "/v1/systems/111" or request.path == "/v1/systems/222": + """Update system information""" + # # unauthorized controlmatrix: + # curl -X 'POST' 'http://localhost:9002/system/111' \ + # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' + + content_length = int(self.headers['Content-Length']) + self.post_data = self.rfile.read(content_length) + self.post_data_json = json.loads(self.post_data) + if '111' in request.path: + system_id = '111' + elif '222' in request.path: + system_id = '222' + self.SYSTEMS[system_id]['name'] = self.post_data_json.get('name', self.SYSTEM['name']) + self.SYSTEM[system_id]['purpose'] = self.post_data_json.get('purpose', "MISSING CONTENT") + # Send response + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = self.mk_system_info_response(system_id) + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + else: + # Path not found + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message":"Path not found"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + +def main(): + httpd = HTTPServer(('localhost', PORT), SimpleHTTPRequestHandler) + httpd.serve_forever() + +if __name__ == "__main__": + main() diff --git a/integrations/controlmatrix/urls.py b/integrations/controlmatrix/urls.py new file mode 100644 index 000000000..25769de44 --- /dev/null +++ b/integrations/controlmatrix/urls.py @@ -0,0 +1,17 @@ +from django.conf.urls import url +from . import views + +urlpatterns = [ + url(r"^identify$", views.integration_identify, name='integration_identify'), + url(r"^endpoint(?P.*)$", views.integration_endpoint, name='integration_endpoint'), # Ex: /integrations/example/endpoint/system/111 + url(r"^post_endpoint(?P.*)$", views.integration_endpoint_post, name='integration_endpoint'), + + url(r"^get_system_info_test/(?P.*)$", views.get_system_info, name='get_system_info_test'), + url(r"^get_multiple_system_info/(?P.*)$", views.get_multiple_system_info, name='get_multiple_system_info'), + url(r"^system/(?P.*)$", views.system_info, name='csam_system_info'), + url(r"^update_system_description_test/(?P.*)$", views.update_system_description_test, name='update_system_description_test'), + url(r"^update_system_description$", views.update_system_description, name='update_system_description'), + + url(r"^create_system_from_remote/(?P.*)$", views.create_system_from_remote, name='create_system_from_remote'), + +] diff --git a/integrations/controlmatrix/views.py b/integrations/controlmatrix/views.py new file mode 100644 index 000000000..228c690ee --- /dev/null +++ b/integrations/controlmatrix/views.py @@ -0,0 +1,327 @@ +import json +import time +from datetime import datetime +from django.shortcuts import get_object_or_404, render +from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect +from django.db.models import Q +from integrations.models import Integration, Endpoint +from controls.models import System +from siteapp.models import Organization, Portfolio +from siteapp.views import get_compliance_apps_catalog_for_user, start_app +from guidedmodules.app_loading import ModuleDefinitionError +from .communicate import ControlmatrixCommunication + +# Change 'example' to name of integration, e.g., 'csam' and 'csam_system_id' +INTEGRATION_NAME = 'controlmatrix' +INTEGRATION_SYSTEM_ID = 'controlmatrix_system_id' + +try: + INTEGRATION = get_object_or_404(Integration, name=INTEGRATION_NAME) +except: + HttpResponseNotFound(f'

404 - Integration configuration missing. Create Integration database record.

') + +def set_integration(): + return ControlmatrixCommunication() + +def integration_identify(request): + """Integration returns an identification""" + + communication = set_integration() + return HttpResponse(f"Attempting to communicate with {INTEGRATION_NAME} integration: {communication.identify()}") + +def integration_endpoint(request, endpoint=None): + """Communicate with an integrated service""" + + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def integration_endpoint_post(request, endpoint=None): + """Communicate with an integrated service using POST""" + + post_data = { + "name": "My IT System2", + "description": "This is a more complex test system" + } + communication = set_integration() + data = communication.post_response(endpoint, data=json.dumps(post_data)) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate using POST with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

endpoint: {endpoint}.

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def get_system_info(request, system_id=2): + """Retrieve the system information from CSAM""" + + # system = System.objects.get(pk=system_id) + system = get_object_or_404(System, pk=system_id) + # TODO: Check user permission to view + integration_system_id = system.info.get(INTEGRATION_SYSTEM_ID, None) + if integration_system_id is None: + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

System '{system_id}' does not have an associated '{integration_system_id}'.

" + f"") + + endpoint = f'/v1/systems/{integration_system_id}' + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def system_info(request, system_id=2): + """Retrieve the system information from CSAM""" + system = get_object_or_404(System, pk=system_id) + # try: + # # system = System.objects.get(pk=system_id) + # system = get_object_or_404(System, pk=system_id) + # except: + # return HttpResponse( + # f"" + # f"

now: {datetime.now()}

" + # f"

System '{system_id}' does not exist.

" + # f"") + + # TODO: Check user permission to view + communication = set_integration() + integration_system_id = system.info.get(INTEGRATION_SYSTEM_ID, None) + if integration_system_id is None: + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

System '{system_id}' does not have an associated '{integration_system_id}'.

" + f"") + + endpoint = f'/v1/systems/{integration_system_id}' + # is there local information? + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + # TODO: Refresh data if empty + if created: + # Cache not available + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep.data = data + ep.save() + else: + # Cache available + cached = True + pass + + context = { + "system": system, + "cached": True, + "communication": communication, + "ep": ep + } + from siteapp import settings + # settings.TEMPLATES[0]['DIRS'].append('/Users/gregelinadmin/Documents/workspace/govready-q-private/integrations/{INTEGRATION_NAME}/templates/') + # print(2,"========= TEMPLATES", settings.TEMPLATES[0]['DIRS']) + return render(request, "{INTEGRATION_NAME}/system.html", context) + +def get_multiple_system_info(request, system_id_list="1,2"): + """Get and cach system info for multiple systems""" + systems_updated = [] + systems = System.objects.filter(pk__in=system_id_list.split(",")) + for s in systems: + integration_system_id = s.info.get(INTEGRATION_SYSTEM_ID, None) + if integration_system_id is None: + print(f"System id {s.id} has no example_system_id") + else: + endpoint = f'/v1/systems/{integration_system_id}' + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + msg = f"System id {s.id} info updated from {INTEGRATION_NAME} system {integration_system_id}" + print(msg) + systems_updated.append(msg) + time.sleep(0.25) + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"get_multiple_system_info for system ids {system_id_list}

" + f"

now: {datetime.now()}

" + f"

Result:

" + f"
{systems_updated}
" + f"") + +def update_system_description_test(request, system_id=2): + """Test updating system description in CSAM""" + + params={"src_obj_type": "system", "src_obj_id": system_id} + data = update_system_description(params) + return HttpResponse( + f"

Attempting to update CSAM description of System id {system_id}...' " + f"

now: {datetime.now()}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def update_system_description(request, params={"src_obj_type": "system", "src_obj_id": 2}): + """Update System description in CSAM""" + + system_id = params['src_obj_id'] + system = System.objects.get(pk=system_id) + # TODO: Check user permission to update + integration_system_id = system.info.get(INTEGRATION_SYSTEM_ID, None) + # print("10, ========== integration_system_id", integration_system_id) + if example_system_id is not None: + new_description = "This is the new system description." + endpoint = f"/v1/systems/{integration_system_id}" + post_data = { + "description": new_description + } + communication = set_integration() + data = communication.post_response(endpoint, data=json.dumps(post_data)) + result = data + return result + +def create_system_from_remote(request, remote_system_id): + """Create a system in GovReady-Q based on info from integrated service""" + + print("Create a system in GovReady-Q based on info from integrated service") + + communication = set_integration() + integration_system_id = int(remote_system_id) + endpoint = f'/v1/systems/{integration_system_id}' + # is there local information? + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + # TODO: Refresh data if empty + if created: + # Cache not available + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep.data = data + ep.save() + else: + # Cache available + cached = True + pass + + # Check if system aleady exists with example_system_id + if not System.objects.filter(Q(info__contains={INTEGRATION_SYSTEM_ID: integration_system_id})).exists(): + # Create new system + # What is default template? + source_slug = "govready-q-files-startpack" + app_name = "speedyssp" + # can user start the app? + # Is this a module the user has access to? The app store + # does some authz based on the organization. + catalog = get_compliance_apps_catalog_for_user(request.user) + for app_catalog_info in catalog: + if app_catalog_info["key"] == source_slug + "/" + app_name: + # We found it. + break + else: + raise Http404() + # Start the most recent version of the app. + appver = app_catalog_info["versions"][0] + organization = Organization.objects.first() # temporary + folder = None + task = None + q = None + # Get portfolio project should be included in. + if request.GET.get("portfolio"): + portfolio = Portfolio.objects.get(id=request.GET.get("portfolio")) + else: + if not request.user.default_portfolio: + request.user.create_default_portfolio_if_missing() + portfolio = request.user.default_portfolio + try: + project = start_app(appver, organization, request.user, folder, task, q, portfolio) + except ModuleDefinitionError as e: + error = str(e) + # Associate System with INTEGRATION_NAME system + new_system = project.system + new_system.info = {INTEGRATION_SYSTEM_ID: integration_system_id} + new_system.save() + # Update System name to INTEGRATION_NAME system name + nsre = new_system.root_element + nsre.name = ep.data['name'] + nsre.save() + # Update System Project title to INTEGRATION_NAME system name + prt = project.root_task + prt.title_override = ep.data['name'] + prt.save() + # Redirect to the new project. + # return HttpResponseRedirect(project.get_absolute_url()) + msg = f"Created new System in GovReady based on {INTEGRATION_NAME} system id {integration_system_id}." + else: + systems = System.objects.filter(Q(info__contains={INTEGRATION_SYSTEM_ID: integration_system_id})) + if len(systems) == 1: + system = systems[0] + # Assume one project per system + project = system.projects.all()[0] + msg = f"System aleady exists in GovReady based on {INTEGRATION_NAME} system id {integration_system_id}." + else: + project = None + msg = f"Multiple systems aleady exists in GovReady based on {INTEGRATION_NAME} system id {integration_system_id}." + msg = msg + f"They are: " + ",".join(str(systems)) + return HttpResponse( + f"

{msg}

" + f"

now: {datetime.now()}

" + f"

Returned data:

" + f"
{json.dumps(ep.data,indent=4)}
" + f"") + return HttpResponse( + f"

{msg}

" + f"

visit system {project.title} at {project.get_absolute_url()}

" + f"

now: {datetime.now()}

" + f"

Returned data:

" + f"
{json.dumps(ep.data,indent=4)}
" + f"") diff --git a/integrations/csam/README.md b/integrations/csam/README.md index db18751b6..8d8ecffde 100644 --- a/integrations/csam/README.md +++ b/integrations/csam/README.md @@ -1,6 +1,6 @@ ## Configure -Create an Integration record in Django admin: +### Step 1: Create an Integration record in Django admin: Name: csam Description: Integration to support CSAM version 4.10 diff --git a/integrations/csam/views.py b/integrations/csam/views.py index 8e81e28e5..4c51e1c07 100644 --- a/integrations/csam/views.py +++ b/integrations/csam/views.py @@ -157,7 +157,6 @@ def get_system_info(request, system_id=2): def get_paired_remote_system_info_using_local_system_id(request, system_id=2): """Retrieve the system information from CSAM""" - system = get_object_or_404(System, pk=system_id) # try: # # system = System.objects.get(pk=system_id) diff --git a/integrations/management/commands/example_stub/README.md b/integrations/management/commands/example_stub/README.md new file mode 100644 index 000000000..1eb662902 --- /dev/null +++ b/integrations/management/commands/example_stub/README.md @@ -0,0 +1,52 @@ +# ABOUT Integration Example + +## Configure + +### Step 1: Create an Integration record in Django admin: + +- Name: example +- Description: Integration to support Example service +- Config: +```json +{ + "base_url": "http://example-test.agency.gov/example/api", + "personal_access_token": "" +} +``` +- Config schema: +```json +{} +``` + +For local dev and testing, create an Integration record in Django admin for CSAM mock service: + +- Name: example +- Description: Integration to support Example service +- Config: +```json +{ + "base_url": "http://localhost:9009", + "personal_access_token": "FAD619" +} +``` +- Config schema: +```json +{} +``` + +### Step 2: Add route to `integration.urls.py` + +```python +url(r"^example/", include("integrations.example.urls")), +``` + +## Testing with Mock Service + +The Example integration includes a mock Example service you can launch in the terminal to test your integration. + +To launch the mock service do the following in a separate terminal from the root directory of GovReady-Q: + +```python +pip install click +python integrations/example/mock.py +``` \ No newline at end of file diff --git a/integrations/management/commands/example_stub/__init__.py b/integrations/management/commands/example_stub/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integrations/management/commands/example_stub/communicate.py b/integrations/management/commands/example_stub/communicate.py new file mode 100644 index 000000000..1baa7c7c4 --- /dev/null +++ b/integrations/management/commands/example_stub/communicate.py @@ -0,0 +1,70 @@ +import requests +import json +from base64 import b64encode +from urllib.parse import urlparse +from integrations.utils.integration import Communication +from integrations.models import Integration, Endpoint + + +class ExampleCommunication(Communication): + + DESCRIPTION = { + "name": "IntegrationStub", + "description": "IntegrationStub Service", + "version": "0.1", + "integration_db_record": False, + "mock": { + "base_url": "http:/localhost:9009", + "personal_access_token": None + } + } + + def __init__(self, **kwargs): + assert self.DESCRIPTION, "Developer must assign a description dict" + self.__is_authenticated = False + self.error_msg = {} + self.auth_dict = {} + self.data = None + self.base_url = "https://example.com/api" + + def identify(self): + """Identify which Communication subclass""" + identity_str = f"This is {self.DESCRIPTION['name']} version {self.DESCRIPTION['version']}" + print(identity_str) + return identity_str + + def setup(self, **kwargs): + pass + + def get_response(self, endpoint, headers=None, verify=False): + response = requests.get(f"{self.base_url}{endpoint}") + self.status_code = response.status_code + if self.status_code == 200: + self.data = response.json() + elif self.status_code == 404: + print("404 - page not found") + else: + pass + return self.data + + def authenticate(self, user=None, passwd=None): + """Authenticate with service""" + pass + + @property + def is_authenticated(self): + return self.__is_authenticated + + @is_authenticated.setter + def is_authenticated(self, value): + self.__is_authenticated = value + + def extract_data(self, authentication, identifiers): + """Extract data""" + pass + + def transform_data(self, data, system_id=None, title=None, description=None, deployment_uuid=None): + pass + + def load_data(self, data): + pass diff --git a/integrations/management/commands/example_stub/mock.py b/integrations/management/commands/example_stub/mock.py new file mode 100644 index 000000000..e5c48a2ff --- /dev/null +++ b/integrations/management/commands/example_stub/mock.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +############################################################## +# +# A simple Python webserver to generate mock Integration API results +# +# Requirements: +# +# pip install click +# +# Usage: +# +# # Start mock service +# python3 integrations/example/mock.py +# +# Accessing: +# +# curl localhost:9009/endpoint +# curl -X 'GET' 'http://127.0.0.1:9009/v1/system/111' +# curl localhost:9002/v1/system/111 # requires authentication +# +############################################################## + +# Parse command-line arguments +import click + +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import urlparse, parse_qs +import json +from abc import ABC +#from urllib.parse import urlparse +from django.utils import timezone +from time import time + + +PORT = 9002 +MOCK_SRVC = "CSAM" + +class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + + # Mock data + SYSTEMS = { "111": {"id": 111, + "someKey": "Some Value" + }, + "222": {"id": 222, + "someKey": "Some Value" + } + } + + def mk_system_info_response(self, system_id): + return self.SYSTEMS[system_id] + + def do_GET(self, method=None): + """Parse and route GET request""" + # Parse path + request = urlparse(self.path) + params = parse_qs(request.query) + print(f"request.path: {request.path}, params: {params}") + # params are received as arrays, so get first element in array + # system_id = params.get('system_id', [0])[0] + + # Route GET request + if request.path == "/v1/test/hello": + """Reply with 'hello'""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message":"hello"} + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif request.path == "/v1/test/authenticate-test": + """Test authentication""" + # Test authentication by reading headers and looking for 'Authentication' + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + # Read headers + if 'Authorization' in self.headers: + print("Authorization header:", self.headers['Authorization']) + data = {"reply": "Success", + "Authorization": self.headers['Authorization'], + "pat": self.headers['Authorization'].split("Bearer ")[-1] + } + else: + data = {"reply": "Fail", + "Authorization": None, + "pat": None + } + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + elif request.path == "/v1/systems/111" or request.path == "/v1/systems/222": + """Reply with system information""" + + system_id = re.search(r"/v1/systems/([0-9]{1,8})", request.path).group(1).strip() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + if '111' in request.path: + data = self.mk_system_info_response('111') + elif '222' in request.path: + data = self.mk_system_info_response('222') + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + else: + """Reply with Path not found""" + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message":"Path not found"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + def do_POST(self): + """Parse and route POST request""" + request = urlparse(self.path) + params = parse_qs(request.query) + print("request.path:", request.path) + print("** params", params) + + # Route POST request + if request.path == "/v1/test/hello": + """Reply with 'hello'""" + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message": "hello, POST"} + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + + if request.path == "/v1/systems/111" or request.path == "/v1/systems/222": + """Update system information""" + # # unauthorized example: + # curl -X 'POST' 'http://localhost:9002/system/111' \ + # -H 'accept: application/json;odata.metadata=minimal;odata.streaming=true' + + content_length = int(self.headers['Content-Length']) + self.post_data = self.rfile.read(content_length) + self.post_data_json = json.loads(self.post_data) + if '111' in request.path: + system_id = '111' + elif '222' in request.path: + system_id = '222' + self.SYSTEMS[system_id]['name'] = self.post_data_json.get('name', self.SYSTEM['name']) + self.SYSTEM[system_id]['purpose'] = self.post_data_json.get('purpose', "MISSING CONTENT") + # Send response + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = self.mk_system_info_response(system_id) + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + else: + # Path not found + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + data = {"message":"Path not found"} + # Send the JSON response + self.wfile.write(json.dumps(data, indent=4).encode('UTF-8')) + +def main(): + httpd = HTTPServer(('localhost', PORT), SimpleHTTPRequestHandler) + httpd.serve_forever() + +if __name__ == "__main__": + main() diff --git a/integrations/management/commands/example_stub/urls.py b/integrations/management/commands/example_stub/urls.py new file mode 100644 index 000000000..25769de44 --- /dev/null +++ b/integrations/management/commands/example_stub/urls.py @@ -0,0 +1,17 @@ +from django.conf.urls import url +from . import views + +urlpatterns = [ + url(r"^identify$", views.integration_identify, name='integration_identify'), + url(r"^endpoint(?P.*)$", views.integration_endpoint, name='integration_endpoint'), # Ex: /integrations/example/endpoint/system/111 + url(r"^post_endpoint(?P.*)$", views.integration_endpoint_post, name='integration_endpoint'), + + url(r"^get_system_info_test/(?P.*)$", views.get_system_info, name='get_system_info_test'), + url(r"^get_multiple_system_info/(?P.*)$", views.get_multiple_system_info, name='get_multiple_system_info'), + url(r"^system/(?P.*)$", views.system_info, name='csam_system_info'), + url(r"^update_system_description_test/(?P.*)$", views.update_system_description_test, name='update_system_description_test'), + url(r"^update_system_description$", views.update_system_description, name='update_system_description'), + + url(r"^create_system_from_remote/(?P.*)$", views.create_system_from_remote, name='create_system_from_remote'), + +] diff --git a/integrations/management/commands/example_stub/views.py b/integrations/management/commands/example_stub/views.py new file mode 100644 index 000000000..8f6a98397 --- /dev/null +++ b/integrations/management/commands/example_stub/views.py @@ -0,0 +1,327 @@ +import json +import time +from datetime import datetime +from django.shortcuts import get_object_or_404, render +from django.http import HttpResponse, HttpResponseNotFound, HttpResponseRedirect +from django.db.models import Q +from integrations.models import Integration, Endpoint +from controls.models import System +from siteapp.models import Organization, Portfolio +from siteapp.views import get_compliance_apps_catalog_for_user, start_app +from guidedmodules.app_loading import ModuleDefinitionError +from .communicate import ExampleCommunication + +# Change 'example' to name of integration, e.g., 'csam' and 'csam_system_id' +INTEGRATION_NAME = 'example' +INTEGRATION_SYSTEM_ID = 'example_system_id' + +try: + INTEGRATION = get_object_or_404(Integration, name=INTEGRATION_NAME) +except: + HttpResponseNotFound(f'

404 - Integration configuration missing. Create Integration database record.

') + +def set_integration(): + return ExampleCommunication() + +def integration_identify(request): + """Integration returns an identification""" + + communication = set_integration() + return HttpResponse(f"Attempting to communicate with {INTEGRATION_NAME} integration: {communication.identify()}") + +def integration_endpoint(request, endpoint=None): + """Communicate with an integrated service""" + + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def integration_endpoint_post(request, endpoint=None): + """Communicate with an integrated service using POST""" + + post_data = { + "name": "My IT System2", + "description": "This is a more complex test system" + } + communication = set_integration() + data = communication.post_response(endpoint, data=json.dumps(post_data)) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate using POST with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

endpoint: {endpoint}.

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def get_system_info(request, system_id=2): + """Retrieve the system information from CSAM""" + + # system = System.objects.get(pk=system_id) + system = get_object_or_404(System, pk=system_id) + # TODO: Check user permission to view + integration_system_id = system.info.get(INTEGRATION_SYSTEM_ID, None) + if integration_system_id is None: + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

System '{system_id}' does not have an associated '{integration_system_id}'.

" + f"") + + endpoint = f'/v1/systems/{integration_system_id}' + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

endpoint: {endpoint}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def system_info(request, system_id=2): + """Retrieve the system information from CSAM""" + system = get_object_or_404(System, pk=system_id) + # try: + # # system = System.objects.get(pk=system_id) + # system = get_object_or_404(System, pk=system_id) + # except: + # return HttpResponse( + # f"" + # f"

now: {datetime.now()}

" + # f"

System '{system_id}' does not exist.

" + # f"") + + # TODO: Check user permission to view + communication = set_integration() + integration_system_id = system.info.get(INTEGRATION_SYSTEM_ID, None) + if integration_system_id is None: + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"integration: {communication.identify()}

" + f"

now: {datetime.now()}

" + f"

System '{system_id}' does not have an associated '{integration_system_id}'.

" + f"") + + endpoint = f'/v1/systems/{integration_system_id}' + # is there local information? + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + # TODO: Refresh data if empty + if created: + # Cache not available + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep.data = data + ep.save() + else: + # Cache available + cached = True + pass + + context = { + "system": system, + "cached": True, + "communication": communication, + "ep": ep + } + from siteapp import settings + # settings.TEMPLATES[0]['DIRS'].append('/Users/gregelinadmin/Documents/workspace/govready-q-private/integrations/{INTEGRATION_NAME}/templates/') + # print(2,"========= TEMPLATES", settings.TEMPLATES[0]['DIRS']) + return render(request, "{INTEGRATION_NAME}/system.html", context) + +def get_multiple_system_info(request, system_id_list="1,2"): + """Get and cach system info for multiple systems""" + systems_updated = [] + systems = System.objects.filter(pk__in=system_id_list.split(",")) + for s in systems: + integration_system_id = s.info.get(INTEGRATION_SYSTEM_ID, None) + if integration_system_id is None: + print(f"System id {s.id} has no example_system_id") + else: + endpoint = f'/v1/systems/{integration_system_id}' + communication = set_integration() + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + ep.data = data + ep.save() + msg = f"System id {s.id} info updated from {INTEGRATION_NAME} system {integration_system_id}" + print(msg) + systems_updated.append(msg) + time.sleep(0.25) + + return HttpResponse( + f"

Attempting to communicate with '{INTEGRATION_NAME}' " + f"get_multiple_system_info for system ids {system_id_list}

" + f"

now: {datetime.now()}

" + f"

Result:

" + f"
{systems_updated}
" + f"") + +def update_system_description_test(request, system_id=2): + """Test updating system description in CSAM""" + + params={"src_obj_type": "system", "src_obj_id": system_id} + data = update_system_description(params) + return HttpResponse( + f"

Attempting to update CSAM description of System id {system_id}...' " + f"

now: {datetime.now()}

" + f"

Returned data:

" + f"
{json.dumps(data,indent=4)}
" + f"") + +def update_system_description(request, params={"src_obj_type": "system", "src_obj_id": 2}): + """Update System description in CSAM""" + + system_id = params['src_obj_id'] + system = System.objects.get(pk=system_id) + # TODO: Check user permission to update + integration_system_id = system.info.get(INTEGRATION_SYSTEM_ID, None) + # print("10, ========== integration_system_id", integration_system_id) + if example_system_id is not None: + new_description = "This is the new system description." + endpoint = f"/v1/systems/{integration_system_id}" + post_data = { + "description": new_description + } + communication = set_integration() + data = communication.post_response(endpoint, data=json.dumps(post_data)) + result = data + return result + +def create_system_from_remote(request, remote_system_id): + """Create a system in GovReady-Q based on info from integrated service""" + + print("Create a system in GovReady-Q based on info from integrated service") + + communication = set_integration() + integration_system_id = int(remote_system_id) + endpoint = f'/v1/systems/{integration_system_id}' + # is there local information? + ep, created = Endpoint.objects.get_or_create( + integration=INTEGRATION, + endpoint_path=endpoint + ) + # TODO: Refresh data if empty + if created: + # Cache not available + data = communication.get_response(endpoint) + # Cache remote data locally in database + ep.data = data + ep.save() + else: + # Cache available + cached = True + pass + + # Check if system aleady exists with example_system_id + if not System.objects.filter(Q(info__contains={INTEGRATION_SYSTEM_ID: integration_system_id})).exists(): + # Create new system + # What is default template? + source_slug = "govready-q-files-startpack" + app_name = "speedyssp" + # can user start the app? + # Is this a module the user has access to? The app store + # does some authz based on the organization. + catalog = get_compliance_apps_catalog_for_user(request.user) + for app_catalog_info in catalog: + if app_catalog_info["key"] == source_slug + "/" + app_name: + # We found it. + break + else: + raise Http404() + # Start the most recent version of the app. + appver = app_catalog_info["versions"][0] + organization = Organization.objects.first() # temporary + folder = None + task = None + q = None + # Get portfolio project should be included in. + if request.GET.get("portfolio"): + portfolio = Portfolio.objects.get(id=request.GET.get("portfolio")) + else: + if not request.user.default_portfolio: + request.user.create_default_portfolio_if_missing() + portfolio = request.user.default_portfolio + try: + project = start_app(appver, organization, request.user, folder, task, q, portfolio) + except ModuleDefinitionError as e: + error = str(e) + # Associate System with INTEGRATION_NAME system + new_system = project.system + new_system.info = {INTEGRATION_SYSTEM_ID: integration_system_id} + new_system.save() + # Update System name to INTEGRATION_NAME system name + nsre = new_system.root_element + nsre.name = ep.data['name'] + nsre.save() + # Update System Project title to INTEGRATION_NAME system name + prt = project.root_task + prt.title_override = ep.data['name'] + prt.save() + # Redirect to the new project. + # return HttpResponseRedirect(project.get_absolute_url()) + msg = f"Created new System in GovReady based on {INTEGRATION_NAME} system id {integration_system_id}." + else: + systems = System.objects.filter(Q(info__contains={INTEGRATION_SYSTEM_ID: integration_system_id})) + if len(systems) == 1: + system = systems[0] + # Assume one project per system + project = system.projects.all()[0] + msg = f"System aleady exists in GovReady based on {INTEGRATION_NAME} system id {integration_system_id}." + else: + project = None + msg = f"Multiple systems aleady exists in GovReady based on {INTEGRATION_NAME} system id {integration_system_id}." + msg = msg + f"They are: " + ",".join(str(systems)) + return HttpResponse( + f"

{msg}

" + f"

now: {datetime.now()}

" + f"

Returned data:

" + f"
{json.dumps(ep.data,indent=4)}
" + f"") + return HttpResponse( + f"

{msg}

" + f"

visit system {project.title} at {project.get_absolute_url()}

" + f"

now: {datetime.now()}

" + f"

Returned data:

" + f"
{json.dumps(ep.data,indent=4)}
" + f"") diff --git a/integrations/management/commands/makeintegration.py b/integrations/management/commands/makeintegration.py new file mode 100644 index 000000000..c29661a52 --- /dev/null +++ b/integrations/management/commands/makeintegration.py @@ -0,0 +1,90 @@ +import os +import shutil + +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction, models +from django.db.utils import OperationalError +from django.conf import settings +from pathlib import PurePath + +class Command(BaseCommand): + """Create an empty integration from a model integrations directory""" + + help = 'Create an empty integration from a model integrations directory' + + def add_arguments(self, parser): + parser.add_argument('integration_name', nargs='?', type=str) + + def handle(self, *args, **options): + + # Configure + integration_name = options['integration_name'] + self.stdout.write(f"integration_name is '{integration_name}'") + integrations_dir = os.path.join('integrations', integration_name) + + # Fail if integration_name already exists + if os.path.exists(integrations_dir): + raise CommandError(f"Integration named '{integration_name}' already exists.") + return + + # Copy stub files + + # Make integrations directory + shutil.copytree(os.path.join('integrations', 'management', 'commands', 'example_stub'), integrations_dir, copy_function=shutil.copy) + self.stdout.write(f"Integration folder created") + + self.stdout.write(f"Integration '{integration_name}' created.") + # Rename Communication subclass + try: + # Update communicate.py + with open(os.path.join(integrations_dir, 'communicate.py'), "r+") as f: + orig_file = f.read() + new_file = orig_file.replace('ExampleCommunication', f'{integration_name.title()}Communication') + new_file = new_file.replace("https://example.com/api", f"https://{integration_name}.com/api") + f.seek(0) + f.write(new_file) + f.truncate() + print(f"File {os.path.join(integrations_dir, 'communicate.py')} updated") + except: + print(f"Error updating {os.path.join(integrations_dir, 'communicate.py')}") + try: + # Update views.py + with open(os.path.join(integrations_dir, 'views.py'), "r+") as f: + orig_file = f.read() + new_file = orig_file.replace('ExampleCommunication', f'{integration_name.title()}Communication') + new_file = new_file.replace("INTEGRATION_NAME = 'example'", f"INTEGRATION_NAME = '{integration_name}'") + new_file = new_file.replace("INTEGRATION_SYSTEM_ID = 'example_system_id'", f"INTEGRATION_SYSTEM_ID = '{integration_name}_system_id'") + new_file = new_file.replace("return ExampleCommunication()", f"return {integration_name}Communication()") + new_file = new_file.replace("IntegrationStub", f"{integration_name}") + f.seek(0) + f.write(new_file) + f.truncate() + print(f"File {os.path.join(integrations_dir, 'views.py')} updated") + except: + print(f"Error updating {os.path.join(integrations_dir, 'views.py')}") + try: + # Update README.md + with open(os.path.join(integrations_dir, 'README.md'), "r+") as f: + orig_file = f.read() + new_file = orig_file.replace('example', f'{integration_name}') + f.seek(0) + f.write(new_file) + f.truncate() + print(f"File {os.path.join(integrations_dir, 'README.md')} updated") + except: + print(f"Error updating {os.path.join(integrations_dir, 'README.md')}") + try: + # Update mock.py + with open(os.path.join(integrations_dir, 'mock.py'), "r+") as f: + orig_file = f.read() + new_file = orig_file.replace('example', f'{integration_name}') + f.seek(0) + f.write(new_file) + f.truncate() + print(f"File {os.path.join(integrations_dir, 'mock.py')} updated") + except: + print(f"Error updating {os.path.join(integrations_dir, 'mock.py')}") + + return + diff --git a/integrations/utils/integration.py b/integrations/utils/integration.py index 5dcf138e4..8d33ffb54 100644 --- a/integrations/utils/integration.py +++ b/integrations/utils/integration.py @@ -4,6 +4,8 @@ import sys from abc import ABC from urllib.parse import urlparse +import importlib +from integrations.models import Integration class HelperMixin: @@ -106,3 +108,24 @@ def transform_data(self, data, system_id=None, title=None, description=None, dep def load_data(self, data): raise NotImplementedError() + +def get_control_data_enhancements(request, catalog_key, cl_id): + """Return additional data about control""" + + integration_module = importlib.import_module(f'integrations.controlmatrix') + communication_class = integration_module.views.set_integration() + extracted_data = communication_class.extract_data(authentication=None, identifiers=[cl_id]) + control_matrix = extracted_data[0] + return control_matrix + # Return data first matching integration + # TODO: Better to loop through all integrations and expand data dictionary returned + # for i in Integration.objects.all(): + # im = importlib.import_module(f'integrations.{i.name}') + # cc = im.views.set_integration() + # control_data = (cc.extract_data(None,["ac-3"])) + # if control_data is not None: + # print(i.name, control_data) + # break + # return control_data[0] + + diff --git a/nlp/management/commands/CandidateEntity_pandas.py b/nlp/management/commands/CandidateEntity_pandas.py index 28cc919f5..23f83b843 100644 --- a/nlp/management/commands/CandidateEntity_pandas.py +++ b/nlp/management/commands/CandidateEntity_pandas.py @@ -3,10 +3,6 @@ from django.core.management.base import BaseCommand -from controls.enums.statements import StatementTypeEnum -from controls.models import Statement, ImportRecord -from controls.utilities import oscalize_control_id -from siteapp.models import User, Project, Organization # import xlsxio import os diff --git a/templates/control-description-modal.html b/templates/control-description-modal.html new file mode 100644 index 000000000..8176494da --- /dev/null +++ b/templates/control-description-modal.html @@ -0,0 +1,61 @@ + + + +{% block scripts %} + +{% endblock %} diff --git a/templates/controls/detail.html b/templates/controls/detail.html index e5377cebf..c7082efbe 100644 --- a/templates/controls/detail.html +++ b/templates/controls/detail.html @@ -16,6 +16,28 @@ max-width: 950px; margin: auto; } + + a { color: #666; } + + .control-text-modal { + white-space: pre-wrap; + font-size: 11pt; + font-family: sans-serif; + /* text-align: justify; */ + /* text-justify: inter-word; */ + /* line-height: 18px; */ + } + + .modal-content { + position: fixed; + margin: auto; + width: 130%; + /* height: calc(100% - 53px); */ + -webkit-transform: translate3d(0%, 0, 0); + -ms-transform: translate3d(0%, 0, 0); + -o-transform: translate3d(0%, 0, 0); + transform: translate3d(0%, 0, 0); + } {% endblock %} @@ -35,7 +57,7 @@
{% if control.title is not None %}

- {{ control.id_display }} {{ control.title }} + {{ control.id_display }} ({{ control.title }})

{% else %} Control was not found in the catalog. @@ -52,38 +74,67 @@

-

 

- -
-
Official Control Text
-
{% if control.title is not None %} -
-
-

Description

{% if control.description %}{{ control.description }}{% else %}No description provided.{% endif %}
-

Guidance

{% if control.guidance %}{{ control.guidance }}{% else %}No guidance provided.{% endif %} -
-
-

Links

- {% if links %} - +
+ {% if control.title is not None %} + Description +

+ {% if control.description|wordcount < 50 %}{{ control.description }}{% else %}{{ control.description|truncatewords:65 }}{% endif %} +   >>>View full control +

+
+ Part of +

{{ control.family_title }} ({{ catalog.catalog_key_display }})

+ Conntinuous Monitoring + {% if control_matrix is not None %} +

+ The required control monitoring for {{ control.id_display|upper }} is {{ control_matrix.REVIEW_FREQUENCY|lower }} {% if control_matrix.AUTOMATED == 'YES' %} and automated{% endif %}. {{ control_matrix_doc.discussion }} +

+ Responsibility +

+ When hosted in the cloud, responsibility for {{ control.id_display|upper }} is likely {% if control_matrix.CLOUD_SHARED == 'YES' %} shared between you and{% else %}inherited from{% endif %} your cloud service provider. +

+ {% else %} +

Monitoring information for {{ control.id_display|upper }} is not available.

+ {% endif %} + Guidance + {% if control.guidance %} +

+ {% if control.guidance|wordcount < 50 %}{{ control.guidance }}{% else %}{{ control.guidance|truncatewords:55 }}{% endif %} +   >>>View full control +

+
{% else %} - No links provided. +

No guidance provided.

{% endif %} -
+ {% else %} +
+

The control {{ control.id }} was not found in the control catalog.

+
+ {% endif %} +
+ -
{% else %}

Control is not found in the control catalog.

{% endif %} + + {% if control.title is not None %}{% include "control-description-modal.html" %}{% endif %} +
diff --git a/templates/controls/editor.html b/templates/controls/editor.html index fbc08f8c8..b14fdcc6f 100644 --- a/templates/controls/editor.html +++ b/templates/controls/editor.html @@ -24,7 +24,7 @@ h3 { color: #888; - #font-weight: bold; + /* font-weight: bold; */ margin-top: 20px; } @@ -33,14 +33,13 @@ margin: auto; } - .control-text { + .control-text-modal { white-space: pre-wrap; font-size: 11pt; - /*max-width: 700px;*/ - font-family: trebuchet ms, sans-serif; - text-align: justify; - /*text-justify: inter-word;*/ - line-height: 24px; + font-family: sans-serif; + /* text-align: justify; */ + /* text-justify: inter-word; */ + /* line-height: 18px; */ } #control-lookup input { @@ -59,9 +58,7 @@ resize: vertical; } - a { - color: #666; - } + a { color: #666; } #common-tab-count, #component-tab-count { display: inline-block; @@ -98,6 +95,22 @@ border-radius: 0px; padding: 0px 0px 8px 12px; } + .modal-content { + position: fixed; + margin: auto; + width: 130%; + /* height: calc(100% - 53px); */ + -webkit-transform: translate3d(0%, 0, 0); + -ms-transform: translate3d(0%, 0, 0); + -o-transform: translate3d(0%, 0, 0); + transform: translate3d(0%, 0, 0); + } + + @media screen and (max-width: 700px) { + #control_description_modal { + width: 400px; + } + } {% endblock %} @@ -107,14 +120,7 @@

- System Control: {{ control.id_display|upper }} {{ control.title }} -

-

- {% if control.title is not None %} - {{ catalog.catalog_key_display }} - {% else %} -

The control {{ control.id }} was not found in the control catalog.

- {% endif %} + {{ control.id_display|upper }} ({{ control.title }})

@@ -134,22 +140,30 @@

-