From a5e0aba5cfedaa8edbb72a9b9492344b74b7bb16 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 24 Apr 2024 11:39:25 +0200 Subject: [PATCH 01/86] Horrible initial commit for library management --- backend/ciso_assistant/settings.py | 4 +- backend/core/apps.py | 68 +++++++-- backend/core/management/commands/status.py | 2 +- ...ibrary_alter_framework_library_and_more.py | 98 ++++++++++++ backend/core/models.py | 131 +++++++++++++--- backend/core/serializers.py | 9 +- backend/core/tests/test_models.py | 28 ++-- backend/core/urls.py | 5 +- backend/core/utils.py | 7 +- backend/library/helpers.py | 6 +- .../libraries/critical_risk_matrix_3x3.yaml | 2 +- backend/library/serializers.py | 43 ++++-- backend/library/utils.py | 89 ++++++----- backend/library/views.py | 140 +++++++++++++----- .../components/ModelTable/ModelTable.svelte | 8 +- frontend/src/lib/utils/crud.ts | 26 +++- frontend/src/lib/utils/table.ts | 6 +- .../routes/(app)/libraries/+page.server.ts | 71 ++++++++- .../src/routes/(app)/libraries/+page.svelte | 29 ++-- .../(app)/libraries/[id=urn]/+page.server.ts | 2 +- .../(app)/libraries/[id=urn]/+server.ts | 4 +- .../(app)/libraries/[id=urn]/tree/+server.ts | 7 +- frontend/tests/utils/page-content.ts | 4 +- frontend/tests/utils/test-data.ts | 8 +- frontend/tests/utils/test-utils.ts | 2 +- 25 files changed, 617 insertions(+), 182 deletions(-) create mode 100644 backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index caeb7095c..d66f008dd 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -269,8 +269,8 @@ def set_ciso_assistant_url(_, __, event_dict): # https://docs.djangoproject.com/en/4.2/ref/settings/#databases # SQLIte file can be changed, useful for tests -SQLITE_FILE = os.environ.get("SQLITE_FILE", BASE_DIR / "db/ciso-assistant.sqlite3") - +SQLITE_FILE = os.environ.get('SQLITE_FILE', BASE_DIR / "db/ciso-assistant.sqlite3") +LIBRARIES_PATH = library_path = BASE_DIR / "library/libraries" if "POSTGRES_NAME" in os.environ: DATABASES = { diff --git a/backend/core/apps.py b/backend/core/apps.py index 4846d60cc..fcff5bd2c 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from django.db.models.signals import post_migrate -from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL -import os +from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL, LIBRARIES_PATH +import sys, os AUDITOR_PERMISSIONS_LIST = [ "view_project", @@ -21,7 +21,7 @@ "view_requirementnode", "view_evidence", "view_framework", - "view_library", + "view_loadedlibrary", "view_user", ] @@ -44,7 +44,8 @@ "view_requirementnode", "view_evidence", "view_framework", - "view_library", + "view_storedlibrary", + "view_loadedlibrary", "view_user", ] @@ -97,7 +98,8 @@ "view_riskmatrix", "view_requirementnode", "view_framework", - "view_library", + "view_storedlibrary", + "view_loadedlibrary", "view_user", ] @@ -155,7 +157,8 @@ "delete_evidence", "view_requirementnode", "view_framework", - "view_library", + "view_storedlibrary", + "view_loadedlibrary", "view_user", ] @@ -232,15 +235,17 @@ "delete_framework", "view_requirementnode", "view_requirementlevel", # Permits to see the object on api by an admin - "view_library", - "add_library", - "delete_library", + "view_storedlibrary", + "add_storedlibrary", + "delete_storedlibrary", + "view_loadedlibrary", + "add_loadedlibrary", + "delete_loadedlibrary", "backup", "restore", ] - -def startup(**kwargs): +def startup(sender: AppConfig,**kwargs): """ Implement CISO Assistant 1.0 default Roles and User Groups during migrate This makes sure root folder and global groups are defined before any other object is created @@ -248,6 +253,7 @@ def startup(**kwargs): """ from django.contrib.auth.models import Permission from iam.models import Folder, Role, RoleAssignment, User, UserGroup + from core.models import StoredLibrary print("startup handler: initialize database") @@ -349,13 +355,53 @@ def startup(**kwargs): except Exception as e: print(e) # NOTE: Add this exception in the logger +from django.db import connection class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "core" verbose_name = "Core" + def _ready(self): + from core.models import StoredLibrary + import time + + db_tables = set(connection.introspection.table_names()) + required_models = [StoredLibrary] + required_tables = set(model._meta.db_table for model in required_models) + if not required_tables.issubset(db_tables) : + return None + + """for lib in StoredLibrary.objects.all() : + # Delete this loop ! + lib.delete()""" + + # ./library/libraries/soc2-2017.yaml + + # Remove the 3 following lines after + from core.models import LoadedLibrary + for lib in LoadedLibrary.objects.all() : + lib.delete() + + start = time.perf_counter() + StoredLibrary.__init_class__() + print(f"Importing new libraries...") + for fname in os.listdir(LIBRARIES_PATH) : + fname = str(LIBRARIES_PATH / fname) + if fname.endswith(".yaml") : + error = StoredLibrary.store_library_file(fname) + if error is not None : + print(f"[ERROR] Can't import libary file '{fname}' : {error}",file=sys.stderr) + end = time.perf_counter() + print(f"Execution time = {end-start}") + stored_libaries = [*StoredLibrary.objects.all()] + print(f"There are {len(stored_libaries)} stored libraries :D !") + print(f"Stored libaries : {stored_libaries}") + print("Django initialization finished !") + def ready(self): + self._ready() # avoid post_migrate handler if we are in the main, as it interferes with restore if not os.environ.get("RUN_MAIN"): post_migrate.connect(startup, sender=self) + diff --git a/backend/core/management/commands/status.py b/backend/core/management/commands/status.py index f97ae92bf..f039779d7 100644 --- a/backend/core/management/commands/status.py +++ b/backend/core/management/commands/status.py @@ -9,7 +9,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): nb_users = User.objects.all().count() nb_first_login = User.objects.filter(first_login=True).count() - nb_libraries = Library.objects.all().count() + nb_libraries = LoadedLibrary.objects.all().count() nb_domains = Folder.objects.filter(content_type="DO").count() nb_projects = Project.objects.all().count() nb_assets = Asset.objects.all().count() diff --git a/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py b/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py new file mode 100644 index 000000000..47ff8a104 --- /dev/null +++ b/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py @@ -0,0 +1,98 @@ +# Generated by Django 5.0.4 on 2024-04-18 17:38 + +import django.db.models.deletion +import iam.models +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0008_alter_complianceassessment_status_and_more'), + ('iam', '0002_purge_validator'), + ] + + operations = [ + migrations.CreateModel( + name='LoadedLibrary', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='UpdatedÒ at')), + ('is_published', models.BooleanField(default=False, verbose_name='published')), + ('urn', models.CharField(blank=True, max_length=100, null=True, unique=True, verbose_name='URN')), + ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), + ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), + ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), + ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), + ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), + ('description', models.TextField(blank=True, null=True, verbose_name='Description')), + ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), + ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), + ('version', models.IntegerField(verbose_name='Version')), + ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), + ('builtin', models.BooleanField(default=False)), + ('objects_meta', models.JSONField()), + ('dependencies', models.ManyToManyField(blank=True, to='core.loadedlibrary', verbose_name='Dependencies')), + ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AlterField( + model_name='framework', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='frameworks', to='core.loadedlibrary'), + ), + migrations.AlterField( + model_name='referencecontrol', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='reference_controls', to='core.loadedlibrary'), + ), + migrations.AlterField( + model_name='riskmatrix', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='risk_matrices', to='core.loadedlibrary'), + ), + migrations.AlterField( + model_name='threat', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='threats', to='core.loadedlibrary'), + ), + migrations.CreateModel( + name='StoredLibrary', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='UpdatedÒ at')), + ('is_published', models.BooleanField(default=False, verbose_name='published')), + ('urn', models.CharField(blank=True, max_length=100, null=True, unique=True, verbose_name='URN')), + ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), + ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), + ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), + ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), + ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), + ('description', models.TextField(blank=True, null=True, verbose_name='Description')), + ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), + ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), + ('version', models.IntegerField(verbose_name='Version')), + ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), + ('builtin', models.BooleanField(default=False)), + ('objects_meta', models.JSONField()), + ('dependencies', models.CharField(max_length=16384, null=True)), + ('is_obsolete', models.BooleanField(default=False)), + ('is_imported', models.BooleanField(default=False)), + ('hash_checksum', models.CharField(max_length=64)), + ('content', models.TextField()), + ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ], + options={ + 'abstract': False, + }, + ), + migrations.DeleteModel( + name='Library', + ), + ] diff --git a/backend/core/models.py b/backend/core/models.py index 04762a1ad..0a94be5c2 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -8,16 +8,18 @@ from .base_models import * from .validators import validate_file_size, validate_file_name -from .utils import camel_case +from .utils import camel_case, sha256 from iam.models import FolderMixin, PublishInRootFolderMixin from django.core import serializers import os +import re import json +import yaml from django.urls import reverse from datetime import date, datetime -from typing import Self +from typing import Union, List, Self from django.utils.html import format_html from structlog import get_logger @@ -83,19 +85,14 @@ def display_long(self) -> str: def __str__(self) -> str: return self.display_short +class LibraryMixin(ReferentialObjectMixin): + class Meta: + abstract = True -class Library(ReferentialObjectMixin): copyright = models.CharField( max_length=4096, null=True, blank=True, verbose_name=_("Copyright") ) version = models.IntegerField(null=False, verbose_name=_("Version")) - provider = models.CharField( - max_length=100, - blank=True, - null=True, - help_text=_("Provider of the library"), - verbose_name=_("Provider"), - ) packager = models.CharField( max_length=100, blank=True, @@ -103,6 +100,106 @@ class Library(ReferentialObjectMixin): help_text=_("Packager of the library"), verbose_name=_("Packager"), ) + builtin = models.BooleanField(default=False) + objects_meta = models.JSONField() + dependencies = models.CharField(blank=False,null=True,max_length=16384) + + def get_dependencies(self) -> List[str] : # We should add some kind of descriptor to LibaryMixin so that model_instance.dependencies directly returns a list instead of a string, using a function to get the desired value of a field is bad + return [] if self.dependencies is None else self.dependencies.split(" ") + +class StoredLibrary(LibraryMixin): + is_obsolete = models.BooleanField(default=False) + is_imported = models.BooleanField(default=False) + hash_checksum = models.CharField(max_length=64) + content = models.TextField() + + REQUIRED_FIELDS = {"urn","name","version","objects"} + FIELDS_VERIFIERS = {} + HASH_CHECKSUM_SET = set() # For now a library isn't updated if its SHA256 checksum has already been registered in the database. + + @classmethod + def __init_class__(cls) : + cls.HASH_CHECKSUM_SET = set( + value["hash_checksum"] + for value in cls.objects.values("hash_checksum") + ) + + @staticmethod + def store_library_file(fname: str) -> Union[str,None] : + with open(fname,"rb") as f : + library_content = f.read() + + hash_checksum = sha256(library_content) + if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET : + return None # We do not store the libary if its hash checksum is in the database. + + # urn(str:max_length=100), version(int), name(str:max_length=200) + + try : + library_data = yaml.safe_load(library_content) + except Exception : + return "Invalid formatted file, the library file must be formatted in YAML." + + missing_fields = StoredLibrary.REQUIRED_FIELDS - set(library_data.keys()) + + if missing_fields : + return "The following fields are missing : {}".format(", ".join(repr(field) for field in missing_fields)) + + urn = library_data["urn"] + locale = library_data.get("locale","en") + version = library_data["version"] + + library_matches = [*StoredLibrary.objects.filter( + urn=urn, + locale=locale + )] + if any( + libary.version >= version + for libary in library_matches + ) : + # The library isn't stored if it's obsolete due to be a too old version of itself. + return "A library with the urn '{}', a locale '{}' with a superior superior or equal to {} is already stored in the database.".format(urn,locale,version) + + for library in library_matches : + libary.is_obsolete = True + library.save() # If a user delete a library from the libary store we must set the is_obsolete value of its most recent obsolete version to False. + + objects_meta = { + key: len(value) + for key, value in library_data["objects"].items() + } + + dependencies = library_data.get("dependencies") + if dependencies is not None : + dependencies = " ".join(dependencies) # Which means we have to make the URN format more restrictive so that an URN is considered invalid if it contains any kind of whitespace. + + library_objects = json.dumps(library_data["objects"]) + StoredLibrary.objects.create( + name=library_data["name"], + is_published=True, + urn=urn, + locale=locale, + version=version, + ref_id=library_data["ref_id"], + default_locale=False, # We don't care about this value yet. + description=library_data.get("description"), + annotation=library_data.get("annotation"), + copyright=library_data.get("copyright"), + provider=library_data.get("provider"), + packager=library_data.get("packager"), + objects_meta=objects_meta, + dependencies=dependencies, + builtin=library_data.get("builtin",False), # We have to add a "builtin: true" line to every builtin library file. + hash_checksum=hash_checksum, + content=library_objects + ) + + def loads(self) -> Union[str,None] : + from library.utils import LibraryImporter + library_importer = LibraryImporter(self) + return library_importer.import_library() + +class LoadedLibrary(LibraryMixin): dependencies = models.ManyToManyField( "self", blank=True, verbose_name=_("Dependencies"), symmetrical=False ) @@ -152,7 +249,7 @@ def reference_count(self) -> int: ) .distinct() .count() - + Library.objects.filter(dependencies=self).distinct().count() + + LoadedLibrary.objects.filter(dependencies=self).distinct().count() ) def delete(self, *args, **kwargs): @@ -160,17 +257,17 @@ def delete(self, *args, **kwargs): raise ValueError( "This library is still referenced by some risk or compliance assessments" ) - dependent_libraries = Library.objects.filter(dependencies=self) + dependent_libraries = LoadedLibrary.objects.filter(dependencies=self) if dependent_libraries: raise ValueError( f"This library is a dependency of {dependent_libraries.count()} other libraries" ) - super(Library, self).delete(*args, **kwargs) + super(LoadedLibrary, self).delete(*args, **kwargs) class Threat(ReferentialObjectMixin, PublishInRootFolderMixin): library = models.ForeignKey( - Library, on_delete=models.CASCADE, null=True, blank=True, related_name="threats" + LoadedLibrary, on_delete=models.CASCADE, null=True, blank=True, related_name="threats" ) fields_to_check = ["ref_id", "name"] @@ -204,7 +301,7 @@ class ReferenceControl(ReferentialObjectMixin): ] library = models.ForeignKey( - Library, + LoadedLibrary, on_delete=models.CASCADE, null=True, blank=True, @@ -247,7 +344,7 @@ def __str__(self): class RiskMatrix(ReferentialObjectMixin): library = models.ForeignKey( - Library, + LoadedLibrary, on_delete=models.CASCADE, null=True, blank=True, @@ -329,7 +426,7 @@ def __str__(self) -> str: class Framework(ReferentialObjectMixin): library = models.ForeignKey( - Library, + LoadedLibrary, on_delete=models.CASCADE, null=True, blank=True, diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 4a2b27f59..b4a071cfd 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -163,18 +163,17 @@ class ReferenceControlReadSerializer(ReferenceControlWriteSerializer): folder = FieldsRelatedField() library = FieldsRelatedField(["name", "urn"]) - -class LibraryReadSerializer(BaseModelSerializer): +"""class LibraryReadSerializer(BaseModelSerializer): class Meta: - model = Library + model = LoadedLibrary fields = "__all__" class LibraryWriteSerializer(BaseModelSerializer): class Meta: - model = Library + model = LoadedLibrary fields = "__all__" - +""" class ThreatWriteSerializer(BaseModelSerializer): class Meta: diff --git a/backend/core/tests/test_models.py b/backend/core/tests/test_models.py index 068b00014..25f8c6699 100644 --- a/backend/core/tests/test_models.py +++ b/backend/core/tests/test_models.py @@ -18,7 +18,7 @@ Asset, Threat, RiskMatrix, - Library, + LoadedLibrary, Framework, ) from django.contrib.auth import get_user_model @@ -863,7 +863,7 @@ class TestLibrary: pytestmark = pytest.mark.django_db def test_library_creation(self): - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -877,7 +877,7 @@ def test_library_creation(self): assert library.folder == Folder.get_root_folder() def test_library_reference_count_zero_if_unused(self): - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -890,7 +890,7 @@ def test_library_reference_count_zero_if_unused(self): def test_library_reference_count_incremented_when_framework_is_referenced_by_compliance_assessment_and_decremented_when_compliance_assessment_is_deleted( self, ): - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -923,7 +923,7 @@ def test_library_reference_count_incremented_when_framework_is_referenced_by_com def test_library_reference_count_incremented_when_reference_control_is_referenced_by_complance_assessment_and_decremented_when_compliance_assessment_is_deleted( self, ): - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -983,7 +983,7 @@ def test_library_reference_count_incremented_when_risk_matrix_is_referenced_by_r domain = Folder.objects.create(name="Domain", description="Domain description") project = Project.objects.create(name="Project", folder=domain) - library = Library.objects.get() + library = LoadedLibrary.objects.get() risk_matrix = RiskMatrix.objects.get() assert library.reference_count == 0 @@ -1010,7 +1010,7 @@ def test_library_reference_count_incremented_when_threat_is_referenced_by_risk_s risk_matrix = RiskMatrix.objects.get() - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -1055,7 +1055,7 @@ def test_library_reference_count_incremented_when_reference_control_is_reference risk_matrix = RiskMatrix.objects.get() - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -1101,7 +1101,7 @@ def test_library_reference_count_incremented_when_reference_control_is_reference def test_library_reference_count_must_be_zero_for_library_deletion( self, ): - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -1132,18 +1132,18 @@ def test_library_reference_count_must_be_zero_for_library_deletion( library.delete() - assert Library.objects.count() == 0 + assert LoadedLibrary.objects.count() == 0 @pytest.mark.usefixtures("domain_project_fixture") def test_library_cannot_be_deleted_if_it_is_a_dependency_of_other_libraries(self): - dependency_library = Library.objects.create( + dependency_library = LoadedLibrary.objects.create( name="Dependency Library", description="Dependency Library description", folder=Folder.get_root_folder(), locale="en", version=1, ) - library = Library.objects.create( + library = LoadedLibrary.objects.create( name="Library", description="Library description", folder=Folder.get_root_folder(), @@ -1156,7 +1156,7 @@ def test_library_cannot_be_deleted_if_it_is_a_dependency_of_other_libraries(self dependency_library.delete() library.delete() - assert Library.objects.count() == 1 + assert LoadedLibrary.objects.count() == 1 dependency_library.delete() - assert Library.objects.count() == 0 + assert LoadedLibrary.objects.count() == 0 diff --git a/backend/core/urls.py b/backend/core/urls.py index 5be718e5c..abcf57f96 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -1,6 +1,6 @@ from .views import * from django.contrib.auth import views as auth_views -from library.views import LibraryViewSet +from library.views import StoredLibraryViewSet, LoadedLibraryViewSet from django.urls import include, path @@ -43,7 +43,8 @@ RequirementAssessmentViewSet, basename="requirement-assessments", ) -router.register(r"libraries", LibraryViewSet, basename="libraries") +router.register(r"stored-libraries", StoredLibraryViewSet, basename="stored-libraries") +router.register(r"loaded-libraries", LoadedLibraryViewSet, basename="loaded-libraries") urlpatterns = [ diff --git a/backend/core/utils.py b/backend/core/utils.py index 7c6775169..71d308587 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -1,13 +1,18 @@ from django.utils.translation import gettext_lazy as _ from re import sub from enum import Enum - +import hashlib def camel_case(s): s = sub(r"(_|-)+", " ", s).title().replace(" ", "") return "".join([s[0].lower(), s[1:]]) +def sha256(string: bytes) -> str: + """Return the SHA256-hashed hexadecimal representation of the bytes object given as argument.""" + h = hashlib.new("SHA256") + h.update(string) + return h.hexdigest() class RoleCodename(Enum): ADMINISTRATOR = "BI-RL-ADM" diff --git a/backend/library/helpers.py b/backend/library/helpers.py index 79659e8fd..2024b82b2 100644 --- a/backend/library/helpers.py +++ b/backend/library/helpers.py @@ -2,16 +2,16 @@ from iam.models import Folder -def preview_library(library) -> dict[str, list]: +def preview_library(library_objects) -> dict[str, list]: """ Function to create temporary requirement nodes list Used to display requirements in tree view inside library detail view """ preview = {} requirement_nodes_list = [] - if library["objects"]["framework"].get("requirement_nodes"): + if library_objects["framework"].get("requirement_nodes"): index = 0 - for requirement_node in library["objects"]["framework"]["requirement_nodes"]: + for requirement_node in library_objects["framework"]["requirement_nodes"]: index += 1 requirement_nodes_list.append( RequirementNode( diff --git a/backend/library/libraries/critical_risk_matrix_3x3.yaml b/backend/library/libraries/critical_risk_matrix_3x3.yaml index ba72f3327..87eee8a28 100644 --- a/backend/library/libraries/critical_risk_matrix_3x3.yaml +++ b/backend/library/libraries/critical_risk_matrix_3x3.yaml @@ -1,7 +1,7 @@ urn: urn:intuitem:risk:library:critical_risk_matrix_3x3 locale: en ref_id: critical_3x3 -name: Critical risk matrix 3x3 +nam: Critical risk matrix 3x3 description: Critical risk matrix 3x3 version: 1 provider: intuitem diff --git a/backend/library/serializers.py b/backend/library/serializers.py index b75407092..acc205ede 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -1,11 +1,10 @@ -from core.models import Library +from core.models import StoredLibrary, LoadedLibrary from core.serializers import ( BaseModelSerializer, ) from rest_framework import serializers - -class LibraryObjectSerializer(serializers.Serializer): +"""class LibraryObjectSerializer(serializers.Serializer): type = serializers.ChoiceField( choices=[ "risk_matrix", @@ -15,22 +14,44 @@ class LibraryObjectSerializer(serializers.Serializer): ] ) fields = serializers.DictField(child=serializers.CharField()) +""" - -class LibrarySerializer(serializers.Serializer): +class StoredLibrarySerializer(serializers.ModelSerializer): + # Not used yet + class Meta: + model = StoredLibrary + fields = ["name","description","locale","version","builtin","objects_meta"] + + # name = serializers.CharField() + # description = serializers.CharField() + # locale = serializers.ChoiceField(choices=["en", "fr"]) + # version = serializers.CharField() + # copyright = serializers.CharField() + # builtin = serializers.BooleanField() + # objects_meta = serializers.JSONField() + +""" +class StoredLibraryReadSerializer(StoredLibraryWriteSerializer): + content = serializers.SerializerMethodField() + + def get_content(self, content: bytes): + return content.encode("utf-8") # Should we enforce UTF-8 for library files ? +""" + +class LoadedLibrarySerializer(serializers.Serializer): name = serializers.CharField() description = serializers.CharField() locale = serializers.ChoiceField(choices=["en", "fr"]) - objects = LibraryObjectSerializer(many=True) - format_version = serializers.CharField() + # objects = LibraryObjectSerializer(many=True) + version = serializers.CharField() copyright = serializers.CharField() + builtin = serializers.BooleanField() - -class LibraryModelSerializer(BaseModelSerializer): +"""class LibraryModelSerializer(BaseModelSerializer): class Meta: - model = Library + model = LoadedLibrary fields = "__all__" - +""" class LibraryUploadSerializer(serializers.Serializer): file = serializers.FileField(required=True) diff --git a/backend/library/utils.py b/backend/library/utils.py index b06b74878..ac654f547 100644 --- a/backend/library/utils.py +++ b/backend/library/utils.py @@ -11,7 +11,8 @@ from ciso_assistant import settings from core.models import ( Framework, - Library, + StoredLibrary, + LoadedLibrary, RequirementNode, RiskMatrix, ReferenceControl, @@ -27,7 +28,6 @@ URN_REGEX = r"^urn:([a-zA-Z0-9_-]+):([a-zA-Z0-9_-]+):([a-zA-Z0-9_-]+)(?::([a-zA-Z0-9_-]+))?:(.+)$" - def match_urn(urn_string): match = re.match(URN_REGEX, urn_string) if match: @@ -75,7 +75,7 @@ def get_available_libraries(): libs = list(yaml.safe_load_all(file)) AVAILABLE_LIBRARIES[(fname,os.path.getmtime(fname))] = libs for _lib in libs : - if (lib := Library.objects.filter(urn=_lib["urn"]).first()) is not None: + if (lib := LoadedLibrary.objects.filter(urn=_lib["urn"]).first()) is not None: _lib["id"] = lib.id _lib["reference_count"] = lib.reference_count libraries.append(_lib) @@ -111,7 +111,7 @@ def get_library(urn: str) -> dict | None: # First, try to fetch the library from the database.dd try: - lib = Library.objects.get(urn=urn) + lib = LoadedLibrary.objects.get(urn=urn) return { "id": lib.id, "urn": lib.urn, @@ -123,7 +123,7 @@ def get_library(urn: str) -> dict | None: "reference_count": lib.reference_count, "objects": lib._objects, } - except Library.DoesNotExist: + except LoadedLibrary.DoesNotExist: # Construct the path to the YAML file from the urn. filename = urn.split(":")[-1] libraries_path = settings.BASE_DIR / "library/libraries" @@ -282,7 +282,7 @@ def init(self) -> Union[str, None]: ) is not None: return requirement_node_import_error - def import_framework(self, library_object: Library): + def import_framework(self, library_object: LoadedLibrary): framework_object = Framework.objects.create( folder=Folder.get_root_folder(), library=library_object, @@ -311,7 +311,7 @@ def is_valid(self) -> Union[str, None]: if missing_fields := self.REQUIRED_FIELDS - set(self.threat_data.keys()): return "Missing the following fields : {}".format(", ".join(missing_fields)) - def import_threat(self, library_object: Library): + def import_threat(self, library_object: LoadedLibrary): Threat.objects.create( library=library_object, urn=self.threat_data.get("urn"), @@ -345,7 +345,7 @@ def is_valid(self) -> Union[str, None]: category, ", ".join(ReferenceControlImporter.CATEGORIES) ) - def import_reference_control(self, library_object: Library): + def import_reference_control(self, library_object: LoadedLibrary): ReferenceControl.objects.create( library=library_object, urn=self.reference_control_data.get("urn"), @@ -381,14 +381,19 @@ def is_valid(self) -> Union[str, None]: if missing_fields := self.REQUIRED_FIELDS - set(self.risk_matrix_data.keys()): return "Missing the following fields : {}".format(", ".join(missing_fields)) - def import_risk_matrix(self, library_object: Library): + def import_risk_matrix(self, library_object: LoadedLibrary): matrix_data = { key: value for key, value in self.risk_matrix_data.items() if key in self.MATRIX_FIELDS } - RiskMatrix.objects.create( + print("\n\n\n" + "-"*60) + print("YES I HAVE BEEN CALLED") + print(json.dumps(matrix_data,indent=4)) + print("-"*60) + + created = RiskMatrix.objects.create( library=library_object, folder=Folder.get_root_folder(), name=self.risk_matrix_data.get("name"), @@ -402,14 +407,17 @@ def import_risk_matrix(self, library_object: Library): default_locale=library_object.default_locale, # Change this in the future ? is_published=True, ) + print(f"Object created ===> {created}") class LibraryImporter: + # The word "import" must be replaced by "load" in all classes/methods/variables declared in this file. + REQUIRED_FIELDS = {"ref_id", "urn", "locale", "objects", "version"} OBJECT_FIELDS = ["threats", "reference_controls", "risk_matrix", "framework"] - def __init__(self, data: dict): - self._library_data = data + def __init__(self, library: dict): + self._library = library self._framework_importer = None self._threats = [] self._reference_controls = [] @@ -493,13 +501,13 @@ def init_framework(self, framework_data: dict) -> Union[str, None]: return self._framework_importer.init() def init(self) -> Union[str, None]: - missing_fields = self.REQUIRED_FIELDS - set(self._library_data.keys()) + """missing_fields = self.REQUIRED_FIELDS - set(self._library_data.keys()) if missing_fields: return "The following fields are missing in the library: {}".format( ", ".join(missing_fields) - ) + )""" - library_objects = self._library_data["objects"] + library_objects = json.loads(self._library.content) if not any( object_field in library_objects for object_field in self.OBJECT_FIELDS @@ -540,31 +548,40 @@ def init(self) -> Union[str, None]: def check_and_import_dependencies(self): """Check and import library dependencies.""" - dependencies = self._library_data.get("dependencies", []) - for dependency in dependencies: - if not Library.objects.filter(urn=dependency).exists(): - import_library_view(get_library(dependency)) + dependencies = self._library.get_dependencies() + + for dependency_urn in dependencies: + if not LoadedLibrary.objects.filter(urn=dependency_urn).exists(): + # import_library_view(get_library(dependency)) + dependency = StoredLibrary.objects.get(urn=dependency_urn,is_obsolete=False) # We only fetch by URN without thinking about what locale, that may be a problem in the future. + error_msg = dependency.loads() + if error_msg is not None : + return error_msg def create_or_update_library(self): """Create or update the library object.""" - _urn = self._library_data["urn"] - _locale = self._library_data["locale"] - _default_locale = not Library.objects.filter(urn=_urn).exists() + _urn = self._library.urn + _locale = self._library.locale + _default_locale = not LoadedLibrary.objects.filter(urn=_urn).exists() - library_object, _created = Library.objects.update_or_create( + print(f"VALUE ===> {self._library}") + print(f"FIELD ===> {self._library.objects_meta}") + library_object, _created = LoadedLibrary.objects.update_or_create( defaults={ - "ref_id": self._library_data["ref_id"], - "name": self._library_data.get("name"), - "description": self._library_data.get("description", None), + "ref_id": self._library.ref_id, + "name": self._library.name, + "description": self._library.description, "urn": _urn, "locale": _locale, "default_locale": _default_locale, - "version": self._library_data.get("version", None), - "provider": self._library_data.get("provider", None), - "packager": self._library_data.get("packager", None), - "copyright": self._library_data.get("copyright", None), - "folder": Folder.get_root_folder(), # TODO: make this configurable + "version": self._library.version, + "provider": self._library.provider, + "packager": self._library.packager, + "copyright": self._library.copyright, + "folder": Folder.get_root_folder(), # TODO: make this configurable, "is_published": True, + "builtin": self._library.builtin, + "objects_meta": self._library.objects_meta }, urn=_urn, locale=_locale, @@ -590,17 +607,19 @@ def _import_library(self) : library_object = self.create_or_update_library() self.import_objects(library_object) library_object.dependencies.set( - Library.objects.filter( - urn__in=self._library_data.get("dependencies", []) + LoadedLibrary.objects.filter( + urn__in=self._library.get_dependencies() ) ) def import_library(self): """Main method to import a library.""" if (error_message := self.init()) is not None: - return error_message + return error_message # This error check should be done when storing the Library but no after. - self.check_and_import_dependencies() + error_msg = self.check_and_import_dependencies() + if error_msg is not None : + return error_msg for _ in range(10) : try: diff --git a/backend/library/views.py b/backend/library/views.py index f570e267a..d5c291872 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -2,24 +2,24 @@ from django.core.exceptions import ValidationError from django.db import IntegrityError from django.db.models import QuerySet -from rest_framework import permissions, status +from rest_framework import viewsets, permissions, status from rest_framework.generics import get_object_or_404 from rest_framework.status import ( HTTP_200_OK, HTTP_400_BAD_REQUEST, + HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY, ) from rest_framework.parsers import FileUploadParser from rest_framework.views import APIView - from django.contrib import messages from django.utils.translation import gettext_lazy as _ from django.http import HttpResponse import yaml from core.helpers import get_sorted_requirement_nodes -from core.models import Library +from core.models import StoredLibrary, LoadedLibrary from core.views import BaseModelViewSet from iam.models import RoleAssignment, Folder, Permission from library.validators import validate_file_extension @@ -28,35 +28,114 @@ from rest_framework.decorators import action from rest_framework.response import Response -from .serializers import LibrarySerializer, LibraryUploadSerializer -from .utils import get_available_libraries, get_library, import_library_view +from .serializers import StoredLibrarySerializer, LoadedLibrarySerializer, LibraryUploadSerializer +from .utils import LibraryImporter, get_available_libraries, get_library, import_library_view -class LibraryViewSet(BaseModelViewSet): - serializer_class = LibrarySerializer - parser_classes = [FileUploadParser] +class StoredLibraryViewSet(viewsets.ModelViewSet): + # serializer_class = StoredLibrarySerializer + # parser_classes = [FileUploadParser] # solve issue with URN containing dot, see https://stackoverflow.com/questions/27963899/django-rest-framework-using-dot-in-url lookup_value_regex = r"[\w.:-]+" - model = Library + model = StoredLibrary def list(self, request, *args, **kwargs): - if not "view_library" in request.user.permissions: + if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - return Response({"results": get_available_libraries()}) + available_libraries = StoredLibrary.objects.filter(is_obsolete=False).values("name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") # The frontend doesn't receive the obsolete libraries for now. + + return Response({"results": available_libraries}) def retrieve(self, request, *args, pk, **kwargs): - if not "view_library" in request.user.permissions: + if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) + return StoredLibrary.objects.get(pk=urn) # Handle the exception if the pk urn doesn't exist + # raise NotImplementedError() # This method must be implemented. + + @action(detail=True, methods=["get"], url_path="import") + def import_library(self, request, pk=None): + if not RoleAssignment.is_access_allowed( + user=request.user, + perm=Permission.objects.get(codename="add_loadedlibrary"), + folder=Folder.get_root_folder(), + ): + return Response(status=status.HTTP_403_FORBIDDEN) + library = StoredLibrary.objects.get(urn=pk) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. + + # return Response({"error":f"ERROR !!! {str(lib)}"},status=status.HTTP_403_FORBIDDEN) + # library = get_library(pk) + try: + error_msg = library.loads() + if error_msg is not None : + return Response({"status":"error","error":error_msg},status=status.HTTP_400_BAD_REQUEST) # This can cause translation issues + return Response({"status": "success"}) + except Exception as e: + """print(f"ERROR {type(e)}") + print(str(e)) + raise e""" return Response( - status=status.HTTP_403_FORBIDDEN, + { + "error": "Failed to load library, please check if it has dependencies" + }, # This must translated + status=HTTP_422_UNPROCESSABLE_ENTITY, ) - library = get_library(pk) - return Response(library) + + @action(detail=True, methods=["get"]) + def tree(self, request, pk): + lib = StoredLibrary.objects.get(urn=pk) # It should return a 404 error if the query doesn't return anything and therefore raise an exception + library_objects = json.loads(lib.content) # We need caching for this + """library = { + "id": lib.id, + "urn": lib.urn, + "name": lib.name, + "description": lib.description, + "provider": lib.provider, + "packager": lib.packager, + "copyright": lib.copyright, + "reference_count": lib.reference_count, + "objects": library_objects, + }""" + """if not library: + return Response( + data="This library does not exist.", status=HTTP_404_NOT_FOUND + )""" + if not library_objects.get("framework"): + return Response( + data="This library does not include a framework.", + status=HTTP_400_BAD_REQUEST, + ) + preview = preview_library(library_objects) + return Response( + get_sorted_requirement_nodes(preview.get("requirement_nodes"), None) + ) + +class LoadedLibraryViewSet(viewsets.ModelViewSet): + serializer_class = LoadedLibrarySerializer + parser_classes = [FileUploadParser] + + # solve issue with URN containing dot, see https://stackoverflow.com/questions/27963899/django-rest-framework-using-dot-in-url + lookup_value_regex = r"[\w.:-]+" + model = LoadedLibrary + + def list(self, request, *args, **kwargs): + if not "view_storedlibrary" in request.user.permissions: + return Response(status=status.HTTP_403_FORBIDDEN) + available_libraries = LoadedLibrary.objects.all().values("name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") + + return Response({"results": available_libraries}) + # return Response({"results": get_available_libraries()}) + + def retrieve(self, request, *args, pk, **kwargs): + if not "view_loadedlibrary" in request.user.permissions: + return Response(status=status.HTTP_403_FORBIDDEN) + raise NotImplementedError() # This method must me implemented + # library = get_library(pk) + # return Response(library) def destroy(self, request, *args, pk, **kwargs): if not RoleAssignment.is_access_allowed( user=request.user, - perm=Permission.objects.get(codename="delete_library"), + perm=Permission.objects.get(codename="delete_loadedlibrary"), folder=Folder.get_root_folder(), ): return Response(status=status.HTTP_403_FORBIDDEN) @@ -74,7 +153,7 @@ def destroy(self, request, *args, pk, **kwargs): ) try: - Library.objects.get(id=library.get("id")).delete() + LoadedLibrary.objects.get(id=library.get("id")).delete() except IntegrityError as e: # TODO: Log the exception if logging is set up # logging.exception("Integrity error while deleting library: %s", e) @@ -92,41 +171,22 @@ def destroy(self, request, *args, pk, **kwargs): @action(detail=True, methods=["get"]) def tree(self, request, pk): - library = get_library(pk) + lib = get_library(pk) + # library_objects = json.loads(lib.content) # How to this for loaded libarries if not library: return Response( - data="This library does not exist.", status=HTTP_400_BAD_REQUEST + data="This library does not exist.", status=HTTP_404_NOT_FOUND ) if not library["objects"].get("framework"): return Response( data="This library does not include a framework.", status=HTTP_400_BAD_REQUEST, ) - preview = preview_library(library) + preview = preview_library(library_objects) return Response( get_sorted_requirement_nodes(preview.get("requirement_nodes"), None) ) - @action(detail=True, methods=["get"], url_path="import") - def import_library(self, request, pk=None): - if not RoleAssignment.is_access_allowed( - user=request.user, - perm=Permission.objects.get(codename="add_library"), - folder=Folder.get_root_folder(), - ): - return Response(status=status.HTTP_403_FORBIDDEN) - library = get_library(pk) - try: - import_library_view(library) - return Response({"status": "success"}) - except Exception as e: - return Response( - { - "error": "Failed to load library, please check if it has dependencies" - }, - status=HTTP_422_UNPROCESSABLE_ENTITY, - ) - @action(detail=False, methods=["post"], url_path="upload") def upload_library(self, request): if not request.data: diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index b3f988d43..859ec271b 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -72,7 +72,7 @@ const rowMetaData = $rows[rowIndex].meta; /** @event {rowMetaData} selected - Fires when a table row is clicked. */ if (!rowMetaData[identifierField] || !URLModel) return; - goto(`/${URLModel}/${rowMetaData[identifierField]}`); + goto(`/${LocalURLModel}/${rowMetaData[identifierField]}`); } // Row Keydown Handler @@ -89,6 +89,8 @@ export let URLModel: urlModel | undefined = undefined; $: model = URLModel ? URL_MODEL_MAP[URLModel] : undefined; + export let LocalURLModel: urlModel | undefined = undefined; + LocalURLModel = LocalURLModel ?? URLModel; const user = $page.data.user; @@ -265,10 +267,10 @@ {#if row.meta[identifierField]} {@const actionsComponent = field_component_map['actions']} { + const endpoint = `${BASE_API_URL}/stored-libraries/`; + const endpoint2 = `${BASE_API_URL}/loaded-libraries/`; + + const [res,res2] = await Promise.all([fetch(endpoint),fetch(endpoint2)]); + + const storedLibraries = await res.json().then((res) => res.results); + const loadedLibraries = await res2.json().then((res) => res.results); + + const prepareRow = (row) => { + row.overview = [ + `Provider: ${row.provider}`, + `Packager: ${row.packager}`, + ...Object.entries(row.objects_meta).map(([key,value]) => `${key}: ${value}`) + ]; + row.allowDeleteLibrary = row.allowDeleteLibrary = row.reference_count && row.reference_count > 0 ? false : true; + } + + storedLibraries.forEach(prepareRow); + loadedLibraries.forEach(prepareRow); + + // const headData: Record + + const makeHeadData = (modelName: string) => { + return listViewFields['stored-libraries'].body.reduce( + (obj, key, index) => { + obj[key] = listViewFields['stored-libraries'].head[index]; + return obj; + }, + {} + ); + } + + const makeBodyData = (libraries,modelName: string) => tableSourceMapper(libraries, listViewFields[modelName].body); + + const makeLibrariesTable = (libraries: Library[],modelName: string) => { + return { + head: makeHeadData(modelName), + body: makeBodyData(libraries,modelName), + meta: libraries + }; + }; + + const storedLibrariesTable = makeLibrariesTable(storedLibraries,'stored-libraries'); + const loadedLibrariesTable = makeLibrariesTable(loadedLibraries,'loaded-libraries'); + + const schema = z.object({ id: z.string() }); + const deleteForm = await superValidate(schema); + + loadedLibrariesTable.body.push(storedLibrariesTable.body[0]); + loadedLibrariesTable.meta.push(storedLibrariesTable.meta[0]); + + return { storedLibrariesTable, loadedLibrariesTable, deleteForm } +}) satisfies PageServerLoad; + + + + + +const old_load = (async ({ fetch }) => { const endpoint = `${BASE_API_URL}/libraries/`; const res = await fetch(endpoint); @@ -77,6 +140,10 @@ export const load = (async ({ fetch }) => { return { libraries, defaultLibrariesTable, importedLibrariesTable, deleteForm }; }) satisfies PageServerLoad; + +// ----------------------------------------------------------- // + + export const actions: Actions = { upload: async (event) => { const formData = await event.request.formData(); @@ -85,7 +152,7 @@ export const actions: Actions = { if (formData.has('file')) { const { file } = Object.fromEntries(formData) as { file: File }; // Should i check if attachment.size > 0 ? - const endpoint = `${BASE_API_URL}/libraries/upload/`; + const endpoint = `${BASE_API_URL}/loaded-libraries/upload/`; const req = await event.fetch(endpoint, { method: 'POST', headers: { @@ -117,7 +184,7 @@ export const actions: Actions = { const deleteForm = await superValidate(formData, schema); const id = deleteForm.data.id; - const endpoint = `${BASE_API_URL}/libraries/${id}/`; + const endpoint = `${BASE_API_URL}/loaded-libraries/${id}/`; if (!deleteForm.valid) { console.error(deleteForm.errors); diff --git a/frontend/src/routes/(app)/libraries/+page.svelte b/frontend/src/routes/(app)/libraries/+page.svelte index 22ddd2ee0..7aa314f15 100644 --- a/frontend/src/routes/(app)/libraries/+page.svelte +++ b/frontend/src/routes/(app)/libraries/+page.svelte @@ -10,37 +10,40 @@ export let data; import { TabGroup, Tab } from '@skeletonlabs/skeleton'; - let tabSet: number = data.importedLibrariesTable.body.length > 0 ? 0 : 1; - $: if (data.importedLibrariesTable.body.length === 0) tabSet = 1; + let tabSet: number = data.loadedLibrariesTable.body.length > 0 ? 0 : 1; + $: if (data.loadedLibrariesTable.body.length === 0) tabSet = 0;
- - {#if data.importedLibrariesTable.body.length > 0} - {m.importedLibraries()} - {m.librariesStore()} + + {#if data.loadedLibrariesTable.body.length > 0} + {m.librariesStore()} + {m.importedLibraries()} {:else}
{m.currentlyNoImportedLibraries()}.
{/if} - + {#if tabSet === 0} {/if} - {#if tabSet === 1} + {#if tabSet === 1} {/if} diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+page.server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+page.server.ts index 8252f9d1a..292b19a2d 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+page.server.ts @@ -5,7 +5,7 @@ import * as m from '$paraglide/messages'; export const actions: Actions = { default: async (event) => { - const endpoint = `${BASE_API_URL}/libraries/${event.params.id}/import`; + const endpoint = `${BASE_API_URL}/stored-libraries/${event.params.id}/import`; const res = await event.fetch(endpoint); if (!res.ok) { const response = await res.json(); diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index 2abb9b113..ebd52c765 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -4,9 +4,9 @@ import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, url }) => { - const endpoint = `${BASE_API_URL}${url.pathname}/${ + const endpoint = `${BASE_API_URL}/stored-libraries/${ url.searchParams ? '?' + url.searchParams.toString() : '' - }`; + }`; // ${url.pathname} const res = await fetch(endpoint); if (!res.ok) { diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index 2abb9b113..d3deb12fd 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -3,10 +3,11 @@ import { BASE_API_URL } from '$lib/utils/constants'; import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -export const GET: RequestHandler = async ({ fetch, url }) => { - const endpoint = `${BASE_API_URL}${url.pathname}/${ +export const GET: RequestHandler = async ({ fetch, params, url }) => { + /* const endpoint = `${BASE_API_URL}${url.pathname}/${ url.searchParams ? '?' + url.searchParams.toString() : '' - }`; + }`; */ + const endpoint = `${BASE_API_URL}/stored-libraries/${params.id}`; const res = await fetch(endpoint); if (!res.ok) { diff --git a/frontend/tests/utils/page-content.ts b/frontend/tests/utils/page-content.ts index ebb62d315..fa87ca4ff 100644 --- a/frontend/tests/utils/page-content.ts +++ b/frontend/tests/utils/page-content.ts @@ -38,8 +38,8 @@ export class PageContent extends BasePage { async createItem(values: { [k: string]: any }, dependency?: any) { if (dependency) { - await this.page.goto('/libraries'); - await this.page.waitForURL('/libraries'); + await this.page.goto('/loaded-libraries'); + await this.page.waitForURL('/loaded-libraries'); await this.importLibrary(dependency.ref || dependency.name, dependency.urn); await this.goto(); diff --git a/frontend/tests/utils/test-data.ts b/frontend/tests/utils/test-data.ts index c0b948b06..8f8c63060 100644 --- a/frontend/tests/utils/test-data.ts +++ b/frontend/tests/utils/test-data.ts @@ -72,7 +72,7 @@ export default { "view_riskmatrix", "view_requirementnode", "view_framework", - "view_library", + "view_loadedlibrary", "view_user", ] }, @@ -96,7 +96,7 @@ export default { "view_requirementnode", "view_evidence", "view_framework", - "view_library", + "view_loadedlibrary", "view_user", ] }, @@ -156,7 +156,7 @@ export default { "delete_evidence", "view_requirementnode", "view_framework", - "view_library", + "view_loadedlibrary", "view_user", ] }, @@ -181,7 +181,7 @@ export default { "view_requirementnode", "view_evidence", "view_framework", - "view_library", + "view_loadedlibrary", "view_user", ] }, diff --git a/frontend/tests/utils/test-utils.ts b/frontend/tests/utils/test-utils.ts index d33a189a5..30bf3bf3b 100644 --- a/frontend/tests/utils/test-utils.ts +++ b/frontend/tests/utils/test-utils.ts @@ -142,7 +142,7 @@ export const test = base.extend({ }, librariesPage: async ({ page }, use) => { - const lPage = new PageContent(page, '/libraries', 'Libraries'); + const lPage = new PageContent(page, '/loaded-ibraries', 'Loaded Libraries'); await use(lPage); }, From cb94245936b4b7ffb9a220e3d23246ff17b8321c Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 24 Apr 2024 16:07:32 +0200 Subject: [PATCH 02/86] Fix library detail view basic objects display --- backend/core/apps.py | 4 ++-- backend/library/serializers.py | 8 +++++++- backend/library/views.py | 14 +++++++++----- frontend/src/lib/utils/types.ts | 2 +- .../src/routes/(app)/libraries/+page.server.ts | 14 +++++++------- .../src/routes/(app)/libraries/[id=urn]/+server.ts | 7 +++---- .../(app)/libraries/[id=urn]/tree/+server.ts | 2 +- 7 files changed, 30 insertions(+), 21 deletions(-) diff --git a/backend/core/apps.py b/backend/core/apps.py index 44b601a0d..e0fda3869 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -377,9 +377,9 @@ def _ready(self): # ./library/libraries/soc2-2017.yaml # Remove the 3 following lines after - from core.models import LoadedLibrary + """from core.models import LoadedLibrary for lib in LoadedLibrary.objects.all() : - lib.delete() + lib.delete()""" start = time.perf_counter() StoredLibrary.__init_class__() diff --git a/backend/library/serializers.py b/backend/library/serializers.py index acc205ede..7bd6d9b51 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -20,7 +20,7 @@ class StoredLibrarySerializer(serializers.ModelSerializer): # Not used yet class Meta: model = StoredLibrary - fields = ["name","description","locale","version","builtin","objects_meta"] + fields = ["id","name","description","locale","version","builtin","objects_meta"] # name = serializers.CharField() # description = serializers.CharField() @@ -30,6 +30,11 @@ class Meta: # builtin = serializers.BooleanField() # objects_meta = serializers.JSONField() +class StoredLibraryDetailedSerializer(serializers.ModelSerializer): + class Meta: + model = StoredLibrary + fields = "__all__" + """ class StoredLibraryReadSerializer(StoredLibraryWriteSerializer): content = serializers.SerializerMethodField() @@ -39,6 +44,7 @@ def get_content(self, content: bytes): """ class LoadedLibrarySerializer(serializers.Serializer): + id = serializers.CharField() name = serializers.CharField() description = serializers.CharField() locale = serializers.ChoiceField(choices=["en", "fr"]) diff --git a/backend/library/views.py b/backend/library/views.py index 72fb19bbc..809ef6f0e 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -28,7 +28,7 @@ from rest_framework.decorators import action from rest_framework.response import Response -from .serializers import StoredLibrarySerializer, LoadedLibrarySerializer, LibraryUploadSerializer +from .serializers import StoredLibraryDetailedSerializer, LoadedLibrarySerializer, LibraryUploadSerializer from .utils import LibraryImporter, get_available_libraries, get_library, import_library_view class StoredLibraryViewSet(viewsets.ModelViewSet): @@ -40,17 +40,21 @@ class StoredLibraryViewSet(viewsets.ModelViewSet): model = StoredLibrary def list(self, request, *args, **kwargs): + print("WOOWOWOWOWOWOWOWOWOW\n"*20) if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - available_libraries = StoredLibrary.objects.filter(is_obsolete=False).values("name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") # The frontend doesn't receive the obsolete libraries for now. + available_libraries = StoredLibrary.objects.filter(is_obsolete=False).values("id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") # The frontend doesn't receive the obsolete libraries for now. return Response({"results": available_libraries}) def retrieve(self, request, *args, pk, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - return StoredLibrary.objects.get(pk=urn) # Handle the exception if the pk urn doesn't exist - # raise NotImplementedError() # This method must be implemented. + lib = StoredLibrary.objects.get(urn=pk,is_obsolete=False) # Handle the exception if the pk urn doesn't exist + print(f"-------- {pk}"*20) + print(f"FETCHED {lib}") + # return Response(StoredLibraryDetailedSerializer(lib).data) + return Response(StoredLibraryDetailedSerializer(lib).data) @action(detail=True, methods=["get"], url_path="import") def import_library(self, request, pk=None): @@ -120,7 +124,7 @@ class LoadedLibraryViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - available_libraries = LoadedLibrary.objects.all().values("name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") + available_libraries = LoadedLibrary.objects.all().values("id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") return Response({"results": available_libraries}) # return Response({"results": get_available_libraries()}) diff --git a/frontend/src/lib/utils/types.ts b/frontend/src/lib/utils/types.ts index 8670e7402..f4561de50 100644 --- a/frontend/src/lib/utils/types.ts +++ b/frontend/src/lib/utils/types.ts @@ -37,7 +37,7 @@ export const URL_MODEL = [ 'frameworks', 'requirements', 'requirement-assessments', - 'libraries' + // 'libraries' ] as const; export type urlModel = (typeof URL_MODEL)[number]; diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index 0e23c5fbb..621e8871c 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -6,26 +6,26 @@ import { setFlash } from 'sveltekit-flash-message/server'; import { setError, superValidate } from 'sveltekit-superforms'; import type { PageServerLoad } from './$types'; import { z } from 'zod'; +import { zod } from 'sveltekit-superforms/adapters'; import { tableSourceMapper } from '@skeletonlabs/skeleton'; import { listViewFields } from '$lib/utils/table'; import type { Library, urlModel } from '$lib/utils/types'; import * as m from '$paraglide/messages'; import { localItems } from '$lib/utils/locales'; import { languageTag } from '$paraglide/runtime'; -import { zod } from 'sveltekit-superforms/adapters'; // ----------------------------------------------------------- // export const load = (async ({ fetch }) => { - const endpoint = `${BASE_API_URL}/stored-libraries/`; - const endpoint2 = `${BASE_API_URL}/loaded-libraries/`; + const stored_libraries_endpoint = `${BASE_API_URL}/stored-libraries/`; + const loaded_libaries_endpoint = `${BASE_API_URL}/loaded-libraries/`; - const [res,res2] = await Promise.all([fetch(endpoint),fetch(endpoint2)]); + const [stored_libraries_res,loaded_libaries_res] = await Promise.all([fetch(stored_libraries_endpoint),fetch(loaded_libaries_endpoint)]); - const storedLibraries = await res.json().then((res) => res.results); - const loadedLibraries = await res2.json().then((res) => res.results); + const storedLibraries = await stored_libraries_res.json().then((res) => res.results); + const loadedLibraries = await loaded_libaries_res.json().then((res) => res.results); const prepareRow = (row) => { row.overview = [ @@ -65,7 +65,7 @@ export const load = (async ({ fetch }) => { const loadedLibrariesTable = makeLibrariesTable(loadedLibraries,'loaded-libraries'); const schema = z.object({ id: z.string() }); - const deleteForm = await superValidate(schema); + const deleteForm = await superValidate(zod(schema)); loadedLibrariesTable.body.push(storedLibrariesTable.body[0]); loadedLibrariesTable.meta.push(storedLibrariesTable.meta[0]); diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index ebd52c765..54d5ed441 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -3,16 +3,15 @@ import { BASE_API_URL } from '$lib/utils/constants'; import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -export const GET: RequestHandler = async ({ fetch, url }) => { - const endpoint = `${BASE_API_URL}/stored-libraries/${ - url.searchParams ? '?' + url.searchParams.toString() : '' - }`; // ${url.pathname} +export const GET: RequestHandler = async ({ fetch, url, params }) => { + const endpoint = `${BASE_API_URL}/stored-libraries/${params.id}/`; const res = await fetch(endpoint); if (!res.ok) { error(res.status as NumericRange<400, 599>, await res.json()); } const data = await res.json(); + data.objects = JSON.parse(data.content); return new Response(JSON.stringify(data), { headers: { diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index d3deb12fd..50faf622f 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -7,7 +7,7 @@ export const GET: RequestHandler = async ({ fetch, params, url }) => { /* const endpoint = `${BASE_API_URL}${url.pathname}/${ url.searchParams ? '?' + url.searchParams.toString() : '' }`; */ - const endpoint = `${BASE_API_URL}/stored-libraries/${params.id}`; + const endpoint = `${BASE_API_URL}/stored-libraries/${params.id}/`; const res = await fetch(endpoint); if (!res.ok) { From d846d97d8b021ed9b7588c7185ce7bcf22b88ad2 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 24 Apr 2024 16:26:21 +0200 Subject: [PATCH 03/86] The dependencies field is not a JSONField --- backend/core/apps.py | 2 +- .../0010_alter_storedlibrary_dependencies.py | 18 ++++++++++++++++++ backend/core/models.py | 6 ++---- .../libraries/critical_risk_matrix_3x3.yaml | 2 +- backend/library/serializers.py | 1 + 5 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 backend/core/migrations/0010_alter_storedlibrary_dependencies.py diff --git a/backend/core/apps.py b/backend/core/apps.py index e0fda3869..935f0c2e0 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -378,7 +378,7 @@ def _ready(self): # Remove the 3 following lines after """from core.models import LoadedLibrary - for lib in LoadedLibrary.objects.all() : + for lib in StoredLibrary.objects.all() : lib.delete()""" start = time.perf_counter() diff --git a/backend/core/migrations/0010_alter_storedlibrary_dependencies.py b/backend/core/migrations/0010_alter_storedlibrary_dependencies.py new file mode 100644 index 000000000..27bf93e4e --- /dev/null +++ b/backend/core/migrations/0010_alter_storedlibrary_dependencies.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.4 on 2024-04-24 14:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0009_loadedlibrary_alter_framework_library_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='storedlibrary', + name='dependencies', + field=models.JSONField(null=True), + ), + ] diff --git a/backend/core/models.py b/backend/core/models.py index 574412e6a..6bdcce3c3 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -102,7 +102,7 @@ class Meta: ) builtin = models.BooleanField(default=False) objects_meta = models.JSONField() - dependencies = models.CharField(blank=False,null=True,max_length=16384) + dependencies = models.JSONField(null=True) # models.CharField(blank=False,null=True,max_length=16384) def get_dependencies(self) -> List[str] : # We should add some kind of descriptor to LibaryMixin so that model_instance.dependencies directly returns a list instead of a string, using a function to get the desired value of a field is bad return [] if self.dependencies is None else self.dependencies.split(" ") @@ -169,9 +169,7 @@ def store_library_file(fname: str) -> Union[str,None] : for key, value in library_data["objects"].items() } - dependencies = library_data.get("dependencies") - if dependencies is not None : - dependencies = " ".join(dependencies) # Which means we have to make the URN format more restrictive so that an URN is considered invalid if it contains any kind of whitespace. + dependencies = library_data.get("dependencies") # I don't want whitespaces in URN anymore nontheless library_objects = json.dumps(library_data["objects"]) StoredLibrary.objects.create( diff --git a/backend/library/libraries/critical_risk_matrix_3x3.yaml b/backend/library/libraries/critical_risk_matrix_3x3.yaml index 87eee8a28..ba72f3327 100644 --- a/backend/library/libraries/critical_risk_matrix_3x3.yaml +++ b/backend/library/libraries/critical_risk_matrix_3x3.yaml @@ -1,7 +1,7 @@ urn: urn:intuitem:risk:library:critical_risk_matrix_3x3 locale: en ref_id: critical_3x3 -nam: Critical risk matrix 3x3 +name: Critical risk matrix 3x3 description: Critical risk matrix 3x3 version: 1 provider: intuitem diff --git a/backend/library/serializers.py b/backend/library/serializers.py index 7bd6d9b51..5cf92c62b 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -31,6 +31,7 @@ class Meta: # objects_meta = serializers.JSONField() class StoredLibraryDetailedSerializer(serializers.ModelSerializer): + dependencies = lambda dependencies: dependencies.split(" ") class Meta: model = StoredLibrary fields = "__all__" From 762b66037217044e58de092e3b767735463eb5b4 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 24 Apr 2024 16:27:12 +0200 Subject: [PATCH 04/86] Remove useless special serializing method from the StoredLibraryDetailedSerializer --- backend/library/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/library/serializers.py b/backend/library/serializers.py index 5cf92c62b..7bd6d9b51 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -31,7 +31,6 @@ class Meta: # objects_meta = serializers.JSONField() class StoredLibraryDetailedSerializer(serializers.ModelSerializer): - dependencies = lambda dependencies: dependencies.split(" ") class Meta: model = StoredLibrary fields = "__all__" From a00d456b315aa6f8a49b741cf647b9ef563157eb Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 24 Apr 2024 18:02:58 +0200 Subject: [PATCH 05/86] Fix typo in the AbstractBaseModel --- backend/core/base_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/base_models.py b/backend/core/base_models.py index 87759c589..27bc5b886 100644 --- a/backend/core/base_models.py +++ b/backend/core/base_models.py @@ -10,7 +10,7 @@ class AbstractBaseModel(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) created_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Created at")) - updated_at = models.DateTimeField(auto_now=True, verbose_name=_("UpdatedÒ at")) + updated_at = models.DateTimeField(auto_now=True, verbose_name=_("Updated at")) is_published = models.BooleanField(_("published"), default=False) class Meta: From fe291dba420eb6896a4f01d18f2cf2a4a6ee0d55 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Sat, 27 Apr 2024 16:01:01 +0200 Subject: [PATCH 06/86] Fix double API urlmodel related bugs --- backend/core/models.py | 26 ++- backend/library/helpers.py | 8 +- backend/library/serializers.py | 5 + backend/library/utils.py | 14 +- backend/library/views.py | 210 +++++++++--------- .../ModelTable/LibraryActions.svelte | 3 +- .../components/ModelTable/ModelTable.svelte | 8 +- .../routes/(app)/libraries/+page.server.ts | 9 +- .../src/routes/(app)/libraries/+page.svelte | 3 +- .../routes/(app)/libraries/[id=urn]/+page.ts | 7 +- .../(app)/libraries/[id=urn]/+server.ts | 8 +- .../(app)/libraries/[id=urn]/tree/+server.ts | 6 +- 12 files changed, 163 insertions(+), 144 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 6bdcce3c3..1a1ff3cb1 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -104,9 +104,6 @@ class Meta: objects_meta = models.JSONField() dependencies = models.JSONField(null=True) # models.CharField(blank=False,null=True,max_length=16384) - def get_dependencies(self) -> List[str] : # We should add some kind of descriptor to LibaryMixin so that model_instance.dependencies directly returns a list instead of a string, using a function to get the desired value of a field is bad - return [] if self.dependencies is None else self.dependencies.split(" ") - class StoredLibrary(LibraryMixin): is_obsolete = models.BooleanField(default=False) is_imported = models.BooleanField(default=False) @@ -125,10 +122,7 @@ def __init_class__(cls) : ) @staticmethod - def store_library_file(fname: str) -> Union[str,None] : - with open(fname,"rb") as f : - library_content = f.read() - + def store_libary_content(library_content: str) -> Union[str,None] : hash_checksum = sha256(library_content) if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET : return None # We do not store the libary if its hash checksum is in the database. @@ -169,7 +163,7 @@ def store_library_file(fname: str) -> Union[str,None] : for key, value in library_data["objects"].items() } - dependencies = library_data.get("dependencies") # I don't want whitespaces in URN anymore nontheless + dependencies = library_data.get("dependencies",[]) # I don't want whitespaces in URN anymore nontheless library_objects = json.dumps(library_data["objects"]) StoredLibrary.objects.create( @@ -192,10 +186,20 @@ def store_library_file(fname: str) -> Union[str,None] : content=library_objects ) + @staticmethod + def store_library_file(fname: str) -> Union[str,None] : + with open(fname,"rb") as f : + library_content = f.read() + return StoredLibrary.store_libary_content(library_content) + def loads(self) -> Union[str,None] : from library.utils import LibraryImporter library_importer = LibraryImporter(self) - return library_importer.import_library() + error_msg = library_importer.import_library() + if error_msg is None : + self.is_imported = True + self.save() + return error_msg class LoadedLibrary(LibraryMixin): dependencies = models.ManyToManyField( @@ -261,7 +265,9 @@ def delete(self, *args, **kwargs): f"This library is a dependency of {dependent_libraries.count()} other libraries" ) super(LoadedLibrary, self).delete(*args, **kwargs) - + stored_library = StoredLibrary.objects.get(urn=self.urn,locale=self.locale,version=self.version) # I don't if it works yet + stored_library.is_imported = False + stored_library.save() class Threat(ReferentialObjectMixin, PublishInRootFolderMixin): library = models.ForeignKey( diff --git a/backend/library/helpers.py b/backend/library/helpers.py index 2024b82b2..c338e6933 100644 --- a/backend/library/helpers.py +++ b/backend/library/helpers.py @@ -1,17 +1,17 @@ from core.models import RequirementNode from iam.models import Folder - -def preview_library(library_objects) -> dict[str, list]: +# Change the name of this function +def preview_library(framework: dict) -> dict[str, list]: """ Function to create temporary requirement nodes list Used to display requirements in tree view inside library detail view """ preview = {} requirement_nodes_list = [] - if library_objects["framework"].get("requirement_nodes"): + if framework.get("requirement_nodes"): index = 0 - for requirement_node in library_objects["framework"]["requirement_nodes"]: + for requirement_node in framework["requirement_nodes"]: index += 1 requirement_nodes_list.append( RequirementNode( diff --git a/backend/library/serializers.py b/backend/library/serializers.py index 7bd6d9b51..667017a15 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -35,6 +35,11 @@ class Meta: model = StoredLibrary fields = "__all__" +class LoadedLibraryDetailedSerializer(serializers.ModelSerializer): + class Meta: + model = LoadedLibrary + fields = "__all__" + """ class StoredLibraryReadSerializer(StoredLibraryWriteSerializer): content = serializers.SerializerMethodField() diff --git a/backend/library/utils.py b/backend/library/utils.py index 19c0e5e30..2c35a1daf 100644 --- a/backend/library/utils.py +++ b/backend/library/utils.py @@ -551,9 +551,10 @@ def init(self) -> Union[str, None]: def check_and_import_dependencies(self): """Check and import library dependencies.""" - dependencies = self._library.get_dependencies() - for dependency_urn in dependencies: + if not self._library.dependencies : + return None + for dependency_urn in self._library.dependencies: if not LoadedLibrary.objects.filter(urn=dependency_urn).exists(): # import_library_view(get_library(dependency)) dependency = StoredLibrary.objects.get(urn=dependency_urn,is_obsolete=False) # We only fetch by URN without thinking about what locale, that may be a problem in the future. @@ -609,11 +610,12 @@ def import_objects(self, library_object): def _import_library(self): library_object = self.create_or_update_library() self.import_objects(library_object) - library_object.dependencies.set( - LoadedLibrary.objects.filter( - urn__in=self._library.get_dependencies() + if (dependencies := self._library.dependencies) : + library_object.dependencies.set( + LoadedLibrary.objects.filter( + urn__in=dependencies + ) ) - ) def import_library(self): """Main method to import a library.""" diff --git a/backend/library/views.py b/backend/library/views.py index 809ef6f0e..ca5f66075 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -28,46 +28,63 @@ from rest_framework.decorators import action from rest_framework.response import Response -from .serializers import StoredLibraryDetailedSerializer, LoadedLibrarySerializer, LibraryUploadSerializer +from .serializers import StoredLibraryDetailedSerializer, LoadedLibrarySerializer, LoadedLibraryDetailedSerializer, LibraryUploadSerializer from .utils import LibraryImporter, get_available_libraries, get_library, import_library_view class StoredLibraryViewSet(viewsets.ModelViewSet): # serializer_class = StoredLibrarySerializer - # parser_classes = [FileUploadParser] + parser_classes = [FileUploadParser] # solve issue with URN containing dot, see https://stackoverflow.com/questions/27963899/django-rest-framework-using-dot-in-url lookup_value_regex = r"[\w.:-]+" model = StoredLibrary + queryset = StoredLibrary.objects.filter(is_obsolete=False) def list(self, request, *args, **kwargs): - print("WOOWOWOWOWOWOWOWOWOW\n"*20) if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - available_libraries = StoredLibrary.objects.filter(is_obsolete=False).values("id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") # The frontend doesn't receive the obsolete libraries for now. + available_libraries = self.queryset.values("id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta","is_imported") # The frontend doesn't receive the obsolete libraries for now. return Response({"results": available_libraries}) + def destroy(self, request, pk): # We may have to also get the locale of the library we want to delete in the future for this method and all other libary viewset methods which goal is to apply an operation on a specific library + if not RoleAssignment.is_access_allowed( + user=request.user, + perm=Permission.objects.get(codename="delete_storedlibrary"), + folder=Folder.get_root_folder(), + ): + return Response(status=status.HTTP_403_FORBIDDEN) + + try : + lib = self.queryset.get(urn=pk) # the libraries with is_obsolete=True are not displayed in the frontend and therefore not meant to be destroyable (at least yet) + except : + return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + + lib.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + def retrieve(self, request, *args, pk, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - lib = StoredLibrary.objects.get(urn=pk,is_obsolete=False) # Handle the exception if the pk urn doesn't exist - print(f"-------- {pk}"*20) - print(f"FETCHED {lib}") - # return Response(StoredLibraryDetailedSerializer(lib).data) + try : + lib = self.queryset.get(urn=pk) + except : + return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) return Response(StoredLibraryDetailedSerializer(lib).data) @action(detail=True, methods=["get"], url_path="import") - def import_library(self, request, pk=None): + def import_library(self, request, pk): if not RoleAssignment.is_access_allowed( user=request.user, perm=Permission.objects.get(codename="add_loadedlibrary"), folder=Folder.get_root_folder(), ): return Response(status=status.HTTP_403_FORBIDDEN) - library = StoredLibrary.objects.get(urn=pk) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. + try : + library = StoredLibrary.objects.get(urn=pk) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. + except : + return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - # return Response({"error":f"ERROR !!! {str(lib)}"},status=status.HTTP_403_FORBIDDEN) - # library = get_library(pk) try: error_msg = library.loads() if error_msg is not None : @@ -86,55 +103,87 @@ def import_library(self, request, pk=None): @action(detail=True, methods=["get"]) def tree(self, request, pk): - lib = StoredLibrary.objects.get(urn=pk) # It should return a 404 error if the query doesn't return anything and therefore raise an exception - library_objects = json.loads(lib.content) # We need caching for this - """library = { - "id": lib.id, - "urn": lib.urn, - "name": lib.name, - "description": lib.description, - "provider": lib.provider, - "packager": lib.packager, - "copyright": lib.copyright, - "reference_count": lib.reference_count, - "objects": library_objects, - }""" - """if not library: - return Response( - data="This library does not exist.", status=HTTP_404_NOT_FOUND - )""" - if not library_objects.get("framework"): + try : + lib = StoredLibrary.objects.get(urn=pk) + except : + return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + + library_objects = json.loads(lib.content) # We may need caching for this + if not (framework := library_objects.get("framework")) : return Response( data="This library does not include a framework.", status=HTTP_400_BAD_REQUEST, ) - preview = preview_library(library_objects) + preview = preview_library(framework) return Response( get_sorted_requirement_nodes(preview.get("requirement_nodes"), None) ) + @action(detail=False, methods=["post"], url_path="upload") + def upload_library(self, request): + if not request.data: + return HttpResponse( + json.dumps({"error": "noFileDetected"}), status=HTTP_400_BAD_REQUEST + ) + + try: + attachment = request.FILES["file"] + validate_file_extension(attachment) + # Use safe_load to prevent arbitrary code execution. + + content = attachment.read() # Should we read it chunck by chunck or ensure that the file size of the libary content is reasonnable before reading ? + + error_msg = StoredLibrary.store_libary_content(content) + + if error_msg is not None: + return HttpResponse( + json.dumps({"error": error_msg}), + status=HTTP_422_UNPROCESSABLE_ENTITY, + ) + + return HttpResponse(json.dumps({}), status=HTTP_200_OK) + except IntegrityError: + return HttpResponse( + json.dumps({"error": "libraryAlreadyImportedError"}), + status=HTTP_400_BAD_REQUEST, + ) + except : + return HttpResponse( + json.dumps({"error": "invalidLibraryFileError"}), + status=HTTP_400_BAD_REQUEST, + ) + class LoadedLibraryViewSet(viewsets.ModelViewSet): - serializer_class = LoadedLibrarySerializer - parser_classes = [FileUploadParser] + # serializer_class = LoadedLibrarySerializer + # parser_classes = [FileUploadParser] # solve issue with URN containing dot, see https://stackoverflow.com/questions/27963899/django-rest-framework-using-dot-in-url lookup_value_regex = r"[\w.:-]+" model = LoadedLibrary + queryset = LoadedLibrary.objects.all() def list(self, request, *args, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - available_libraries = LoadedLibrary.objects.all().values("id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta") - return Response({"results": available_libraries}) - # return Response({"results": get_available_libraries()}) + loaded_libraries = [{ + key: getattr(library,key) + for key in ["id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta","reference_count"] + } + for library in LoadedLibrary.objects.all() + ] + return Response({"results": loaded_libraries}) def retrieve(self, request, *args, pk, **kwargs): if not "view_loadedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - raise NotImplementedError() # This method must me implemented - # library = get_library(pk) - # return Response(library) + try : + lib = LoadedLibrary.objects.get(urn=pk) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist + except : + return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + data = LoadedLibraryDetailedSerializer(lib).data + data["objects"] = lib._objects + return Response(data) def destroy(self, request, *args, pk, **kwargs): if not RoleAssignment.is_access_allowed( @@ -144,84 +193,35 @@ def destroy(self, request, *args, pk, **kwargs): ): return Response(status=status.HTTP_403_FORBIDDEN) - library = get_library(pk) - - if library is None: + try : + lib = LoadedLibrary.objects.get(urn=pk) + except : return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - # "reference_count" is not always defined (is this normal ?) - if library.get("reference_count", 0) != 0: + if lib.reference_count != 0: return Response( data="Library cannot be deleted because it has references.", status=status.HTTP_400_BAD_REQUEST, ) - try: - LoadedLibrary.objects.get(id=library.get("id")).delete() - except IntegrityError as e: - # TODO: Log the exception if logging is set up - # logging.exception("Integrity error while deleting library: %s", e) - print(e) - except Exception as e: - # TODO: Log the exception if logging is set up - # logging.exception("Unexpected error while deleting library: %s", e) - print(e) - return Response( - data="Unexpected error occurred while deleting the library.", - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - + lib.delete() return Response(status=status.HTTP_204_NO_CONTENT) @action(detail=True, methods=["get"]) - def tree(self, request, pk): - lib = get_library(pk) - # library_objects = json.loads(lib.content) # How to this for loaded libarries - if not library: - return Response( - data="This library does not exist.", status=HTTP_404_NOT_FOUND - ) - if not library["objects"].get("framework"): + def tree(self, request, pk): # We must ensure that users that are not allowed to read the content of libraries can't have any access to them either from the /api/{URLModel/{library_urn}/tree view or the /api/{URLModel}/{library_urn} view. + try : + lib = LoadedLibrary.objects.get(urn=pk) + except : + return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + + if lib.frameworks.count() == 0: return Response( - data="This library does not include a framework.", - status=HTTP_400_BAD_REQUEST, + data="This library doesn't contain any framework.", status=HTTP_404_NOT_FOUND ) - preview = preview_library(library_objects) + + framework = lib.frameworks.first() + requirement_nodes = framework.requirement_nodes.all() return Response( - get_sorted_requirement_nodes(preview.get("requirement_nodes"), None) + get_sorted_requirement_nodes(requirement_nodes, None) ) - @action(detail=False, methods=["post"], url_path="upload") - def upload_library(self, request): - if not request.data: - return HttpResponse( - json.dumps({"error": "noFileDetected"}), status=HTTP_400_BAD_REQUEST - ) - - try: - attachment = request.FILES["file"] - validate_file_extension(attachment) - # Use safe_load to prevent arbitrary code execution. - library = yaml.safe_load(attachment) - - # This code doesn't handle the library "dependencies" field yet as decribed in the architecture. - - error_msg = import_library_view(library) - - if error_msg is not None: - return HttpResponse( - json.dumps({"error": error_msg}), - status=HTTP_422_UNPROCESSABLE_ENTITY, - ) - - return HttpResponse(json.dumps({}), status=HTTP_200_OK) - except IntegrityError: - return HttpResponse( - json.dumps({"error": "libraryAlreadyImportedError"}), - status=HTTP_400_BAD_REQUEST, - ) - except: - return HttpResponse( - json.dumps({"error": "invalidLibraryFileError"}), - status=HTTP_400_BAD_REQUEST, - ) diff --git a/frontend/src/lib/components/ModelTable/LibraryActions.svelte b/frontend/src/lib/components/ModelTable/LibraryActions.svelte index b85402d12..650e932a4 100644 --- a/frontend/src/lib/components/ModelTable/LibraryActions.svelte +++ b/frontend/src/lib/components/ModelTable/LibraryActions.svelte @@ -24,7 +24,8 @@ } -{#if !library.id}{#if loading.form && loading.library === library.urn} +{#if !library.is_imported} + {#if loading.form && loading.library === library.urn}
-{#if tabSet === 1} +{#if tabSet === 0}
{#await superValidate(zod(LibraryUploadSchema))}

{m.loadingLibraryUploadButton()}...

diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+page.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+page.ts index b1756f740..3aa9915b2 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+page.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+page.ts @@ -1,10 +1,11 @@ import type { PageLoad } from './$types'; -export const load: PageLoad = async ({ fetch, params }) => { +export const load: PageLoad = async ({ fetch, params, url }) => { const endpoint = `/libraries/${params.id}`; + const queryParams = url.searchParams.toString(); return { - tree: fetch(`/libraries/${params.id}/tree`).then((res) => res.json()) ?? {}, - library: await fetch(endpoint).then((res) => res.json()) + tree: fetch(`${endpoint}/tree?${queryParams}`).then((res) => res.json()) ?? {}, + library: await fetch(`${endpoint}?${queryParams}`).then((res) => res.json()) }; }; diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index 54d5ed441..e77b95394 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -4,14 +4,18 @@ import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, url, params }) => { - const endpoint = `${BASE_API_URL}/stored-libraries/${params.id}/`; + const isLoaded = url.searchParams.has("loaded"); + const URLModel = isLoaded ? "loaded-libraries" : "stored-libraries"; + const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/`; const res = await fetch(endpoint); if (!res.ok) { error(res.status as NumericRange<400, 599>, await res.json()); } const data = await res.json(); - data.objects = JSON.parse(data.content); + if (!isLoaded) { + data.objects = JSON.parse(data.content); + } return new Response(JSON.stringify(data), { headers: { diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index 50faf622f..e63dc91b4 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -4,10 +4,8 @@ import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, params, url }) => { - /* const endpoint = `${BASE_API_URL}${url.pathname}/${ - url.searchParams ? '?' + url.searchParams.toString() : '' - }`; */ - const endpoint = `${BASE_API_URL}/stored-libraries/${params.id}/`; + const URLModel = url.searchParams.has("loaded") ? "loaded-libraries" : "stored-libraries"; + const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/tree`; const res = await fetch(endpoint); if (!res.ok) { From 6e30549fa72ec6fbf7031de4015931c5229850ea Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 12:25:08 +0200 Subject: [PATCH 07/86] Remove test code --- backend/core/apps.py | 48 ++++---------------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/backend/core/apps.py b/backend/core/apps.py index 935f0c2e0..743ef689d 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from django.db.models.signals import post_migrate -from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL, LIBRARIES_PATH -import sys, os +from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL +import os READER_PERMISSIONS_LIST = [ "view_project", @@ -245,7 +245,8 @@ "restore", ] -def startup(sender: AppConfig,**kwargs): + +def startup(sender: AppConfig, **kwargs): """ Implement CISO Assistant 1.0 default Roles and User Groups during migrate This makes sure root folder and global groups are defined before any other object is created @@ -253,7 +254,6 @@ def startup(sender: AppConfig,**kwargs): """ from django.contrib.auth.models import Permission from iam.models import Folder, Role, RoleAssignment, User, UserGroup - from core.models import StoredLibrary print("startup handler: initialize database") @@ -353,53 +353,13 @@ def startup(sender: AppConfig,**kwargs): except Exception as e: print(e) # NOTE: Add this exception in the logger -from django.db import connection class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "core" verbose_name = "Core" - def _ready(self): - from core.models import StoredLibrary - import time - - db_tables = set(connection.introspection.table_names()) - required_models = [StoredLibrary] - required_tables = set(model._meta.db_table for model in required_models) - if not required_tables.issubset(db_tables) : - return None - - """for lib in StoredLibrary.objects.all() : - # Delete this loop ! - lib.delete()""" - - # ./library/libraries/soc2-2017.yaml - - # Remove the 3 following lines after - """from core.models import LoadedLibrary - for lib in StoredLibrary.objects.all() : - lib.delete()""" - - start = time.perf_counter() - StoredLibrary.__init_class__() - print(f"Importing new libraries...") - for fname in os.listdir(LIBRARIES_PATH) : - fname = str(LIBRARIES_PATH / fname) - if fname.endswith(".yaml") : - error = StoredLibrary.store_library_file(fname) - if error is not None : - print(f"[ERROR] Can't import libary file '{fname}' : {error}",file=sys.stderr) - end = time.perf_counter() - print(f"Execution time = {end-start}") - stored_libaries = [*StoredLibrary.objects.all()] - print(f"There are {len(stored_libaries)} stored libraries :D !") - print(f"Stored libaries : {stored_libaries}") - print("Django initialization finished !") - def ready(self): - self._ready() # avoid post_migrate handler if we are in the main, as it interferes with restore if not os.environ.get("RUN_MAIN"): post_migrate.connect(startup, sender=self) - From c3ff61733ca81dd040b0d9e327bf442d97980e3b Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 12:28:01 +0200 Subject: [PATCH 08/86] chore: Merge conflicting migrations --- .../core/migrations/0011_merge_20240429_1027.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 backend/core/migrations/0011_merge_20240429_1027.py diff --git a/backend/core/migrations/0011_merge_20240429_1027.py b/backend/core/migrations/0011_merge_20240429_1027.py new file mode 100644 index 000000000..fa755ee88 --- /dev/null +++ b/backend/core/migrations/0011_merge_20240429_1027.py @@ -0,0 +1,14 @@ +# Generated by Django 5.0.4 on 2024-04-29 10:27 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0009_framework_max_score_framework_min_score_and_more'), + ('core', '0010_alter_storedlibrary_dependencies'), + ] + + operations = [ + ] From e8ac2a378590b436ae1306406f912970fad3e46a Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 12:51:31 +0200 Subject: [PATCH 09/86] chore: Run formatter --- backend/ciso_assistant/settings.py | 2 +- ...ibrary_alter_framework_library_and_more.py | 308 ++++++++++++++---- .../0010_alter_storedlibrary_dependencies.py | 7 +- .../migrations/0011_merge_20240429_1027.py | 8 +- backend/core/models.py | 87 ++--- backend/core/serializers.py | 2 + backend/core/utils.py | 3 + backend/library/helpers.py | 1 + backend/library/serializers.py | 17 +- backend/library/views.py | 124 ++++--- 10 files changed, 412 insertions(+), 147 deletions(-) diff --git a/backend/ciso_assistant/settings.py b/backend/ciso_assistant/settings.py index 2d30045a6..fa4963b07 100644 --- a/backend/ciso_assistant/settings.py +++ b/backend/ciso_assistant/settings.py @@ -275,7 +275,7 @@ def set_ciso_assistant_url(_, __, event_dict): # https://docs.djangoproject.com/en/4.2/ref/settings/#databases # SQLIte file can be changed, useful for tests -SQLITE_FILE = os.environ.get('SQLITE_FILE', BASE_DIR / "db/ciso-assistant.sqlite3") +SQLITE_FILE = os.environ.get("SQLITE_FILE", BASE_DIR / "db/ciso-assistant.sqlite3") LIBRARIES_PATH = library_path = BASE_DIR / "library/libraries" if "POSTGRES_NAME" in os.environ: diff --git a/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py b/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py index 47ff8a104..8cb719dce 100644 --- a/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py +++ b/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py @@ -7,92 +7,276 @@ class Migration(migrations.Migration): - dependencies = [ - ('core', '0008_alter_complianceassessment_status_and_more'), - ('iam', '0002_purge_validator'), + ("core", "0008_alter_complianceassessment_status_and_more"), + ("iam", "0002_purge_validator"), ] operations = [ migrations.CreateModel( - name='LoadedLibrary', + name="LoadedLibrary", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), - ('updated_at', models.DateTimeField(auto_now=True, verbose_name='UpdatedÒ at')), - ('is_published', models.BooleanField(default=False, verbose_name='published')), - ('urn', models.CharField(blank=True, max_length=100, null=True, unique=True, verbose_name='URN')), - ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), - ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), - ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), - ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), - ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), - ('description', models.TextField(blank=True, null=True, verbose_name='Description')), - ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), - ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), - ('version', models.IntegerField(verbose_name='Version')), - ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), - ('builtin', models.BooleanField(default=False)), - ('objects_meta', models.JSONField()), - ('dependencies', models.ManyToManyField(blank=True, to='core.loadedlibrary', verbose_name='Dependencies')), - ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created_at", + models.DateTimeField(auto_now_add=True, verbose_name="Created at"), + ), + ( + "updated_at", + models.DateTimeField(auto_now=True, verbose_name="UpdatedÒ at"), + ), + ( + "is_published", + models.BooleanField(default=False, verbose_name="published"), + ), + ( + "urn", + models.CharField( + blank=True, + max_length=100, + null=True, + unique=True, + verbose_name="URN", + ), + ), + ( + "ref_id", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Reference ID", + ), + ), + ( + "locale", + models.CharField( + default="en", max_length=100, verbose_name="Locale" + ), + ), + ( + "default_locale", + models.BooleanField(default=True, verbose_name="Default locale"), + ), + ( + "provider", + models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider" + ), + ), + ( + "name", + models.CharField(max_length=200, null=True, verbose_name="Name"), + ), + ( + "description", + models.TextField(blank=True, null=True, verbose_name="Description"), + ), + ( + "annotation", + models.TextField(blank=True, null=True, verbose_name="Annotation"), + ), + ( + "copyright", + models.CharField( + blank=True, max_length=4096, null=True, verbose_name="Copyright" + ), + ), + ("version", models.IntegerField(verbose_name="Version")), + ( + "packager", + models.CharField( + blank=True, + help_text="Packager of the library", + max_length=100, + null=True, + verbose_name="Packager", + ), + ), + ("builtin", models.BooleanField(default=False)), + ("objects_meta", models.JSONField()), + ( + "dependencies", + models.ManyToManyField( + blank=True, to="core.loadedlibrary", verbose_name="Dependencies" + ), + ), + ( + "folder", + models.ForeignKey( + default=iam.models.Folder.get_root_folder, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_folder", + to="iam.folder", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.AlterField( - model_name='framework', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='frameworks', to='core.loadedlibrary'), + model_name="framework", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="frameworks", + to="core.loadedlibrary", + ), ), migrations.AlterField( - model_name='referencecontrol', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='reference_controls', to='core.loadedlibrary'), + model_name="referencecontrol", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="reference_controls", + to="core.loadedlibrary", + ), ), migrations.AlterField( - model_name='riskmatrix', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='risk_matrices', to='core.loadedlibrary'), + model_name="riskmatrix", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="risk_matrices", + to="core.loadedlibrary", + ), ), migrations.AlterField( - model_name='threat', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='threats', to='core.loadedlibrary'), + model_name="threat", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="threats", + to="core.loadedlibrary", + ), ), migrations.CreateModel( - name='StoredLibrary', + name="StoredLibrary", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), - ('updated_at', models.DateTimeField(auto_now=True, verbose_name='UpdatedÒ at')), - ('is_published', models.BooleanField(default=False, verbose_name='published')), - ('urn', models.CharField(blank=True, max_length=100, null=True, unique=True, verbose_name='URN')), - ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), - ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), - ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), - ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), - ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), - ('description', models.TextField(blank=True, null=True, verbose_name='Description')), - ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), - ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), - ('version', models.IntegerField(verbose_name='Version')), - ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), - ('builtin', models.BooleanField(default=False)), - ('objects_meta', models.JSONField()), - ('dependencies', models.CharField(max_length=16384, null=True)), - ('is_obsolete', models.BooleanField(default=False)), - ('is_imported', models.BooleanField(default=False)), - ('hash_checksum', models.CharField(max_length=64)), - ('content', models.TextField()), - ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created_at", + models.DateTimeField(auto_now_add=True, verbose_name="Created at"), + ), + ( + "updated_at", + models.DateTimeField(auto_now=True, verbose_name="UpdatedÒ at"), + ), + ( + "is_published", + models.BooleanField(default=False, verbose_name="published"), + ), + ( + "urn", + models.CharField( + blank=True, + max_length=100, + null=True, + unique=True, + verbose_name="URN", + ), + ), + ( + "ref_id", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Reference ID", + ), + ), + ( + "locale", + models.CharField( + default="en", max_length=100, verbose_name="Locale" + ), + ), + ( + "default_locale", + models.BooleanField(default=True, verbose_name="Default locale"), + ), + ( + "provider", + models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider" + ), + ), + ( + "name", + models.CharField(max_length=200, null=True, verbose_name="Name"), + ), + ( + "description", + models.TextField(blank=True, null=True, verbose_name="Description"), + ), + ( + "annotation", + models.TextField(blank=True, null=True, verbose_name="Annotation"), + ), + ( + "copyright", + models.CharField( + blank=True, max_length=4096, null=True, verbose_name="Copyright" + ), + ), + ("version", models.IntegerField(verbose_name="Version")), + ( + "packager", + models.CharField( + blank=True, + help_text="Packager of the library", + max_length=100, + null=True, + verbose_name="Packager", + ), + ), + ("builtin", models.BooleanField(default=False)), + ("objects_meta", models.JSONField()), + ("dependencies", models.CharField(max_length=16384, null=True)), + ("is_obsolete", models.BooleanField(default=False)), + ("is_imported", models.BooleanField(default=False)), + ("hash_checksum", models.CharField(max_length=64)), + ("content", models.TextField()), + ( + "folder", + models.ForeignKey( + default=iam.models.Folder.get_root_folder, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_folder", + to="iam.folder", + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.DeleteModel( - name='Library', + name="Library", ), ] diff --git a/backend/core/migrations/0010_alter_storedlibrary_dependencies.py b/backend/core/migrations/0010_alter_storedlibrary_dependencies.py index 27bf93e4e..d87bebe82 100644 --- a/backend/core/migrations/0010_alter_storedlibrary_dependencies.py +++ b/backend/core/migrations/0010_alter_storedlibrary_dependencies.py @@ -4,15 +4,14 @@ class Migration(migrations.Migration): - dependencies = [ - ('core', '0009_loadedlibrary_alter_framework_library_and_more'), + ("core", "0009_loadedlibrary_alter_framework_library_and_more"), ] operations = [ migrations.AlterField( - model_name='storedlibrary', - name='dependencies', + model_name="storedlibrary", + name="dependencies", field=models.JSONField(null=True), ), ] diff --git a/backend/core/migrations/0011_merge_20240429_1027.py b/backend/core/migrations/0011_merge_20240429_1027.py index fa755ee88..7afaf3980 100644 --- a/backend/core/migrations/0011_merge_20240429_1027.py +++ b/backend/core/migrations/0011_merge_20240429_1027.py @@ -4,11 +4,9 @@ class Migration(migrations.Migration): - dependencies = [ - ('core', '0009_framework_max_score_framework_min_score_and_more'), - ('core', '0010_alter_storedlibrary_dependencies'), + ("core", "0009_framework_max_score_framework_min_score_and_more"), + ("core", "0010_alter_storedlibrary_dependencies"), ] - operations = [ - ] + operations = [] diff --git a/backend/core/models.py b/backend/core/models.py index 2b6fc755f..f0864c14b 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -84,6 +84,7 @@ def display_long(self) -> str: def __str__(self) -> str: return self.display_short + class LibraryMixin(ReferentialObjectMixin): class Meta: abstract = True @@ -101,7 +102,10 @@ class Meta: ) builtin = models.BooleanField(default=False) objects_meta = models.JSONField() - dependencies = models.JSONField(null=True) # models.CharField(blank=False,null=True,max_length=16384) + dependencies = models.JSONField( + null=True + ) # models.CharField(blank=False,null=True,max_length=16384) + class StoredLibrary(LibraryMixin): is_obsolete = models.BooleanField(default=False) @@ -109,60 +113,58 @@ class StoredLibrary(LibraryMixin): hash_checksum = models.CharField(max_length=64) content = models.TextField() - REQUIRED_FIELDS = {"urn","name","version","objects"} + REQUIRED_FIELDS = {"urn", "name", "version", "objects"} FIELDS_VERIFIERS = {} - HASH_CHECKSUM_SET = set() # For now a library isn't updated if its SHA256 checksum has already been registered in the database. + HASH_CHECKSUM_SET = set() # For now a library isn't updated if its SHA256 checksum has already been registered in the database. @classmethod - def __init_class__(cls) : + def __init_class__(cls): cls.HASH_CHECKSUM_SET = set( - value["hash_checksum"] - for value in cls.objects.values("hash_checksum") + value["hash_checksum"] for value in cls.objects.values("hash_checksum") ) @staticmethod - def store_libary_content(library_content: str) -> Union[str,None] : + def store_libary_content(library_content: str) -> Union[str, None]: hash_checksum = sha256(library_content) - if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET : - return None # We do not store the libary if its hash checksum is in the database. + if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET: + return None # We do not store the libary if its hash checksum is in the database. # urn(str:max_length=100), version(int), name(str:max_length=200) - try : + try: library_data = yaml.safe_load(library_content) - except Exception : + except Exception: return "Invalid formatted file, the library file must be formatted in YAML." missing_fields = StoredLibrary.REQUIRED_FIELDS - set(library_data.keys()) - if missing_fields : - return "The following fields are missing : {}".format(", ".join(repr(field) for field in missing_fields)) + if missing_fields: + return "The following fields are missing : {}".format( + ", ".join(repr(field) for field in missing_fields) + ) urn = library_data["urn"] - locale = library_data.get("locale","en") + locale = library_data.get("locale", "en") version = library_data["version"] - library_matches = [*StoredLibrary.objects.filter( - urn=urn, - locale=locale - )] - if any( - libary.version >= version - for libary in library_matches - ) : + library_matches = [*StoredLibrary.objects.filter(urn=urn, locale=locale)] + if any(libary.version >= version for libary in library_matches): # The library isn't stored if it's obsolete due to be a too old version of itself. - return "A library with the urn '{}', a locale '{}' with a superior superior or equal to {} is already stored in the database.".format(urn,locale,version) + return "A library with the urn '{}', a locale '{}' with a superior superior or equal to {} is already stored in the database.".format( + urn, locale, version + ) - for library in library_matches : + for library in library_matches: libary.is_obsolete = True - library.save() # If a user delete a library from the libary store we must set the is_obsolete value of its most recent obsolete version to False. + library.save() # If a user delete a library from the libary store we must set the is_obsolete value of its most recent obsolete version to False. objects_meta = { - key: len(value) - for key, value in library_data["objects"].items() + key: len(value) for key, value in library_data["objects"].items() } - dependencies = library_data.get("dependencies",[]) # I don't want whitespaces in URN anymore nontheless + dependencies = library_data.get( + "dependencies", [] + ) # I don't want whitespaces in URN anymore nontheless library_objects = json.dumps(library_data["objects"]) StoredLibrary.objects.create( @@ -172,7 +174,7 @@ def store_libary_content(library_content: str) -> Union[str,None] : locale=locale, version=version, ref_id=library_data["ref_id"], - default_locale=False, # We don't care about this value yet. + default_locale=False, # We don't care about this value yet. description=library_data.get("description"), annotation=library_data.get("annotation"), copyright=library_data.get("copyright"), @@ -180,26 +182,30 @@ def store_libary_content(library_content: str) -> Union[str,None] : packager=library_data.get("packager"), objects_meta=objects_meta, dependencies=dependencies, - builtin=library_data.get("builtin",False), # We have to add a "builtin: true" line to every builtin library file. + builtin=library_data.get( + "builtin", False + ), # We have to add a "builtin: true" line to every builtin library file. hash_checksum=hash_checksum, - content=library_objects + content=library_objects, ) @staticmethod - def store_library_file(fname: str) -> Union[str,None] : - with open(fname,"rb") as f : + def store_library_file(fname: str) -> Union[str, None]: + with open(fname, "rb") as f: library_content = f.read() return StoredLibrary.store_libary_content(library_content) - def loads(self) -> Union[str,None] : + def loads(self) -> Union[str, None]: from library.utils import LibraryImporter + library_importer = LibraryImporter(self) error_msg = library_importer.import_library() - if error_msg is None : + if error_msg is None: self.is_imported = True self.save() return error_msg + class LoadedLibrary(LibraryMixin): dependencies = models.ManyToManyField( "self", blank=True, verbose_name=_("Dependencies"), symmetrical=False @@ -264,13 +270,20 @@ def delete(self, *args, **kwargs): f"This library is a dependency of {dependent_libraries.count()} other libraries" ) super(LoadedLibrary, self).delete(*args, **kwargs) - stored_library = StoredLibrary.objects.get(urn=self.urn,locale=self.locale,version=self.version) # I don't if it works yet + stored_library = StoredLibrary.objects.get( + urn=self.urn, locale=self.locale, version=self.version + ) # I don't if it works yet stored_library.is_imported = False stored_library.save() + class Threat(ReferentialObjectMixin, PublishInRootFolderMixin): library = models.ForeignKey( - LoadedLibrary, on_delete=models.CASCADE, null=True, blank=True, related_name="threats" + LoadedLibrary, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="threats", ) fields_to_check = ["ref_id", "name"] diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 9416d2736..2abb19b10 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -162,6 +162,7 @@ class ReferenceControlReadSerializer(ReferenceControlWriteSerializer): folder = FieldsRelatedField() library = FieldsRelatedField(["name", "urn"]) + """class LibraryReadSerializer(BaseModelSerializer): class Meta: model = LoadedLibrary @@ -174,6 +175,7 @@ class Meta: fields = "__all__" """ + class ThreatWriteSerializer(BaseModelSerializer): class Meta: model = Threat diff --git a/backend/core/utils.py b/backend/core/utils.py index a91a035a7..bb6eca6f9 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -3,17 +3,20 @@ from enum import Enum import hashlib + def camel_case(s): s = sub(r"(_|-)+", " ", s).title().replace(" ", "") return "".join([s[0].lower(), s[1:]]) + def sha256(string: bytes) -> str: """Return the SHA256-hashed hexadecimal representation of the bytes object given as argument.""" h = hashlib.new("SHA256") h.update(string) return h.hexdigest() + class RoleCodename(Enum): ADMINISTRATOR = "BI-RL-ADM" DOMAIN_MANAGER = "BI-RL-DMA" diff --git a/backend/library/helpers.py b/backend/library/helpers.py index 504b335ff..172a2f7c7 100644 --- a/backend/library/helpers.py +++ b/backend/library/helpers.py @@ -1,5 +1,6 @@ from core.models import RequirementNode + # Change the name of this function def preview_library(framework: dict) -> dict[str, list]: """ diff --git a/backend/library/serializers.py b/backend/library/serializers.py index 667017a15..ff129a42a 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -16,11 +16,20 @@ fields = serializers.DictField(child=serializers.CharField()) """ + class StoredLibrarySerializer(serializers.ModelSerializer): # Not used yet class Meta: model = StoredLibrary - fields = ["id","name","description","locale","version","builtin","objects_meta"] + fields = [ + "id", + "name", + "description", + "locale", + "version", + "builtin", + "objects_meta", + ] # name = serializers.CharField() # description = serializers.CharField() @@ -30,16 +39,19 @@ class Meta: # builtin = serializers.BooleanField() # objects_meta = serializers.JSONField() + class StoredLibraryDetailedSerializer(serializers.ModelSerializer): class Meta: model = StoredLibrary fields = "__all__" + class LoadedLibraryDetailedSerializer(serializers.ModelSerializer): class Meta: model = LoadedLibrary fields = "__all__" + """ class StoredLibraryReadSerializer(StoredLibraryWriteSerializer): content = serializers.SerializerMethodField() @@ -48,6 +60,7 @@ def get_content(self, content: bytes): return content.encode("utf-8") # Should we enforce UTF-8 for library files ? """ + class LoadedLibrarySerializer(serializers.Serializer): id = serializers.CharField() name = serializers.CharField() @@ -58,12 +71,14 @@ class LoadedLibrarySerializer(serializers.Serializer): copyright = serializers.CharField() builtin = serializers.BooleanField() + """class LibraryModelSerializer(BaseModelSerializer): class Meta: model = LoadedLibrary fields = "__all__" """ + class LibraryUploadSerializer(serializers.Serializer): file = serializers.FileField(required=True) diff --git a/backend/library/views.py b/backend/library/views.py index 160266203..48068c40d 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -26,8 +26,19 @@ from rest_framework.decorators import action from rest_framework.response import Response -from .serializers import StoredLibraryDetailedSerializer, LoadedLibrarySerializer, LoadedLibraryDetailedSerializer, LibraryUploadSerializer -from .utils import LibraryImporter, get_available_libraries, get_library, import_library_view +from .serializers import ( + StoredLibraryDetailedSerializer, + LoadedLibrarySerializer, + LoadedLibraryDetailedSerializer, + LibraryUploadSerializer, +) +from .utils import ( + LibraryImporter, + get_available_libraries, + get_library, + import_library_view, +) + class StoredLibraryViewSet(viewsets.ModelViewSet): # serializer_class = StoredLibrarySerializer @@ -41,11 +52,26 @@ class StoredLibraryViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - available_libraries = self.queryset.values("id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta","is_imported") # The frontend doesn't receive the obsolete libraries for now. + available_libraries = self.queryset.values( + "id", + "name", + "description", + "urn", + "ref_id", + "locale", + "version", + "packager", + "provider", + "builtin", + "objects_meta", + "is_imported", + ) # The frontend doesn't receive the obsolete libraries for now. return Response({"results": available_libraries}) - def destroy(self, request, pk): # We may have to also get the locale of the library we want to delete in the future for this method and all other libary viewset methods which goal is to apply an operation on a specific library + def destroy( + self, request, pk + ): # We may have to also get the locale of the library we want to delete in the future for this method and all other libary viewset methods which goal is to apply an operation on a specific library if not RoleAssignment.is_access_allowed( user=request.user, perm=Permission.objects.get(codename="delete_storedlibrary"), @@ -53,9 +79,11 @@ def destroy(self, request, pk): # We may have to also get the locale of the libr ): return Response(status=status.HTTP_403_FORBIDDEN) - try : - lib = self.queryset.get(urn=pk) # the libraries with is_obsolete=True are not displayed in the frontend and therefore not meant to be destroyable (at least yet) - except : + try: + lib = self.queryset.get( + urn=pk + ) # the libraries with is_obsolete=True are not displayed in the frontend and therefore not meant to be destroyable (at least yet) + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) lib.delete() @@ -64,9 +92,9 @@ def destroy(self, request, pk): # We may have to also get the locale of the libr def retrieve(self, request, *args, pk, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - try : + try: lib = self.queryset.get(urn=pk) - except : + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) return Response(StoredLibraryDetailedSerializer(lib).data) @@ -78,15 +106,20 @@ def import_library(self, request, pk): folder=Folder.get_root_folder(), ): return Response(status=status.HTTP_403_FORBIDDEN) - try : - library = StoredLibrary.objects.get(urn=pk) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. - except : + try: + library = StoredLibrary.objects.get( + urn=pk + ) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) try: error_msg = library.loads() - if error_msg is not None : - return Response({"status":"error","error":error_msg},status=status.HTTP_400_BAD_REQUEST) # This can cause translation issues + if error_msg is not None: + return Response( + {"status": "error", "error": error_msg}, + status=status.HTTP_400_BAD_REQUEST, + ) # This can cause translation issues return Response({"status": "success"}) except Exception as e: """print(f"ERROR {type(e)}") @@ -95,19 +128,19 @@ def import_library(self, request, pk): return Response( { "error": "Failed to load library, please check if it has dependencies" - }, # This must translated + }, # This must translated status=HTTP_422_UNPROCESSABLE_ENTITY, ) @action(detail=True, methods=["get"]) def tree(self, request, pk): - try : + try: lib = StoredLibrary.objects.get(urn=pk) - except : + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - library_objects = json.loads(lib.content) # We may need caching for this - if not (framework := library_objects.get("framework")) : + library_objects = json.loads(lib.content) # We may need caching for this + if not (framework := library_objects.get("framework")): return Response( data="This library does not include a framework.", status=HTTP_400_BAD_REQUEST, @@ -129,7 +162,7 @@ def upload_library(self, request): validate_file_extension(attachment) # Use safe_load to prevent arbitrary code execution. - content = attachment.read() # Should we read it chunck by chunck or ensure that the file size of the libary content is reasonnable before reading ? + content = attachment.read() # Should we read it chunck by chunck or ensure that the file size of the libary content is reasonnable before reading ? error_msg = StoredLibrary.store_libary_content(content) @@ -145,12 +178,13 @@ def upload_library(self, request): json.dumps({"error": "libraryAlreadyImportedError"}), status=HTTP_400_BAD_REQUEST, ) - except : + except: return HttpResponse( json.dumps({"error": "invalidLibraryFileError"}), status=HTTP_400_BAD_REQUEST, ) + class LoadedLibraryViewSet(viewsets.ModelViewSet): # serializer_class = LoadedLibrarySerializer # parser_classes = [FileUploadParser] @@ -164,9 +198,23 @@ def list(self, request, *args, **kwargs): if not "view_storedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - loaded_libraries = [{ - key: getattr(library,key) - for key in ["id","name","description","urn","ref_id","locale","version","packager","provider","builtin","objects_meta","reference_count"] + loaded_libraries = [ + { + key: getattr(library, key) + for key in [ + "id", + "name", + "description", + "urn", + "ref_id", + "locale", + "version", + "packager", + "provider", + "builtin", + "objects_meta", + "reference_count", + ] } for library in LoadedLibrary.objects.all() ] @@ -175,9 +223,11 @@ def list(self, request, *args, **kwargs): def retrieve(self, request, *args, pk, **kwargs): if not "view_loadedlibrary" in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) - try : - lib = LoadedLibrary.objects.get(urn=pk) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist - except : + try: + lib = LoadedLibrary.objects.get( + urn=pk + ) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) data = LoadedLibraryDetailedSerializer(lib).data data["objects"] = lib._objects @@ -191,9 +241,9 @@ def destroy(self, request, *args, pk, **kwargs): ): return Response(status=status.HTTP_403_FORBIDDEN) - try : + try: lib = LoadedLibrary.objects.get(urn=pk) - except : + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) if lib.reference_count != 0: @@ -206,20 +256,20 @@ def destroy(self, request, *args, pk, **kwargs): return Response(status=status.HTTP_204_NO_CONTENT) @action(detail=True, methods=["get"]) - def tree(self, request, pk): # We must ensure that users that are not allowed to read the content of libraries can't have any access to them either from the /api/{URLModel/{library_urn}/tree view or the /api/{URLModel}/{library_urn} view. - try : + def tree( + self, request, pk + ): # We must ensure that users that are not allowed to read the content of libraries can't have any access to them either from the /api/{URLModel/{library_urn}/tree view or the /api/{URLModel}/{library_urn} view. + try: lib = LoadedLibrary.objects.get(urn=pk) - except : + except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) if lib.frameworks.count() == 0: return Response( - data="This library doesn't contain any framework.", status=HTTP_404_NOT_FOUND + data="This library doesn't contain any framework.", + status=HTTP_404_NOT_FOUND, ) framework = lib.frameworks.first() requirement_nodes = framework.requirement_nodes.all() - return Response( - get_sorted_requirement_nodes(requirement_nodes, None) - ) - + return Response(get_sorted_requirement_nodes(requirement_nodes, None)) From a329fc64b1d14ebf862965ba7d0a8256514c9e0c Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 12:52:07 +0200 Subject: [PATCH 10/86] Fix some typing and typo --- backend/core/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index f0864c14b..344fa6b2e 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -124,7 +124,7 @@ def __init_class__(cls): ) @staticmethod - def store_libary_content(library_content: str) -> Union[str, None]: + def store_libary_content(library_content: bytes) -> Union[str, None]: hash_checksum = sha256(library_content) if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET: return None # We do not store the libary if its hash checksum is in the database. @@ -145,7 +145,7 @@ def store_libary_content(library_content: str) -> Union[str, None]: urn = library_data["urn"] locale = library_data.get("locale", "en") - version = library_data["version"] + version = int(library_data["version"]) library_matches = [*StoredLibrary.objects.filter(urn=urn, locale=locale)] if any(libary.version >= version for libary in library_matches): @@ -155,7 +155,7 @@ def store_libary_content(library_content: str) -> Union[str, None]: ) for library in library_matches: - libary.is_obsolete = True + library.is_obsolete = True library.save() # If a user delete a library from the libary store we must set the is_obsolete value of its most recent obsolete version to False. objects_meta = { From a52a555050e4148630dbbe518a2d43ac31099228 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 13:05:21 +0200 Subject: [PATCH 11/86] Undo regression-inducing changes to ModelTable component --- .../lib/components/ModelTable/ModelTable.svelte | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index 3eb610b35..d0db8a120 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -72,7 +72,7 @@ const rowMetaData = $rows[rowIndex].meta; /** @event {rowMetaData} selected - Fires when a table row is clicked. */ if (!rowMetaData[identifierField] || !URLModel) return; - goto(`/${LocalURLModel}/${rowMetaData[identifierField]}${detailQueryParameter}`); + goto(`/${URLModel}/${rowMetaData[identifierField]}`); } // Row Keydown Handler @@ -89,10 +89,6 @@ export let URLModel: urlModel | undefined = undefined; $: model = URLModel ? URL_MODEL_MAP[URLModel] : undefined; - export let LocalURLModel: urlModel | undefined = undefined; - LocalURLModel = LocalURLModel ?? URLModel; - export let detailQueryParameter: string | undefined; - detailQueryParameter = detailQueryParameter ? `?${detailQueryParameter}` : ""; const user = $page.data.user; @@ -268,16 +264,16 @@ {#if row.meta[identifierField]} {@const actionsComponent = field_component_map['actions']} - {#if $$slots.actionsHead} From 0e4dd164611ddee9fff9f38b1ee2df20f8a9a1ba Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 14:29:12 +0200 Subject: [PATCH 12/86] Improve libraries table row actions management --- .../ModelTable/LibraryActions.svelte | 2 +- .../components/ModelTable/ModelTable.svelte | 15 +-- frontend/src/lib/utils/crud.ts | 18 ++- frontend/src/lib/utils/types.ts | 2 +- .../routes/(app)/libraries/+page.server.ts | 122 ++++-------------- .../src/routes/(app)/libraries/+page.svelte | 16 +-- 6 files changed, 54 insertions(+), 121 deletions(-) diff --git a/frontend/src/lib/components/ModelTable/LibraryActions.svelte b/frontend/src/lib/components/ModelTable/LibraryActions.svelte index 650e932a4..bcaa1f7d3 100644 --- a/frontend/src/lib/components/ModelTable/LibraryActions.svelte +++ b/frontend/src/lib/components/ModelTable/LibraryActions.svelte @@ -24,7 +24,7 @@ } -{#if !library.is_imported} +{#if Object.hasOwn(library, 'is_imported') && !library.is_imported} {#if loading.form && loading.library === library.urn}
| undefined = undefined; export let URLModel: urlModel | undefined = undefined; - $: model = URLModel ? URL_MODEL_MAP[URLModel] : undefined; const user = $page.data.user; @@ -127,12 +126,11 @@ }); $: field_component_map = FIELD_COMPONENT_MAP[URLModel] ?? {}; - // const field_component_map = FIELD_MAP; const tagMap = FIELD_COLORED_TAG_MAP[URLModel] ?? {}; const taggedKeys = new Set(Object.keys(tagMap)); - // tagged_keys tag_map[key][value] + $: model = source.meta.urlmodel ? URL_MODEL_MAP[source.meta.urlmodel] : URL_MODEL_MAP[URLModel]; $: source, handler.setRows(data); @@ -264,16 +262,17 @@ {#if row.meta[identifierField]} {@const actionsComponent = field_component_map['actions']} + {@const actionsURLModel = source.meta.urlmodel ?? URLModel} {#if $$slots.actionsHead} diff --git a/frontend/src/lib/utils/crud.ts b/frontend/src/lib/utils/crud.ts index a75823479..b7d60bc51 100644 --- a/frontend/src/lib/utils/crud.ts +++ b/frontend/src/lib/utils/crud.ts @@ -404,7 +404,7 @@ export const URL_MODEL_MAP: ModelMap = { { field: 'compliance_assessment', urlModel: 'compliance-assessments' } ] }, - "stored-libraries": { + 'stored-libraries': { name: 'storedlibrary', localName: 'imported library', localNamePlural: 'imported libraries', @@ -412,7 +412,7 @@ export const URL_MODEL_MAP: ModelMap = { verboseName: 'Imported Library', verboseNamePlural: 'Imported Libraries' }, - "loaded-libraries": { + 'loaded-libraries': { name: 'loadedlibrary', localName: 'imported library', localNamePlural: 'imported libraries', @@ -426,14 +426,18 @@ export const FIELD_COMPONENT_MAP = { evidences: { attachment: EvidenceFilePreview }, - "stored-libraries": { + libraries: { locale: LanguageDisplay, actions: LibraryActions }, - "loaded-libraries": { - locale: LanguageDisplay - // actions: LibraryActions - }, + // "stored-libraries": { + // locale: LanguageDisplay, + // actions: LibraryActions + // }, + // "loaded-libraries": { + // locale: LanguageDisplay + // // actions: LibraryActions + // }, 'user-groups': { localization_dict: UserGroupNameDisplay } diff --git a/frontend/src/lib/utils/types.ts b/frontend/src/lib/utils/types.ts index 336dab80a..aea58e5a0 100644 --- a/frontend/src/lib/utils/types.ts +++ b/frontend/src/lib/utils/types.ts @@ -37,7 +37,7 @@ export const URL_MODEL = [ 'frameworks', 'requirements', 'requirement-assessments', - // 'libraries' + 'libraries' ] as const; export type urlModel = (typeof URL_MODEL)[number]; diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index 6aef60eb4..b788e0c40 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -9,139 +9,69 @@ import { z } from 'zod'; import { zod } from 'sveltekit-superforms/adapters'; import { tableSourceMapper } from '@skeletonlabs/skeleton'; import { listViewFields } from '$lib/utils/table'; -import type { Library, urlModel } from '$lib/utils/types'; +import type { Library } from '$lib/utils/types'; import * as m from '$paraglide/messages'; import { localItems } from '$lib/utils/locales'; import { languageTag } from '$paraglide/runtime'; - // ----------------------------------------------------------- // - export const load = (async ({ fetch }) => { const stored_libraries_endpoint = `${BASE_API_URL}/stored-libraries/`; const loaded_libaries_endpoint = `${BASE_API_URL}/loaded-libraries/`; - const [stored_libraries_res,loaded_libaries_res] = await Promise.all([fetch(stored_libraries_endpoint),fetch(loaded_libaries_endpoint)]); + const [stored_libraries_res, loaded_libaries_res] = await Promise.all([ + fetch(stored_libraries_endpoint), + fetch(loaded_libaries_endpoint) + ]); const storedLibraries = await stored_libraries_res.json().then((res) => res.results); const loadedLibraries = await loaded_libaries_res.json().then((res) => res.results); - const prepareRow = (row) => { + const prepareRow = (row: Record) => { row.overview = [ `Provider: ${row.provider}`, `Packager: ${row.packager}`, - ...Object.entries(row.objects_meta).map(([key,value]) => `${key}: ${value}`) + ...Object.entries(row.objects_meta).map(([key, value]) => `${key}: ${value}`) ]; - row.allowDeleteLibrary = row.allowDeleteLibrary = row.reference_count && row.reference_count > 0 ? false : true; - } + row.allowDeleteLibrary = row.allowDeleteLibrary = + row.reference_count && row.reference_count > 0 ? false : true; + }; storedLibraries.forEach(prepareRow); loadedLibraries.forEach(prepareRow); - // const headData: Record - - const makeHeadData = (modelName: string) => { - return listViewFields['stored-libraries'].body.reduce( - (obj, key, index) => { - obj[key] = listViewFields['stored-libraries'].head[index]; - return obj; - }, - {} - ); - } - - const makeBodyData = (libraries,modelName: string) => tableSourceMapper(libraries, listViewFields[modelName].body); - - const makeLibrariesTable = (libraries: Library[],modelName: string) => { - return { - head: makeHeadData(modelName), - body: makeBodyData(libraries,modelName), - meta: libraries - }; - }; - - const storedLibrariesTable = makeLibrariesTable(storedLibraries,'stored-libraries'); - const loadedLibrariesTable = makeLibrariesTable(loadedLibraries,'loaded-libraries'); - - const schema = z.object({ id: z.string() }); - const deleteForm = await superValidate(zod(schema)); - - return { storedLibrariesTable, loadedLibrariesTable, deleteForm } -}) satisfies PageServerLoad; - - + type libraryURLModel = 'stored-libraries' | 'loaded-libraries'; - - -const old_load = (async ({ fetch }) => { - const endpoint = `${BASE_API_URL}/libraries/`; - - const res = await fetch(endpoint); - const libraries: Library[] = await res.json().then((res) => res.results); - - function countObjects(library: Library) { - const result: { [key: string]: any } = new Object(); - for (const [key, value] of Object.entries(library.objects)) { - if (Array.isArray(value)) { - const str = key.charAt(0).toUpperCase() + key.slice(1).replace('_', ' '); - result[str] = value.length; - } else { - for (const [key2, value2] of Object.entries(value)) { - if (key2 === 'requirements') { - const str = key2.charAt(0).toUpperCase() + key2.slice(1); - result[str] = value2.length; - } - } - } - } - return result; - } - - libraries.forEach((row) => { - row.overview = [ - `Provider: ${row.provider}`, - `Packager: ${row.packager}`, - ...Object.entries(countObjects(row)).map(([key, value]) => `${key}: ${value}`) - ]; - row.allowDeleteLibrary = row.reference_count && row.reference_count > 0 ? false : true; - }); - - const headData: Record = listViewFields['libraries' as urlModel].body.reduce( - (obj, key, index) => { - obj[key] = listViewFields['libraries' as urlModel].head[index]; + const makeHeadData = (URLModel: libraryURLModel) => { + return listViewFields[URLModel].body.reduce((obj, key, index) => { + obj[key] = listViewFields[URLModel].head[index]; return obj; - }, - {} - ); + }, {}); + }; - const bodyData = (libraries) => - tableSourceMapper(libraries, listViewFields['libraries' as urlModel].body); + const makeBodyData = (libraries: Library[], URLModel: libraryURLModel) => + tableSourceMapper(libraries, listViewFields[URLModel].body); - const librariesTable: TableSource = (libraries: Library[]) => { + const makeLibrariesTable = (libraries: Library[], URLModel: libraryURLModel) => { return { - head: headData, - body: bodyData(libraries), - meta: libraries + head: makeHeadData(URLModel), + body: makeBodyData(libraries, URLModel), + meta: { urlmodel: URLModel, ...libraries } }; }; - const defaultLibrariesTable = librariesTable( - libraries.filter((lib) => !lib.id && lib.packager === 'intuitem') - ); - - const importedLibrariesTable = librariesTable(libraries.filter((lib) => lib.id)); + const storedLibrariesTable = makeLibrariesTable(storedLibraries, 'stored-libraries'); + const loadedLibrariesTable = makeLibrariesTable(loadedLibraries, 'loaded-libraries'); const schema = z.object({ id: z.string() }); const deleteForm = await superValidate(zod(schema)); - return { libraries, defaultLibrariesTable, importedLibrariesTable, deleteForm }; + return { storedLibrariesTable, loadedLibrariesTable, deleteForm }; }) satisfies PageServerLoad; - // ----------------------------------------------------------- // - export const actions: Actions = { upload: async (event) => { const formData = await event.request.formData(); @@ -179,7 +109,7 @@ export const actions: Actions = { const schema = z.object({ id: z.string().regex(URN_REGEX) }); const deleteForm = await superValidate(formData, zod(schema)); - const URLModel = formData.get("urlmodel"); + const URLModel = formData.get('urlmodel'); const id = deleteForm.data.id; const endpoint = `${BASE_API_URL}/${URLModel}/${id}/`; diff --git a/frontend/src/routes/(app)/libraries/+page.svelte b/frontend/src/routes/(app)/libraries/+page.svelte index d832041e4..db2e860ec 100644 --- a/frontend/src/routes/(app)/libraries/+page.svelte +++ b/frontend/src/routes/(app)/libraries/+page.svelte @@ -16,7 +16,8 @@
- + + {#if data.loadedLibrariesTable.body.length > 0} {m.librariesStore()} {m.importedLibraries()} @@ -26,26 +27,25 @@ {m.currentlyNoImportedLibraries()}.
{/if} - + + {#if tabSet === 0} {/if} - {#if tabSet === 1} + {#if tabSet === 1} + {/if} From aaac73b2e0482a2a900aabded6784d2af04eb2b0 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 14:30:54 +0200 Subject: [PATCH 13/86] Move stored library initialization in app.startup --- backend/core/apps.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/core/apps.py b/backend/core/apps.py index 743ef689d..380e192e0 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -1,8 +1,10 @@ +import sys from django.apps import AppConfig from django.db.models.signals import post_migrate -from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL +from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL, LIBRARIES_PATH import os + READER_PERMISSIONS_LIST = [ "view_project", "view_riskassessment", @@ -360,6 +362,20 @@ class CoreConfig(AppConfig): verbose_name = "Core" def ready(self): + from .models import StoredLibrary + # avoid post_migrate handler if we are in the main, as it interferes with restore + print("startup handler: import libraries") + print(LIBRARIES_PATH) + for fname in os.listdir(LIBRARIES_PATH): + print(f"Importing {fname}") + fname = str(LIBRARIES_PATH / fname) + if fname.endswith(".yaml"): + error = StoredLibrary.store_library_file(fname) + if error is not None: + print( + f"[ERROR] Can't import libary file '{fname}' : {error}", + file=sys.stderr, + ) if not os.environ.get("RUN_MAIN"): post_migrate.connect(startup, sender=self) From af4f9b39a90adc2626962211b7683556404e5443 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 14:31:15 +0200 Subject: [PATCH 14/86] chore: Run formatter --- frontend/src/lib/utils/table.ts | 4 +- .../(app)/libraries/[id=urn]/+server.ts | 4 +- .../(app)/libraries/[id=urn]/tree/+server.ts | 2 +- frontend/tests/utils/test-data.ts | 288 +++++++++--------- 4 files changed, 149 insertions(+), 149 deletions(-) diff --git a/frontend/src/lib/utils/table.ts b/frontend/src/lib/utils/table.ts index 439412bff..0428be0b4 100644 --- a/frontend/src/lib/utils/table.ts +++ b/frontend/src/lib/utils/table.ts @@ -93,11 +93,11 @@ export const listViewFields = { body: ['ref_id', 'name', 'description', 'framework'], meta: ['id', 'urn'] }, - "stored-libraries": { + 'stored-libraries': { head: ['ref', 'name', 'description', 'language', 'overview'], body: ['ref_id', 'name', 'description', 'locale', 'overview'] }, - "loaded-libraries": { + 'loaded-libraries': { head: ['ref', 'name', 'description', 'language', 'overview'], body: ['ref_id', 'name', 'description', 'locale', 'overview'] } diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index e77b95394..d07c79e6f 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -4,8 +4,8 @@ import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, url, params }) => { - const isLoaded = url.searchParams.has("loaded"); - const URLModel = isLoaded ? "loaded-libraries" : "stored-libraries"; + const isLoaded = url.searchParams.has('loaded'); + const URLModel = isLoaded ? 'loaded-libraries' : 'stored-libraries'; const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/`; const res = await fetch(endpoint); diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index e63dc91b4..3ccf59a81 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -4,7 +4,7 @@ import { error, type NumericRange } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, params, url }) => { - const URLModel = url.searchParams.has("loaded") ? "loaded-libraries" : "stored-libraries"; + const URLModel = url.searchParams.has('loaded') ? 'loaded-libraries' : 'stored-libraries'; const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/tree`; const res = await fetch(endpoint); diff --git a/frontend/tests/utils/test-data.ts b/frontend/tests/utils/test-data.ts index 123944438..9890b9262 100644 --- a/frontend/tests/utils/test-data.ts +++ b/frontend/tests/utils/test-data.ts @@ -24,165 +24,165 @@ export default { analyst: { name: 'Analyst', perms: [ - "add_project", - "view_project", - "change_project", - "delete_project", - "add_riskassessment", - "view_riskassessment", - "change_riskassessment", - "delete_riskassessment", - "add_appliedcontrol", - "view_appliedcontrol", - "change_appliedcontrol", - "delete_appliedcontrol", - "add_policy", - "view_policy", - "change_policy", - "delete_policy", - "add_riskscenario", - "view_riskscenario", - "change_riskscenario", - "delete_riskscenario", - "add_riskacceptance", - "view_riskacceptance", - "change_riskacceptance", - "delete_riskacceptance", - "add_complianceassessment", - "view_complianceassessment", - "change_complianceassessment", - "delete_complianceassessment", - "view_requirementassessment", - "change_requirementassessment", - "add_evidence", - "view_evidence", - "change_evidence", - "delete_evidence", - "add_asset", - "view_asset", - "change_asset", - "delete_asset", - "add_threat", - "view_threat", - "change_threat", - "delete_threat", - "view_referencecontrol", - "view_folder", - "view_usergroup", - "view_riskmatrix", - "view_requirementnode", - "view_framework", - "view_loadedlibrary", - "view_user", + 'add_project', + 'view_project', + 'change_project', + 'delete_project', + 'add_riskassessment', + 'view_riskassessment', + 'change_riskassessment', + 'delete_riskassessment', + 'add_appliedcontrol', + 'view_appliedcontrol', + 'change_appliedcontrol', + 'delete_appliedcontrol', + 'add_policy', + 'view_policy', + 'change_policy', + 'delete_policy', + 'add_riskscenario', + 'view_riskscenario', + 'change_riskscenario', + 'delete_riskscenario', + 'add_riskacceptance', + 'view_riskacceptance', + 'change_riskacceptance', + 'delete_riskacceptance', + 'add_complianceassessment', + 'view_complianceassessment', + 'change_complianceassessment', + 'delete_complianceassessment', + 'view_requirementassessment', + 'change_requirementassessment', + 'add_evidence', + 'view_evidence', + 'change_evidence', + 'delete_evidence', + 'add_asset', + 'view_asset', + 'change_asset', + 'delete_asset', + 'add_threat', + 'view_threat', + 'change_threat', + 'delete_threat', + 'view_referencecontrol', + 'view_folder', + 'view_usergroup', + 'view_riskmatrix', + 'view_requirementnode', + 'view_framework', + 'view_loadedlibrary', + 'view_user' ] }, reader: { name: 'Reader', perms: [ - "view_project", - "view_riskassessment", - "view_appliedcontrol", - "view_policy", - "view_riskscenario", - "view_riskacceptance", - "view_asset", - "view_threat", - "view_referencecontrol", - "view_folder", - "view_usergroup", - "view_riskmatrix", - "view_complianceassessment", - "view_requirementassessment", - "view_requirementnode", - "view_evidence", - "view_framework", - "view_loadedlibrary", - "view_user", + 'view_project', + 'view_riskassessment', + 'view_appliedcontrol', + 'view_policy', + 'view_riskscenario', + 'view_riskacceptance', + 'view_asset', + 'view_threat', + 'view_referencecontrol', + 'view_folder', + 'view_usergroup', + 'view_riskmatrix', + 'view_complianceassessment', + 'view_requirementassessment', + 'view_requirementnode', + 'view_evidence', + 'view_framework', + 'view_loadedlibrary', + 'view_user' ] }, domainManager: { name: 'Domain manager', perms: [ - "change_usergroup", - "view_usergroup", - "add_project", - "change_project", - "delete_project", - "view_project", - "add_riskassessment", - "view_riskassessment", - "change_riskassessment", - "delete_riskassessment", - "add_appliedcontrol", - "view_appliedcontrol", - "change_appliedcontrol", - "delete_appliedcontrol", - "add_policy", - "view_policy", - "change_policy", - "delete_policy", - "add_riskscenario", - "view_riskscenario", - "change_riskscenario", - "delete_riskscenario", - "add_riskacceptance", - "view_riskacceptance", - "change_riskacceptance", - "delete_riskacceptance", - "add_asset", - "view_asset", - "change_asset", - "delete_asset", - "add_threat", - "view_threat", - "change_threat", - "delete_threat", - "view_referencecontrol", - "view_folder", - "change_folder", - "add_riskmatrix", - "view_riskmatrix", - "change_riskmatrix", - "delete_riskmatrix", - "add_complianceassessment", - "view_complianceassessment", - "change_complianceassessment", - "delete_complianceassessment", - "view_requirementassessment", - "change_requirementassessment", - "add_evidence", - "view_evidence", - "change_evidence", - "delete_evidence", - "view_requirementnode", - "view_framework", - "view_loadedlibrary", - "view_user", + 'change_usergroup', + 'view_usergroup', + 'add_project', + 'change_project', + 'delete_project', + 'view_project', + 'add_riskassessment', + 'view_riskassessment', + 'change_riskassessment', + 'delete_riskassessment', + 'add_appliedcontrol', + 'view_appliedcontrol', + 'change_appliedcontrol', + 'delete_appliedcontrol', + 'add_policy', + 'view_policy', + 'change_policy', + 'delete_policy', + 'add_riskscenario', + 'view_riskscenario', + 'change_riskscenario', + 'delete_riskscenario', + 'add_riskacceptance', + 'view_riskacceptance', + 'change_riskacceptance', + 'delete_riskacceptance', + 'add_asset', + 'view_asset', + 'change_asset', + 'delete_asset', + 'add_threat', + 'view_threat', + 'change_threat', + 'delete_threat', + 'view_referencecontrol', + 'view_folder', + 'change_folder', + 'add_riskmatrix', + 'view_riskmatrix', + 'change_riskmatrix', + 'delete_riskmatrix', + 'add_complianceassessment', + 'view_complianceassessment', + 'change_complianceassessment', + 'delete_complianceassessment', + 'view_requirementassessment', + 'change_requirementassessment', + 'add_evidence', + 'view_evidence', + 'change_evidence', + 'delete_evidence', + 'view_requirementnode', + 'view_framework', + 'view_loadedlibrary', + 'view_user' ] }, approver: { name: 'Approver', perms: [ - "view_project", - "view_riskassessment", - "view_appliedcontrol", - "view_policy", - "view_riskscenario", - "view_riskacceptance", - "approve_riskacceptance", - "view_asset", - "view_threat", - "view_referencecontrol", - "view_folder", - "view_usergroup", - "view_riskmatrix", - "view_complianceassessment", - "view_requirementassessment", - "view_requirementnode", - "view_evidence", - "view_framework", - "view_loadedlibrary", - "view_user", + 'view_project', + 'view_riskassessment', + 'view_appliedcontrol', + 'view_policy', + 'view_riskscenario', + 'view_riskacceptance', + 'approve_riskacceptance', + 'view_asset', + 'view_threat', + 'view_referencecontrol', + 'view_folder', + 'view_usergroup', + 'view_riskmatrix', + 'view_complianceassessment', + 'view_requirementassessment', + 'view_requirementnode', + 'view_evidence', + 'view_framework', + 'view_loadedlibrary', + 'view_user' ] } }, From 0c8d7393fc552d85c60d0a98ed5fbfd3a88f09d7 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Mon, 29 Apr 2024 20:11:40 +0200 Subject: [PATCH 15/86] WIP: API tests --- backend/app_tests/api/test_utils.py | 2 +- backend/app_tests/conftest.py | 2 +- backend/app_tests/test_vars.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app_tests/api/test_utils.py b/backend/app_tests/api/test_utils.py index 5a6c6e5f6..701d409d2 100644 --- a/backend/app_tests/api/test_utils.py +++ b/backend/app_tests/api/test_utils.py @@ -25,7 +25,7 @@ def get_object_urn(object_name: str, resolved: bool = True): urn_varname = format_urn(object_name) urn = get_var(urn_varname) - return f"{reverse(LIBRARIES_ENDPOINT)}{urn}/" if resolved else eval(urn) + return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/" if resolved else eval(urn) @pytest.mark.django_db def get_test_client_and_folder( diff --git a/backend/app_tests/conftest.py b/backend/app_tests/conftest.py index d54672292..8cf15f291 100644 --- a/backend/app_tests/conftest.py +++ b/backend/app_tests/conftest.py @@ -15,7 +15,7 @@ def __init__(self, *args, **kwargs): @pytest.fixture def app_config(): - startup() + startup(sender=None, **{}) @pytest.fixture diff --git a/backend/app_tests/test_vars.py b/backend/app_tests/test_vars.py index fe9384a92..4cc64d9ca 100644 --- a/backend/app_tests/test_vars.py +++ b/backend/app_tests/test_vars.py @@ -13,7 +13,7 @@ EVIDENCES_ENDPOINT = "evidences-list" FOLDERS_ENDPOINT = "folders-list" FRAMEWORKS_ENDPOINT = "frameworks-list" -LIBRARIES_ENDPOINT = "libraries-list" +STORED_LIBRARIES_ENDPOINT = "stored-libraries-list" RISK_MATRICES_ENDPOINT = "risk-matrices-list" PROJECTS_ENDPOINT = "projects-list" REQUIREMENT_ASSESSMENTS_ENDPOINT = "requirement-assessments-list" From bc0a49d11fd695826e63ed6872bafae8d1c3465c Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Tue, 30 Apr 2024 17:16:14 +0200 Subject: [PATCH 16/86] Initialize stored libraries in a management command It is discouraged to access the database during app initialization as the queries will run during the startup of every management command, and can fail if migrations are pending. https://docs.djangoproject.com/en/5.0/ref/applications/#django.apps.AppConfig.ready --- backend/core/apps.py | 15 +------ backend/library/management/__init__.py | 0 .../library/management/commands/__init__.py | 0 .../management/commands/storelibraries.py | 39 +++++++++++++++++++ backend/library/views.py | 26 +++---------- 5 files changed, 46 insertions(+), 34 deletions(-) create mode 100644 backend/library/management/__init__.py create mode 100644 backend/library/management/commands/__init__.py create mode 100644 backend/library/management/commands/storelibraries.py diff --git a/backend/core/apps.py b/backend/core/apps.py index 380e192e0..685e8da2a 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -1,5 +1,6 @@ import sys from django.apps import AppConfig +from django.db import connection from django.db.models.signals import post_migrate from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL, LIBRARIES_PATH import os @@ -362,20 +363,6 @@ class CoreConfig(AppConfig): verbose_name = "Core" def ready(self): - from .models import StoredLibrary - # avoid post_migrate handler if we are in the main, as it interferes with restore - print("startup handler: import libraries") - print(LIBRARIES_PATH) - for fname in os.listdir(LIBRARIES_PATH): - print(f"Importing {fname}") - fname = str(LIBRARIES_PATH / fname) - if fname.endswith(".yaml"): - error = StoredLibrary.store_library_file(fname) - if error is not None: - print( - f"[ERROR] Can't import libary file '{fname}' : {error}", - file=sys.stderr, - ) if not os.environ.get("RUN_MAIN"): post_migrate.connect(startup, sender=self) diff --git a/backend/library/management/__init__.py b/backend/library/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/library/management/commands/__init__.py b/backend/library/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/library/management/commands/storelibraries.py b/backend/library/management/commands/storelibraries.py new file mode 100644 index 000000000..b1c131379 --- /dev/null +++ b/backend/library/management/commands/storelibraries.py @@ -0,0 +1,39 @@ +import os +import sys +from pathlib import Path +from ciso_assistant.settings import LIBRARIES_PATH +from typing import Any +from django.core.management.base import BaseCommand +from core.models import StoredLibrary + +import structlog + +logger = structlog.getLogger(__name__) + + +class Command(BaseCommand): + help = "Store libraries in the database" + + def add_arguments(self, parser: Any) -> None: + parser.add_argument("--path", type=str, help="Path to library files") + + def handle(self, *args: Any, **options: Any) -> str | None: + StoredLibrary.__init_class__() + logger.info("Storing libraries") + path = Path(options.get("path") or LIBRARIES_PATH) + if path.is_dir(): + library_files = [ + f for f in path.iterdir() if f.is_file and f.suffix == ".yaml" + ] + else: + library_files = [path] + for fname in library_files: + logger.info("Begin library file storage", filename=fname) + error = StoredLibrary.store_library_file(fname) + if error is not None: + logger.error( + "Can't import libary file", + filename=fname, + error=error, + ) + logger.info("End library file storage", filename=fname) diff --git a/backend/library/views.py b/backend/library/views.py index 48068c40d..3b8fa2997 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -1,8 +1,6 @@ import json from django.db import IntegrityError -from django.db.models import QuerySet -from rest_framework import viewsets, permissions, status -from rest_framework.generics import get_object_or_404 +from rest_framework import viewsets, status from rest_framework.status import ( HTTP_200_OK, HTTP_400_BAD_REQUEST, @@ -11,14 +9,10 @@ ) from rest_framework.parsers import FileUploadParser -from django.contrib import messages -from django.utils.translation import gettext_lazy as _ from django.http import HttpResponse -import yaml from core.helpers import get_sorted_requirement_nodes from core.models import StoredLibrary, LoadedLibrary -from core.views import BaseModelViewSet from iam.models import RoleAssignment, Folder, Permission from library.validators import validate_file_extension from .helpers import preview_library @@ -28,15 +22,7 @@ from rest_framework.response import Response from .serializers import ( StoredLibraryDetailedSerializer, - LoadedLibrarySerializer, LoadedLibraryDetailedSerializer, - LibraryUploadSerializer, -) -from .utils import ( - LibraryImporter, - get_available_libraries, - get_library, - import_library_view, ) @@ -50,7 +36,7 @@ class StoredLibraryViewSet(viewsets.ModelViewSet): queryset = StoredLibrary.objects.filter(is_obsolete=False) def list(self, request, *args, **kwargs): - if not "view_storedlibrary" in request.user.permissions: + if "view_storedlibrary" not in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) available_libraries = self.queryset.values( "id", @@ -90,7 +76,7 @@ def destroy( return Response(status=status.HTTP_204_NO_CONTENT) def retrieve(self, request, *args, pk, **kwargs): - if not "view_storedlibrary" in request.user.permissions: + if "view_storedlibrary" not in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) try: lib = self.queryset.get(urn=pk) @@ -121,7 +107,7 @@ def import_library(self, request, pk): status=status.HTTP_400_BAD_REQUEST, ) # This can cause translation issues return Response({"status": "success"}) - except Exception as e: + except Exception: """print(f"ERROR {type(e)}") print(str(e)) raise e""" @@ -195,7 +181,7 @@ class LoadedLibraryViewSet(viewsets.ModelViewSet): queryset = LoadedLibrary.objects.all() def list(self, request, *args, **kwargs): - if not "view_storedlibrary" in request.user.permissions: + if "view_storedlibrary" not in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) loaded_libraries = [ @@ -221,7 +207,7 @@ def list(self, request, *args, **kwargs): return Response({"results": loaded_libraries}) def retrieve(self, request, *args, pk, **kwargs): - if not "view_loadedlibrary" in request.user.permissions: + if "view_loadedlibrary" not in request.user.permissions: return Response(status=status.HTTP_403_FORBIDDEN) try: lib = LoadedLibrary.objects.get( From 50cb4f2d29cd72bd34e85b0b50a3068fbd08fef0 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Tue, 30 Apr 2024 23:17:50 +0200 Subject: [PATCH 17/86] Refactor StoredLibrary model and LibraryMixin Add unique_together constraint on urn, locale, version in LibraryMixin Use proper type hints for Path objects Return a StoredLibrary | None on library file storage methods --- backend/core/models.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 344fa6b2e..b12cad070 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -1,3 +1,4 @@ +from pathlib import Path from django.apps import apps from django.forms.models import model_to_dict from django.contrib.auth import get_user_model @@ -12,13 +13,12 @@ from django.core import serializers import os -import re import json import yaml from django.urls import reverse from datetime import date, datetime -from typing import Union, List, Self +from typing import Union, Self from django.utils.html import format_html from structlog import get_logger @@ -88,7 +88,9 @@ def __str__(self) -> str: class LibraryMixin(ReferentialObjectMixin): class Meta: abstract = True + unique_together = [["urn", "locale", "version"]] + urn = models.CharField(max_length=100, null=True, blank=True, verbose_name=_("URN")) copyright = models.CharField( max_length=4096, null=True, blank=True, verbose_name=_("Copyright") ) @@ -123,40 +125,42 @@ def __init_class__(cls): value["hash_checksum"] for value in cls.objects.values("hash_checksum") ) - @staticmethod - def store_libary_content(library_content: bytes) -> Union[str, None]: + @classmethod + def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None": hash_checksum = sha256(library_content) if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET: - return None # We do not store the libary if its hash checksum is in the database. - - # urn(str:max_length=100), version(int), name(str:max_length=200) - + return None # We do not store the library if its hash checksum is in the database. try: library_data = yaml.safe_load(library_content) - except Exception: - return "Invalid formatted file, the library file must be formatted in YAML." + except yaml.YAMLError as e: + logger.error("Error while loading library content", error=e) + raise e missing_fields = StoredLibrary.REQUIRED_FIELDS - set(library_data.keys()) if missing_fields: - return "The following fields are missing : {}".format( + err = "The following fields are missing : {}".format( ", ".join(repr(field) for field in missing_fields) ) + logger.error("Error while loading library content", error=err) + raise ValueError(err) urn = library_data["urn"] locale = library_data.get("locale", "en") version = int(library_data["version"]) library_matches = [*StoredLibrary.objects.filter(urn=urn, locale=locale)] - if any(libary.version >= version for libary in library_matches): + if any(library.version >= version for library in library_matches): # The library isn't stored if it's obsolete due to be a too old version of itself. - return "A library with the urn '{}', a locale '{}' with a superior superior or equal to {} is already stored in the database.".format( + err = "A library with the urn '{}', a locale '{}' with a superior superior or equal to {} is already stored in the database.".format( urn, locale, version ) + logger.error("Error while loading library content", error=err) + raise ValueError(err) for library in library_matches: library.is_obsolete = True - library.save() # If a user delete a library from the libary store we must set the is_obsolete value of its most recent obsolete version to False. + library.save() # If a user delete a library from the library store we must set the is_obsolete value of its most recent obsolete version to False. objects_meta = { key: len(value) for key, value in library_data["objects"].items() @@ -167,7 +171,7 @@ def store_libary_content(library_content: bytes) -> Union[str, None]: ) # I don't want whitespaces in URN anymore nontheless library_objects = json.dumps(library_data["objects"]) - StoredLibrary.objects.create( + return StoredLibrary.objects.create( name=library_data["name"], is_published=True, urn=urn, @@ -189,11 +193,11 @@ def store_libary_content(library_content: bytes) -> Union[str, None]: content=library_objects, ) - @staticmethod - def store_library_file(fname: str) -> Union[str, None]: + @classmethod + def store_library_file(cls, fname: Path) -> "StoredLibrary | None": with open(fname, "rb") as f: library_content = f.read() - return StoredLibrary.store_libary_content(library_content) + return StoredLibrary.store_library_content(library_content) def loads(self) -> Union[str, None]: from library.utils import LibraryImporter From 09eb41e019297a6f4911b23560f168a55ad84dcc Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Tue, 30 Apr 2024 23:18:51 +0200 Subject: [PATCH 18/86] Fix typo: libary -> library --- backend/library/utils.py | 2 +- backend/library/views.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/library/utils.py b/backend/library/utils.py index 5349a2f24..09f9f6261 100644 --- a/backend/library/utils.py +++ b/backend/library/utils.py @@ -533,7 +533,7 @@ def init(self) -> Union[str, None]: if not any( object_field in library_objects for object_field in self.OBJECT_FIELDS ): - return "The libary 'objects' field data must contain at least one of the following fields : {}".format( + return "The library 'objects' field data must contain at least one of the following fields : {}".format( ", ".join(self.OBJECT_FIELDS) ) diff --git a/backend/library/views.py b/backend/library/views.py index 3b8fa2997..bc093e0d2 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -57,7 +57,7 @@ def list(self, request, *args, **kwargs): def destroy( self, request, pk - ): # We may have to also get the locale of the library we want to delete in the future for this method and all other libary viewset methods which goal is to apply an operation on a specific library + ): # We may have to also get the locale of the library we want to delete in the future for this method and all other library viewset methods which goal is to apply an operation on a specific library if not RoleAssignment.is_access_allowed( user=request.user, perm=Permission.objects.get(codename="delete_storedlibrary"), @@ -148,13 +148,13 @@ def upload_library(self, request): validate_file_extension(attachment) # Use safe_load to prevent arbitrary code execution. - content = attachment.read() # Should we read it chunck by chunck or ensure that the file size of the libary content is reasonnable before reading ? + content = attachment.read() # Should we read it chunck by chunck or ensure that the file size of the library content is reasonnable before reading ? - error_msg = StoredLibrary.store_libary_content(content) - - if error_msg is not None: + try: + StoredLibrary.store_library_content(content) + except ValueError as e: return HttpResponse( - json.dumps({"error": error_msg}), + json.dumps({"error": e}), status=HTTP_422_UNPROCESSABLE_ENTITY, ) From 45b220583b8c1a89bdf2bd2e25eb4a2c906bd002 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:01:23 +0200 Subject: [PATCH 19/86] Subclass BaseModelViewset for StoredLibraryViewSet --- backend/library/serializers.py | 13 ++--- backend/library/views.py | 89 +++++++++++++++------------------- 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/backend/library/serializers.py b/backend/library/serializers.py index ff129a42a..4c05c0667 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -25,20 +25,17 @@ class Meta: "id", "name", "description", + "urn", + "ref_id", "locale", "version", + "packager", + "provider", "builtin", "objects_meta", + "is_imported", ] - # name = serializers.CharField() - # description = serializers.CharField() - # locale = serializers.ChoiceField(choices=["en", "fr"]) - # version = serializers.CharField() - # copyright = serializers.CharField() - # builtin = serializers.BooleanField() - # objects_meta = serializers.JSONField() - class StoredLibraryDetailedSerializer(serializers.ModelSerializer): class Meta: diff --git a/backend/library/views.py b/backend/library/views.py index bc093e0d2..d3a309e2e 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -13,6 +13,7 @@ from core.helpers import get_sorted_requirement_nodes from core.models import StoredLibrary, LoadedLibrary +from core.views import BaseModelViewSet from iam.models import RoleAssignment, Folder, Permission from library.validators import validate_file_extension from .helpers import preview_library @@ -23,11 +24,11 @@ from .serializers import ( StoredLibraryDetailedSerializer, LoadedLibraryDetailedSerializer, + StoredLibrarySerializer, ) -class StoredLibraryViewSet(viewsets.ModelViewSet): - # serializer_class = StoredLibrarySerializer +class StoredLibraryViewSet(BaseModelViewSet): parser_classes = [FileUploadParser] # solve issue with URN containing dot, see https://stackoverflow.com/questions/27963899/django-rest-framework-using-dot-in-url @@ -35,54 +36,42 @@ class StoredLibraryViewSet(viewsets.ModelViewSet): model = StoredLibrary queryset = StoredLibrary.objects.filter(is_obsolete=False) - def list(self, request, *args, **kwargs): - if "view_storedlibrary" not in request.user.permissions: - return Response(status=status.HTTP_403_FORBIDDEN) - available_libraries = self.queryset.values( - "id", - "name", - "description", - "urn", - "ref_id", - "locale", - "version", - "packager", - "provider", - "builtin", - "objects_meta", - "is_imported", - ) # The frontend doesn't receive the obsolete libraries for now. - - return Response({"results": available_libraries}) - - def destroy( - self, request, pk - ): # We may have to also get the locale of the library we want to delete in the future for this method and all other library viewset methods which goal is to apply an operation on a specific library - if not RoleAssignment.is_access_allowed( - user=request.user, - perm=Permission.objects.get(codename="delete_storedlibrary"), - folder=Folder.get_root_folder(), - ): - return Response(status=status.HTTP_403_FORBIDDEN) - - try: - lib = self.queryset.get( - urn=pk - ) # the libraries with is_obsolete=True are not displayed in the frontend and therefore not meant to be destroyable (at least yet) - except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - - lib.delete() - return Response(status=status.HTTP_204_NO_CONTENT) - - def retrieve(self, request, *args, pk, **kwargs): - if "view_storedlibrary" not in request.user.permissions: - return Response(status=status.HTTP_403_FORBIDDEN) - try: - lib = self.queryset.get(urn=pk) - except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - return Response(StoredLibraryDetailedSerializer(lib).data) + filterset_fields = ["urn", "locale", "version", "packager", "provider"] + search_fields = ["name", "description", "urn"] + + def get_serializer_class(self): + if self.action == "list": + return StoredLibrarySerializer + return StoredLibraryDetailedSerializer + + # def destroy( + # self, request, pk + # ): # We may have to also get the locale of the library we want to delete in the future for this method and all other library viewset methods which goal is to apply an operation on a specific library + # if not RoleAssignment.is_access_allowed( + # user=request.user, + # perm=Permission.objects.get(codename="delete_storedlibrary"), + # folder=Folder.get_root_folder(), + # ): + # return Response(status=status.HTTP_403_FORBIDDEN) + # + # try: + # lib = self.queryset.get( + # urn=pk + # ) # the libraries with is_obsolete=True are not displayed in the frontend and therefore not meant to be destroyable (at least yet) + # except: + # return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + # + # lib.delete() + # return Response(status=status.HTTP_204_NO_CONTENT) + + # def retrieve(self, request, *args, pk, **kwargs): + # if "view_storedlibrary" not in request.user.permissions: + # return Response(status=status.HTTP_403_FORBIDDEN) + # try: + # lib = self.queryset.get(urn=pk) + # except: + # return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + # return Response(StoredLibraryDetailedSerializer(lib).data) @action(detail=True, methods=["get"], url_path="import") def import_library(self, request, pk): From 933b4470afcbb7c92872018841d92d008e1ef723 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:01:51 +0200 Subject: [PATCH 20/86] Remove dead code --- backend/library/views.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/backend/library/views.py b/backend/library/views.py index d3a309e2e..933ba9aeb 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -44,35 +44,6 @@ def get_serializer_class(self): return StoredLibrarySerializer return StoredLibraryDetailedSerializer - # def destroy( - # self, request, pk - # ): # We may have to also get the locale of the library we want to delete in the future for this method and all other library viewset methods which goal is to apply an operation on a specific library - # if not RoleAssignment.is_access_allowed( - # user=request.user, - # perm=Permission.objects.get(codename="delete_storedlibrary"), - # folder=Folder.get_root_folder(), - # ): - # return Response(status=status.HTTP_403_FORBIDDEN) - # - # try: - # lib = self.queryset.get( - # urn=pk - # ) # the libraries with is_obsolete=True are not displayed in the frontend and therefore not meant to be destroyable (at least yet) - # except: - # return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - # - # lib.delete() - # return Response(status=status.HTTP_204_NO_CONTENT) - - # def retrieve(self, request, *args, pk, **kwargs): - # if "view_storedlibrary" not in request.user.permissions: - # return Response(status=status.HTTP_403_FORBIDDEN) - # try: - # lib = self.queryset.get(urn=pk) - # except: - # return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) - # return Response(StoredLibraryDetailedSerializer(lib).data) - @action(detail=True, methods=["get"], url_path="import") def import_library(self, request, pk): if not RoleAssignment.is_access_allowed( From 9c036ff90a9f99c781ffec5c8dd261bd6ea3549b Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:33:46 +0200 Subject: [PATCH 21/86] Refactor library content and tree display --- backend/library/views.py | 7 ++++++- .../(app)/libraries/[id=urn]/+server.ts | 20 ++++++++++++++++--- .../(app)/libraries/[id=urn]/tree/+server.ts | 18 ++++++++++++++--- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/backend/library/views.py b/backend/library/views.py index 933ba9aeb..40f0b00c5 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -44,6 +44,11 @@ def get_serializer_class(self): return StoredLibrarySerializer return StoredLibraryDetailedSerializer + @action(detail=True, methods=["get"]) + def content(self, request, pk): + lib = StoredLibrary.objects.get(id=pk) + return Response(lib.content) + @action(detail=True, methods=["get"], url_path="import") def import_library(self, request, pk): if not RoleAssignment.is_access_allowed( @@ -81,7 +86,7 @@ def import_library(self, request, pk): @action(detail=True, methods=["get"]) def tree(self, request, pk): try: - lib = StoredLibrary.objects.get(urn=pk) + lib = StoredLibrary.objects.get(id=pk) except: return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index d07c79e6f..518510005 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -6,17 +6,31 @@ import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, url, params }) => { const isLoaded = url.searchParams.has('loaded'); const URLModel = isLoaded ? 'loaded-libraries' : 'stored-libraries'; - const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/`; + const endpoint = `${BASE_API_URL}/${URLModel}/?urn=${params.id}`; const res = await fetch(endpoint); if (!res.ok) { error(res.status as NumericRange<400, 599>, await res.json()); } - const data = await res.json(); + const data = await res + .json() + .then((res) => res.results) + .then((res) => res.reduce((acc, curr) => (acc.version > curr.version ? acc : curr))); // Get the latest version of the library + + const uuid = data.id; + const contentEndpoint = `${BASE_API_URL}/${URLModel}/${uuid}/content`; + const contentRes = await fetch(contentEndpoint); + if (!contentRes.ok) { + error(contentRes.status as NumericRange<400, 599>, await contentRes.json()); + } + const content = await contentRes.json(); + data.objects = content; if (!isLoaded) { - data.objects = JSON.parse(data.content); + data.objects = JSON.parse(content); } + console.log(data); + return new Response(JSON.stringify(data), { headers: { 'Content-Type': 'application/json' diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index 3ccf59a81..5d2119ae1 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -5,15 +5,27 @@ import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, params, url }) => { const URLModel = url.searchParams.has('loaded') ? 'loaded-libraries' : 'stored-libraries'; - const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/tree`; + const endpoint = `${BASE_API_URL}/${URLModel}/?urn=${params.id}`; const res = await fetch(endpoint); if (!res.ok) { error(res.status as NumericRange<400, 599>, await res.json()); } - const data = await res.json(); + const data = await res + .json() + .then((res) => res.results) + .then((res) => res.reduce((acc, curr) => (acc.version > curr.version ? acc : curr))); // Get the latest version of the library - return new Response(JSON.stringify(data), { + const uuid = data.id; + const treeEndpoint = `${BASE_API_URL}/${URLModel}/${uuid}/tree`; + const treeRes = await fetch(treeEndpoint); + if (!treeRes.ok) { + error(treeRes.status as NumericRange<400, 599>, await treeRes.json()); + } + + const tree = await treeRes.json(); + + return new Response(JSON.stringify(tree), { headers: { 'Content-Type': 'application/json' } From 91867950f31ed0c36efff6f24ff2ecf0f0f13ee1 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:45:31 +0200 Subject: [PATCH 22/86] Add useFocusTrap prop --- frontend/src/lib/components/Forms/Form.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/Forms/Form.svelte b/frontend/src/lib/components/Forms/Form.svelte index 23c5fdad0..a255c0ff0 100644 --- a/frontend/src/lib/components/Forms/Form.svelte +++ b/frontend/src/lib/components/Forms/Form.svelte @@ -21,6 +21,8 @@ export let onSubmit = (submit_data: any) => {}; export let taintedMessage: string | null = m.taintedFormMessage(); + export let useFocusTrap = true; + export let debug = false; // set to true to enable SuperDebug component function handleFormUpdated({ form, closeModal }: { form: any; closeModal: boolean }) { @@ -48,7 +50,7 @@ {/if} -
+ {#if $errors._errors} {#each $errors._errors as error}

{error}

From 082011598829f5aebe7a35ba6e3bcb3e59602a0b Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:46:33 +0200 Subject: [PATCH 23/86] Stop using focus traps for library upload form This prevents the annoying automatic scroll to the file input at the bottom of the libraries store page --- frontend/src/routes/(app)/libraries/+page.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/routes/(app)/libraries/+page.svelte b/frontend/src/routes/(app)/libraries/+page.svelte index db2e860ec..6ffc1a2de 100644 --- a/frontend/src/routes/(app)/libraries/+page.svelte +++ b/frontend/src/routes/(app)/libraries/+page.svelte @@ -64,6 +64,7 @@ let:form validators={zod(LibraryUploadSchema)} action="?/upload" + useFocusTrap={false} onSubmit={() => { const fileInput = document.querySelector(`input[type="file"]`); console.log(fileInput); From 9dca9e012e0c305c2a364b68cf0df619ec7f9b48 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:49:17 +0200 Subject: [PATCH 24/86] Delete debug logs --- frontend/src/routes/(app)/libraries/[id=urn]/+server.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index 518510005..28c9594bf 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -29,8 +29,6 @@ export const GET: RequestHandler = async ({ fetch, url, params }) => { data.objects = JSON.parse(content); } - console.log(data); - return new Response(JSON.stringify(data), { headers: { 'Content-Type': 'application/json' From 2aed24eedd4e41d9290815b05e2de2f8f0340cc6 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:49:42 +0200 Subject: [PATCH 25/86] Update storelibraries command --- .../management/commands/storelibraries.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/backend/library/management/commands/storelibraries.py b/backend/library/management/commands/storelibraries.py index b1c131379..d4f7395d5 100644 --- a/backend/library/management/commands/storelibraries.py +++ b/backend/library/management/commands/storelibraries.py @@ -1,12 +1,9 @@ -import os -import sys from pathlib import Path -from ciso_assistant.settings import LIBRARIES_PATH -from typing import Any -from django.core.management.base import BaseCommand -from core.models import StoredLibrary import structlog +from ciso_assistant.settings import LIBRARIES_PATH +from core.models import StoredLibrary +from django.core.management.base import BaseCommand logger = structlog.getLogger(__name__) @@ -14,12 +11,11 @@ class Command(BaseCommand): help = "Store libraries in the database" - def add_arguments(self, parser: Any) -> None: + def add_arguments(self, parser) -> None: parser.add_argument("--path", type=str, help="Path to library files") - def handle(self, *args: Any, **options: Any) -> str | None: + def handle(self, *args, **options): StoredLibrary.__init_class__() - logger.info("Storing libraries") path = Path(options.get("path") or LIBRARIES_PATH) if path.is_dir(): library_files = [ @@ -28,12 +24,13 @@ def handle(self, *args: Any, **options: Any) -> str | None: else: library_files = [path] for fname in library_files: - logger.info("Begin library file storage", filename=fname) - error = StoredLibrary.store_library_file(fname) - if error is not None: - logger.error( - "Can't import libary file", + # logger.info("Begin library file storage", filename=fname) + library = StoredLibrary.store_library_file(fname) + if library: + logger.info( + "Successfully stored library", filename=fname, - error=error, + library=library, ) - logger.info("End library file storage", filename=fname) + # else: + # logger.info("Library is up to date", filename=fname) From 7c1135b267f0161860f865b519c7004db8a693e3 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:50:01 +0200 Subject: [PATCH 26/86] Call storelibraries command from core.apps.startup --- backend/core/apps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/core/apps.py b/backend/core/apps.py index 685e8da2a..ed9edb367 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -4,6 +4,7 @@ from django.db.models.signals import post_migrate from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL, LIBRARIES_PATH import os +from django.core.management import call_command READER_PERMISSIONS_LIST = [ @@ -356,6 +357,8 @@ def startup(sender: AppConfig, **kwargs): except Exception as e: print(e) # NOTE: Add this exception in the logger + call_command("storelibraries") + class CoreConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" From c342c25abd653e3a344f5c50f9aff6aee8dc6a6d Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:50:15 +0200 Subject: [PATCH 27/86] chore: Run migrations --- ...2_remove_requirementnode_level_and_more.py | 118 ++++++++++++++++++ ...dated_at_alter_role_updated_at_and_more.py | 38 ++++++ 2 files changed, 156 insertions(+) create mode 100644 backend/core/migrations/0012_remove_requirementnode_level_and_more.py create mode 100644 backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py diff --git a/backend/core/migrations/0012_remove_requirementnode_level_and_more.py b/backend/core/migrations/0012_remove_requirementnode_level_and_more.py new file mode 100644 index 000000000..7eda06377 --- /dev/null +++ b/backend/core/migrations/0012_remove_requirementnode_level_and_more.py @@ -0,0 +1,118 @@ +# Generated by Django 5.0.4 on 2024-04-30 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0011_merge_20240429_1027'), + ] + + operations = [ + migrations.RemoveField( + model_name='requirementnode', + name='level', + ), + migrations.AlterField( + model_name='appliedcontrol', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='asset', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='complianceassessment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='evidence', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='framework', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='loadedlibrary', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='loadedlibrary', + name='urn', + field=models.CharField(blank=True, max_length=100, null=True, verbose_name='URN'), + ), + migrations.AlterField( + model_name='project', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='referencecontrol', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='requirementassessment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='requirementnode', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskacceptance', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskassessment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskmatrix', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskscenario', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='storedlibrary', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='storedlibrary', + name='urn', + field=models.CharField(blank=True, max_length=100, null=True, verbose_name='URN'), + ), + migrations.AlterField( + model_name='threat', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterUniqueTogether( + name='loadedlibrary', + unique_together={('urn', 'locale', 'version')}, + ), + migrations.AlterUniqueTogether( + name='storedlibrary', + unique_together={('urn', 'locale', 'version')}, + ), + migrations.DeleteModel( + name='RequirementLevel', + ), + ] diff --git a/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py b/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py new file mode 100644 index 000000000..922833008 --- /dev/null +++ b/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 5.0.4 on 2024-04-30 21:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('iam', '0002_purge_validator'), + ] + + operations = [ + migrations.AlterField( + model_name='folder', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='role', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='roleassignment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='user', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='usergroup', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + ] From a2df2b448ba0cfcd9c0cd3040ffe4862b7a4da9e Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 01:57:56 +0200 Subject: [PATCH 28/86] chore: Run migrations --- ...ibrary_alter_framework_library_and_more.py | 282 ------------------ .../0010_alter_storedlibrary_dependencies.py | 17 -- ...lter_appliedcontrol_updated_at_and_more.py | 170 +++++++++++ .../migrations/0011_merge_20240429_1027.py | 12 - ...2_remove_requirementnode_level_and_more.py | 118 -------- 5 files changed, 170 insertions(+), 429 deletions(-) delete mode 100644 backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py delete mode 100644 backend/core/migrations/0010_alter_storedlibrary_dependencies.py create mode 100644 backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py delete mode 100644 backend/core/migrations/0011_merge_20240429_1027.py delete mode 100644 backend/core/migrations/0012_remove_requirementnode_level_and_more.py diff --git a/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py b/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py deleted file mode 100644 index 8cb719dce..000000000 --- a/backend/core/migrations/0009_loadedlibrary_alter_framework_library_and_more.py +++ /dev/null @@ -1,282 +0,0 @@ -# Generated by Django 5.0.4 on 2024-04-18 17:38 - -import django.db.models.deletion -import iam.models -import uuid -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("core", "0008_alter_complianceassessment_status_and_more"), - ("iam", "0002_purge_validator"), - ] - - operations = [ - migrations.CreateModel( - name="LoadedLibrary", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ( - "created_at", - models.DateTimeField(auto_now_add=True, verbose_name="Created at"), - ), - ( - "updated_at", - models.DateTimeField(auto_now=True, verbose_name="UpdatedÒ at"), - ), - ( - "is_published", - models.BooleanField(default=False, verbose_name="published"), - ), - ( - "urn", - models.CharField( - blank=True, - max_length=100, - null=True, - unique=True, - verbose_name="URN", - ), - ), - ( - "ref_id", - models.CharField( - blank=True, - max_length=100, - null=True, - verbose_name="Reference ID", - ), - ), - ( - "locale", - models.CharField( - default="en", max_length=100, verbose_name="Locale" - ), - ), - ( - "default_locale", - models.BooleanField(default=True, verbose_name="Default locale"), - ), - ( - "provider", - models.CharField( - blank=True, max_length=200, null=True, verbose_name="Provider" - ), - ), - ( - "name", - models.CharField(max_length=200, null=True, verbose_name="Name"), - ), - ( - "description", - models.TextField(blank=True, null=True, verbose_name="Description"), - ), - ( - "annotation", - models.TextField(blank=True, null=True, verbose_name="Annotation"), - ), - ( - "copyright", - models.CharField( - blank=True, max_length=4096, null=True, verbose_name="Copyright" - ), - ), - ("version", models.IntegerField(verbose_name="Version")), - ( - "packager", - models.CharField( - blank=True, - help_text="Packager of the library", - max_length=100, - null=True, - verbose_name="Packager", - ), - ), - ("builtin", models.BooleanField(default=False)), - ("objects_meta", models.JSONField()), - ( - "dependencies", - models.ManyToManyField( - blank=True, to="core.loadedlibrary", verbose_name="Dependencies" - ), - ), - ( - "folder", - models.ForeignKey( - default=iam.models.Folder.get_root_folder, - on_delete=django.db.models.deletion.CASCADE, - related_name="%(class)s_folder", - to="iam.folder", - ), - ), - ], - options={ - "abstract": False, - }, - ), - migrations.AlterField( - model_name="framework", - name="library", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="frameworks", - to="core.loadedlibrary", - ), - ), - migrations.AlterField( - model_name="referencecontrol", - name="library", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="reference_controls", - to="core.loadedlibrary", - ), - ), - migrations.AlterField( - model_name="riskmatrix", - name="library", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="risk_matrices", - to="core.loadedlibrary", - ), - ), - migrations.AlterField( - model_name="threat", - name="library", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="threats", - to="core.loadedlibrary", - ), - ), - migrations.CreateModel( - name="StoredLibrary", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ( - "created_at", - models.DateTimeField(auto_now_add=True, verbose_name="Created at"), - ), - ( - "updated_at", - models.DateTimeField(auto_now=True, verbose_name="UpdatedÒ at"), - ), - ( - "is_published", - models.BooleanField(default=False, verbose_name="published"), - ), - ( - "urn", - models.CharField( - blank=True, - max_length=100, - null=True, - unique=True, - verbose_name="URN", - ), - ), - ( - "ref_id", - models.CharField( - blank=True, - max_length=100, - null=True, - verbose_name="Reference ID", - ), - ), - ( - "locale", - models.CharField( - default="en", max_length=100, verbose_name="Locale" - ), - ), - ( - "default_locale", - models.BooleanField(default=True, verbose_name="Default locale"), - ), - ( - "provider", - models.CharField( - blank=True, max_length=200, null=True, verbose_name="Provider" - ), - ), - ( - "name", - models.CharField(max_length=200, null=True, verbose_name="Name"), - ), - ( - "description", - models.TextField(blank=True, null=True, verbose_name="Description"), - ), - ( - "annotation", - models.TextField(blank=True, null=True, verbose_name="Annotation"), - ), - ( - "copyright", - models.CharField( - blank=True, max_length=4096, null=True, verbose_name="Copyright" - ), - ), - ("version", models.IntegerField(verbose_name="Version")), - ( - "packager", - models.CharField( - blank=True, - help_text="Packager of the library", - max_length=100, - null=True, - verbose_name="Packager", - ), - ), - ("builtin", models.BooleanField(default=False)), - ("objects_meta", models.JSONField()), - ("dependencies", models.CharField(max_length=16384, null=True)), - ("is_obsolete", models.BooleanField(default=False)), - ("is_imported", models.BooleanField(default=False)), - ("hash_checksum", models.CharField(max_length=64)), - ("content", models.TextField()), - ( - "folder", - models.ForeignKey( - default=iam.models.Folder.get_root_folder, - on_delete=django.db.models.deletion.CASCADE, - related_name="%(class)s_folder", - to="iam.folder", - ), - ), - ], - options={ - "abstract": False, - }, - ), - migrations.DeleteModel( - name="Library", - ), - ] diff --git a/backend/core/migrations/0010_alter_storedlibrary_dependencies.py b/backend/core/migrations/0010_alter_storedlibrary_dependencies.py deleted file mode 100644 index d87bebe82..000000000 --- a/backend/core/migrations/0010_alter_storedlibrary_dependencies.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 5.0.4 on 2024-04-24 14:20 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("core", "0009_loadedlibrary_alter_framework_library_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="storedlibrary", - name="dependencies", - field=models.JSONField(null=True), - ), - ] diff --git a/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py new file mode 100644 index 000000000..5c697ce4d --- /dev/null +++ b/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py @@ -0,0 +1,170 @@ +# Generated by Django 5.0.4 on 2024-04-30 23:57 + +import django.db.models.deletion +import iam.models +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0010_rename_score_definition_framework_scores_definition_and_more'), + ('iam', '0003_alter_folder_updated_at_alter_role_updated_at_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='appliedcontrol', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='asset', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='complianceassessment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='evidence', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='framework', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='project', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='referencecontrol', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='requirementassessment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='requirementnode', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskacceptance', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskassessment', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskmatrix', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='riskscenario', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.AlterField( + model_name='threat', + name='updated_at', + field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + ), + migrations.CreateModel( + name='LoadedLibrary', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='Updated at')), + ('is_published', models.BooleanField(default=False, verbose_name='published')), + ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), + ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), + ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), + ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), + ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), + ('description', models.TextField(blank=True, null=True, verbose_name='Description')), + ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), + ('urn', models.CharField(blank=True, max_length=100, null=True, verbose_name='URN')), + ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), + ('version', models.IntegerField(verbose_name='Version')), + ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), + ('builtin', models.BooleanField(default=False)), + ('objects_meta', models.JSONField()), + ('dependencies', models.ManyToManyField(blank=True, to='core.loadedlibrary', verbose_name='Dependencies')), + ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ], + options={ + 'abstract': False, + 'unique_together': {('urn', 'locale', 'version')}, + }, + ), + migrations.AlterField( + model_name='framework', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='frameworks', to='core.loadedlibrary'), + ), + migrations.AlterField( + model_name='referencecontrol', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='reference_controls', to='core.loadedlibrary'), + ), + migrations.AlterField( + model_name='riskmatrix', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='risk_matrices', to='core.loadedlibrary'), + ), + migrations.AlterField( + model_name='threat', + name='library', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='threats', to='core.loadedlibrary'), + ), + migrations.CreateModel( + name='StoredLibrary', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), + ('updated_at', models.DateTimeField(auto_now=True, verbose_name='Updated at')), + ('is_published', models.BooleanField(default=False, verbose_name='published')), + ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), + ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), + ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), + ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), + ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), + ('description', models.TextField(blank=True, null=True, verbose_name='Description')), + ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), + ('urn', models.CharField(blank=True, max_length=100, null=True, verbose_name='URN')), + ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), + ('version', models.IntegerField(verbose_name='Version')), + ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), + ('builtin', models.BooleanField(default=False)), + ('objects_meta', models.JSONField()), + ('dependencies', models.JSONField(null=True)), + ('is_obsolete', models.BooleanField(default=False)), + ('is_imported', models.BooleanField(default=False)), + ('hash_checksum', models.CharField(max_length=64)), + ('content', models.TextField()), + ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ], + options={ + 'abstract': False, + 'unique_together': {('urn', 'locale', 'version')}, + }, + ), + migrations.DeleteModel( + name='Library', + ), + ] diff --git a/backend/core/migrations/0011_merge_20240429_1027.py b/backend/core/migrations/0011_merge_20240429_1027.py deleted file mode 100644 index 7afaf3980..000000000 --- a/backend/core/migrations/0011_merge_20240429_1027.py +++ /dev/null @@ -1,12 +0,0 @@ -# Generated by Django 5.0.4 on 2024-04-29 10:27 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("core", "0009_framework_max_score_framework_min_score_and_more"), - ("core", "0010_alter_storedlibrary_dependencies"), - ] - - operations = [] diff --git a/backend/core/migrations/0012_remove_requirementnode_level_and_more.py b/backend/core/migrations/0012_remove_requirementnode_level_and_more.py deleted file mode 100644 index 7eda06377..000000000 --- a/backend/core/migrations/0012_remove_requirementnode_level_and_more.py +++ /dev/null @@ -1,118 +0,0 @@ -# Generated by Django 5.0.4 on 2024-04-30 21:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('core', '0011_merge_20240429_1027'), - ] - - operations = [ - migrations.RemoveField( - model_name='requirementnode', - name='level', - ), - migrations.AlterField( - model_name='appliedcontrol', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='asset', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='complianceassessment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='evidence', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='framework', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='loadedlibrary', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='loadedlibrary', - name='urn', - field=models.CharField(blank=True, max_length=100, null=True, verbose_name='URN'), - ), - migrations.AlterField( - model_name='project', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='referencecontrol', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='requirementassessment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='requirementnode', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='riskacceptance', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='riskassessment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='riskmatrix', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='riskscenario', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='storedlibrary', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterField( - model_name='storedlibrary', - name='urn', - field=models.CharField(blank=True, max_length=100, null=True, verbose_name='URN'), - ), - migrations.AlterField( - model_name='threat', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), - ), - migrations.AlterUniqueTogether( - name='loadedlibrary', - unique_together={('urn', 'locale', 'version')}, - ), - migrations.AlterUniqueTogether( - name='storedlibrary', - unique_together={('urn', 'locale', 'version')}, - ), - migrations.DeleteModel( - name='RequirementLevel', - ), - ] From 04791e793678b08eafcd6a65a3cb3beec589fe62 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 02:33:37 +0200 Subject: [PATCH 29/86] Add null check for source.meta --- frontend/src/lib/components/ModelTable/ModelTable.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index 8740c30e8..a9649cc43 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -130,7 +130,7 @@ const tagMap = FIELD_COLORED_TAG_MAP[URLModel] ?? {}; const taggedKeys = new Set(Object.keys(tagMap)); - $: model = source.meta.urlmodel ? URL_MODEL_MAP[source.meta.urlmodel] : URL_MODEL_MAP[URLModel]; + $: model = source.meta?.urlmodel ? URL_MODEL_MAP[source.meta.urlmodel] : URL_MODEL_MAP[URLModel]; $: source, handler.setRows(data); From 3a2bf84835c1386cb6c3c2598696a4d9b7f58a7f Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 02:44:48 +0200 Subject: [PATCH 30/86] Rename StoredLibrary.loads -> load --- backend/core/models.py | 2 +- backend/library/utils.py | 2 +- backend/library/views.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index d85d54d1d..11060f4a0 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -199,7 +199,7 @@ def store_library_file(cls, fname: Path) -> "StoredLibrary | None": library_content = f.read() return StoredLibrary.store_library_content(library_content) - def loads(self) -> Union[str, None]: + def load(self) -> Union[str, None]: from library.utils import LibraryImporter library_importer = LibraryImporter(self) diff --git a/backend/library/utils.py b/backend/library/utils.py index 1397b73ee..52ce7e4e9 100644 --- a/backend/library/utils.py +++ b/backend/library/utils.py @@ -584,7 +584,7 @@ def check_and_import_dependencies(self): dependency = StoredLibrary.objects.get( urn=dependency_urn, is_obsolete=False ) # We only fetch by URN without thinking about what locale, that may be a problem in the future. - error_msg = dependency.loads() + error_msg = dependency.load() if error_msg is not None: return error_msg diff --git a/backend/library/views.py b/backend/library/views.py index 40f0b00c5..247e91d57 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -65,7 +65,7 @@ def import_library(self, request, pk): return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) try: - error_msg = library.loads() + error_msg = library.load() if error_msg is not None: return Response( {"status": "error", "error": error_msg}, From 1d6e062c237b6b0de8474835188d9bb82c79a154 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 02:47:18 +0200 Subject: [PATCH 31/86] Fix unit tests --- backend/core/tests/test_helpers.py | 8 +++++--- backend/core/tests/test_models.py | 32 +++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/backend/core/tests/test_helpers.py b/backend/core/tests/test_helpers.py index ef5abe66b..315491ab5 100644 --- a/backend/core/tests/test_helpers.py +++ b/backend/core/tests/test_helpers.py @@ -8,9 +8,11 @@ @pytest.fixture def risk_matrix_fixture(): - import_library_view( - get_library("urn:intuitem:risk:library:critical_risk_matrix_5x5") - ) + library = StoredLibrary.objects.filter( + urn="urn:intuitem:risk:library:critical_risk_matrix_5x5" + ).last() + assert library is not None + library.load() @pytest.mark.django_db diff --git a/backend/core/tests/test_models.py b/backend/core/tests/test_models.py index 25f8c6699..cd795e0ac 100644 --- a/backend/core/tests/test_models.py +++ b/backend/core/tests/test_models.py @@ -16,6 +16,7 @@ Evidence, RiskAcceptance, Asset, + StoredLibrary, Threat, RiskMatrix, LoadedLibrary, @@ -42,9 +43,11 @@ def domain_project_fixture(): @pytest.fixture def risk_matrix_fixture(): - library = get_library("urn:intuitem:risk:library:critical_risk_matrix_5x5") + library = StoredLibrary.objects.filter( + urn="urn:intuitem:risk:library:critical_risk_matrix_5x5" + ).last() assert library is not None - import_library_view(library) + library.load() @pytest.mark.django_db @@ -869,6 +872,7 @@ def test_library_creation(self): folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) assert library.name == "Library" assert library.description == "Library description" @@ -883,6 +887,7 @@ def test_library_reference_count_zero_if_unused(self): folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) assert library.reference_count == 0 @@ -896,6 +901,7 @@ def test_library_reference_count_incremented_when_framework_is_referenced_by_com folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) framework = Framework.objects.create( name="Framework", @@ -929,6 +935,7 @@ def test_library_reference_count_incremented_when_reference_control_is_reference folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) framework = Framework.objects.create( name="Framework", @@ -1016,6 +1023,7 @@ def test_library_reference_count_incremented_when_threat_is_referenced_by_risk_s folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) threat = Threat.objects.create( name="Threat", @@ -1061,6 +1069,7 @@ def test_library_reference_count_incremented_when_reference_control_is_reference folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) reference_control = ReferenceControl.objects.create( name="ReferenceControl", @@ -1107,6 +1116,7 @@ def test_library_reference_count_must_be_zero_for_library_deletion( folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) framework = Framework.objects.create( name="Framework", @@ -1130,7 +1140,10 @@ def test_library_reference_count_must_be_zero_for_library_deletion( assert library.reference_count == 0 - library.delete() + try: # wrapping in try/except to avoid raising exception due to StoredLibrary not existing + library.delete() + except: + None assert LoadedLibrary.objects.count() == 0 @@ -1142,6 +1155,7 @@ def test_library_cannot_be_deleted_if_it_is_a_dependency_of_other_libraries(self folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) library = LoadedLibrary.objects.create( name="Library", @@ -1149,14 +1163,22 @@ def test_library_cannot_be_deleted_if_it_is_a_dependency_of_other_libraries(self folder=Folder.get_root_folder(), locale="en", version=1, + objects_meta={}, ) library.dependencies.add(dependency_library) with pytest.raises(ValueError): dependency_library.delete() - library.delete() + try: # wrapping in try/except to avoid raising exception due to StoredLibrary not existing + library.delete() + except: + None + assert LoadedLibrary.objects.count() == 1 - dependency_library.delete() + try: # wrapping in try/except to avoid raising exception due to StoredLibrary not existing + dependency_library.delete() + except: + None assert LoadedLibrary.objects.count() == 0 From bf862fc2ce8b16e8a2f642566124e1e625334759 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 02:48:24 +0200 Subject: [PATCH 32/86] chore: Run formatter --- ...lter_appliedcontrol_updated_at_and_more.py | 388 +++++++++++++----- ...dated_at_alter_role_updated_at_and_more.py | 33 +- 2 files changed, 298 insertions(+), 123 deletions(-) diff --git a/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py index 5c697ce4d..909bb3232 100644 --- a/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py @@ -7,164 +7,340 @@ class Migration(migrations.Migration): - dependencies = [ - ('core', '0010_rename_score_definition_framework_scores_definition_and_more'), - ('iam', '0003_alter_folder_updated_at_alter_role_updated_at_and_more'), + ("core", "0010_rename_score_definition_framework_scores_definition_and_more"), + ("iam", "0003_alter_folder_updated_at_alter_role_updated_at_and_more"), ] operations = [ migrations.AlterField( - model_name='appliedcontrol', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="appliedcontrol", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='asset', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="asset", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='complianceassessment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="complianceassessment", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='evidence', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="evidence", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='framework', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="framework", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='project', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="project", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='referencecontrol', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="referencecontrol", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='requirementassessment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="requirementassessment", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='requirementnode', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="requirementnode", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='riskacceptance', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="riskacceptance", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='riskassessment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="riskassessment", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='riskmatrix', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="riskmatrix", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='riskscenario', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="riskscenario", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='threat', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="threat", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.CreateModel( - name='LoadedLibrary', + name="LoadedLibrary", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), - ('updated_at', models.DateTimeField(auto_now=True, verbose_name='Updated at')), - ('is_published', models.BooleanField(default=False, verbose_name='published')), - ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), - ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), - ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), - ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), - ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), - ('description', models.TextField(blank=True, null=True, verbose_name='Description')), - ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), - ('urn', models.CharField(blank=True, max_length=100, null=True, verbose_name='URN')), - ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), - ('version', models.IntegerField(verbose_name='Version')), - ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), - ('builtin', models.BooleanField(default=False)), - ('objects_meta', models.JSONField()), - ('dependencies', models.ManyToManyField(blank=True, to='core.loadedlibrary', verbose_name='Dependencies')), - ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created_at", + models.DateTimeField(auto_now_add=True, verbose_name="Created at"), + ), + ( + "updated_at", + models.DateTimeField(auto_now=True, verbose_name="Updated at"), + ), + ( + "is_published", + models.BooleanField(default=False, verbose_name="published"), + ), + ( + "ref_id", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Reference ID", + ), + ), + ( + "locale", + models.CharField( + default="en", max_length=100, verbose_name="Locale" + ), + ), + ( + "default_locale", + models.BooleanField(default=True, verbose_name="Default locale"), + ), + ( + "provider", + models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider" + ), + ), + ( + "name", + models.CharField(max_length=200, null=True, verbose_name="Name"), + ), + ( + "description", + models.TextField(blank=True, null=True, verbose_name="Description"), + ), + ( + "annotation", + models.TextField(blank=True, null=True, verbose_name="Annotation"), + ), + ( + "urn", + models.CharField( + blank=True, max_length=100, null=True, verbose_name="URN" + ), + ), + ( + "copyright", + models.CharField( + blank=True, max_length=4096, null=True, verbose_name="Copyright" + ), + ), + ("version", models.IntegerField(verbose_name="Version")), + ( + "packager", + models.CharField( + blank=True, + help_text="Packager of the library", + max_length=100, + null=True, + verbose_name="Packager", + ), + ), + ("builtin", models.BooleanField(default=False)), + ("objects_meta", models.JSONField()), + ( + "dependencies", + models.ManyToManyField( + blank=True, to="core.loadedlibrary", verbose_name="Dependencies" + ), + ), + ( + "folder", + models.ForeignKey( + default=iam.models.Folder.get_root_folder, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_folder", + to="iam.folder", + ), + ), ], options={ - 'abstract': False, - 'unique_together': {('urn', 'locale', 'version')}, + "abstract": False, + "unique_together": {("urn", "locale", "version")}, }, ), migrations.AlterField( - model_name='framework', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='frameworks', to='core.loadedlibrary'), + model_name="framework", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="frameworks", + to="core.loadedlibrary", + ), ), migrations.AlterField( - model_name='referencecontrol', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='reference_controls', to='core.loadedlibrary'), + model_name="referencecontrol", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="reference_controls", + to="core.loadedlibrary", + ), ), migrations.AlterField( - model_name='riskmatrix', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='risk_matrices', to='core.loadedlibrary'), + model_name="riskmatrix", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="risk_matrices", + to="core.loadedlibrary", + ), ), migrations.AlterField( - model_name='threat', - name='library', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='threats', to='core.loadedlibrary'), + model_name="threat", + name="library", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="threats", + to="core.loadedlibrary", + ), ), migrations.CreateModel( - name='StoredLibrary', + name="StoredLibrary", fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='Created at')), - ('updated_at', models.DateTimeField(auto_now=True, verbose_name='Updated at')), - ('is_published', models.BooleanField(default=False, verbose_name='published')), - ('ref_id', models.CharField(blank=True, max_length=100, null=True, verbose_name='Reference ID')), - ('locale', models.CharField(default='en', max_length=100, verbose_name='Locale')), - ('default_locale', models.BooleanField(default=True, verbose_name='Default locale')), - ('provider', models.CharField(blank=True, max_length=200, null=True, verbose_name='Provider')), - ('name', models.CharField(max_length=200, null=True, verbose_name='Name')), - ('description', models.TextField(blank=True, null=True, verbose_name='Description')), - ('annotation', models.TextField(blank=True, null=True, verbose_name='Annotation')), - ('urn', models.CharField(blank=True, max_length=100, null=True, verbose_name='URN')), - ('copyright', models.CharField(blank=True, max_length=4096, null=True, verbose_name='Copyright')), - ('version', models.IntegerField(verbose_name='Version')), - ('packager', models.CharField(blank=True, help_text='Packager of the library', max_length=100, null=True, verbose_name='Packager')), - ('builtin', models.BooleanField(default=False)), - ('objects_meta', models.JSONField()), - ('dependencies', models.JSONField(null=True)), - ('is_obsolete', models.BooleanField(default=False)), - ('is_imported', models.BooleanField(default=False)), - ('hash_checksum', models.CharField(max_length=64)), - ('content', models.TextField()), - ('folder', models.ForeignKey(default=iam.models.Folder.get_root_folder, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)s_folder', to='iam.folder')), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created_at", + models.DateTimeField(auto_now_add=True, verbose_name="Created at"), + ), + ( + "updated_at", + models.DateTimeField(auto_now=True, verbose_name="Updated at"), + ), + ( + "is_published", + models.BooleanField(default=False, verbose_name="published"), + ), + ( + "ref_id", + models.CharField( + blank=True, + max_length=100, + null=True, + verbose_name="Reference ID", + ), + ), + ( + "locale", + models.CharField( + default="en", max_length=100, verbose_name="Locale" + ), + ), + ( + "default_locale", + models.BooleanField(default=True, verbose_name="Default locale"), + ), + ( + "provider", + models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider" + ), + ), + ( + "name", + models.CharField(max_length=200, null=True, verbose_name="Name"), + ), + ( + "description", + models.TextField(blank=True, null=True, verbose_name="Description"), + ), + ( + "annotation", + models.TextField(blank=True, null=True, verbose_name="Annotation"), + ), + ( + "urn", + models.CharField( + blank=True, max_length=100, null=True, verbose_name="URN" + ), + ), + ( + "copyright", + models.CharField( + blank=True, max_length=4096, null=True, verbose_name="Copyright" + ), + ), + ("version", models.IntegerField(verbose_name="Version")), + ( + "packager", + models.CharField( + blank=True, + help_text="Packager of the library", + max_length=100, + null=True, + verbose_name="Packager", + ), + ), + ("builtin", models.BooleanField(default=False)), + ("objects_meta", models.JSONField()), + ("dependencies", models.JSONField(null=True)), + ("is_obsolete", models.BooleanField(default=False)), + ("is_imported", models.BooleanField(default=False)), + ("hash_checksum", models.CharField(max_length=64)), + ("content", models.TextField()), + ( + "folder", + models.ForeignKey( + default=iam.models.Folder.get_root_folder, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_folder", + to="iam.folder", + ), + ), ], options={ - 'abstract': False, - 'unique_together': {('urn', 'locale', 'version')}, + "abstract": False, + "unique_together": {("urn", "locale", "version")}, }, ), migrations.DeleteModel( - name='Library', + name="Library", ), ] diff --git a/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py b/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py index 922833008..07acec635 100644 --- a/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py +++ b/backend/iam/migrations/0003_alter_folder_updated_at_alter_role_updated_at_and_more.py @@ -4,35 +4,34 @@ class Migration(migrations.Migration): - dependencies = [ - ('iam', '0002_purge_validator'), + ("iam", "0002_purge_validator"), ] operations = [ migrations.AlterField( - model_name='folder', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="folder", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='role', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="role", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='roleassignment', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="roleassignment", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='user', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="user", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), migrations.AlterField( - model_name='usergroup', - name='updated_at', - field=models.DateTimeField(auto_now=True, verbose_name='Updated at'), + model_name="usergroup", + name="updated_at", + field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), ] From efcbdbe8e023de85ca656789beba9448ab66b53f Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Wed, 1 May 2024 02:53:25 +0200 Subject: [PATCH 33/86] Fix href in functional tests --- frontend/tests/utils/page-content.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/tests/utils/page-content.ts b/frontend/tests/utils/page-content.ts index effea8eef..4a50f1ace 100644 --- a/frontend/tests/utils/page-content.ts +++ b/frontend/tests/utils/page-content.ts @@ -38,8 +38,8 @@ export class PageContent extends BasePage { async createItem(values: { [k: string]: any }, dependency?: any) { if (dependency) { - await this.page.goto('/loaded-libraries'); - await this.page.waitForURL('/loaded-libraries'); + await this.page.goto('/libraries'); + await this.page.waitForURL('/libraries'); await this.importLibrary(dependency.ref || dependency.name, dependency.urn); await this.goto(); @@ -65,7 +65,7 @@ export class PageContent extends BasePage { } } - async importLibrary(ref: string, urn?: string, language: string = 'English') { + async importLibrary(ref: string, urn?: string, language = 'English') { if ( (await this.tab('Imported libraries').isVisible()) && (await this.tab('Imported libraries').getAttribute('aria-selected')) === 'true' From 33006f2c675668a1c405c57a0a57f1e8fa4fba14 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:04:04 +0200 Subject: [PATCH 34/86] Pass API tests (finally) --- backend/app_tests/api/test_api_libraries.py | 30 ++++++++++++------- .../api/test_api_requirement_nodes.py | 5 ++-- backend/app_tests/api/test_utils.py | 18 ++++++++--- backend/app_tests/test_vars.py | 2 +- backend/core/apps.py | 2 +- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/backend/app_tests/api/test_api_libraries.py b/backend/app_tests/api/test_api_libraries.py index 2076ca19d..d83eb9041 100644 --- a/backend/app_tests/api/test_api_libraries.py +++ b/backend/app_tests/api/test_api_libraries.py @@ -1,6 +1,8 @@ +import json import pytest from rest_framework.test import APIClient -from core.models import Framework +from app_tests.test_vars import TEST_FRAMEWORK_URN, TEST_RISK_MATRIX_URN +from core.models import Framework, StoredLibrary from core.models import RiskMatrix from iam.models import Folder from rest_framework import status @@ -17,7 +19,7 @@ class TestLibrariesUnauthenticated: def test_get_libraries(self): """test to get libraries from the API without authentication""" - EndpointTestsQueries.get_object(self.client, "Libraries") + EndpointTestsQueries.get_object(self.client, "Stored libraries") def test_import_frameworks(self): """test to import libraries with the API without authentication""" @@ -50,7 +52,7 @@ def test_get_libraries(self, test): """test to get libraries from the API with authentication""" EndpointTestsQueries.Auth.get_object( - test.client, "Libraries", base_count=-1, user_group=test.user_group + test.client, "Stored libraries", base_count=-1, user_group=test.user_group ) def test_import_frameworks(self, test): @@ -58,8 +60,11 @@ def test_import_frameworks(self, test): # Uses the API endpoint to get library details with the admin client lib_detail_response = test.admin_client.get( - EndpointTestsUtils.get_object_urn("Framework") - ).json()["objects"]["framework"] + EndpointTestsUtils.get_referential_object_url_from_urn( + test.client, TEST_FRAMEWORK_URN, StoredLibrary + ) + ).json()["content"] + lib_detail_response = json.loads(lib_detail_response)["framework"] # Asserts that the library is not already imported assert ( @@ -78,7 +83,7 @@ def test_import_frameworks(self, test): assert Framework.objects.all().count() == ( 1 if not EndpointTestsUtils.expected_request_response( - "add", "library", str(test.folder), test.user_group + "add", "loadedlibrary", str(test.folder), test.user_group )[0] else 0 ), "Frameworks are not correctly imported in the database" @@ -96,7 +101,7 @@ def test_import_frameworks(self, test): base_count=1, user_group=test.user_group, fails=EndpointTestsUtils.expected_request_response( - "add", "library", str(test.folder), test.user_group + "add", "loadedlibrary", str(test.folder), test.user_group )[0], ) @@ -121,8 +126,11 @@ def test_import_risk_matrix(self, test): # Uses the API endpoint to get library details with the admin client lib_detail_response = test.admin_client.get( - EndpointTestsUtils.get_object_urn("Risk matrix") - ).json()["objects"]["risk_matrix"][0] + EndpointTestsUtils.get_referential_object_url_from_urn( + test.client, TEST_RISK_MATRIX_URN, StoredLibrary + ) + ).json()["content"] + lib_detail_response = json.loads(lib_detail_response)["risk_matrix"][0] # Asserts that the library is not already imported assert ( @@ -139,7 +147,7 @@ def test_import_risk_matrix(self, test): assert RiskMatrix.objects.all().count() == ( 1 if not EndpointTestsUtils.expected_request_response( - "add", "library", str(test.folder), test.user_group + "add", "loadedlibrary", str(test.folder), test.user_group )[0] else 0 ), "Risk matrices are not correctly imported in the database" @@ -158,7 +166,7 @@ def test_import_risk_matrix(self, test): base_count=1, user_group=test.user_group, fails=EndpointTestsUtils.expected_request_response( - "add", "library", str(test.folder), test.user_group + "add", "loadedlibrary", str(test.folder), test.user_group )[0], ) diff --git a/backend/app_tests/api/test_api_requirement_nodes.py b/backend/app_tests/api/test_api_requirement_nodes.py index 5d0ac84fe..f97905b67 100644 --- a/backend/app_tests/api/test_api_requirement_nodes.py +++ b/backend/app_tests/api/test_api_requirement_nodes.py @@ -1,6 +1,7 @@ import pytest from rest_framework.test import APIClient -from core.models import RequirementNode, Framework +from app_tests.test_vars import TEST_FRAMEWORK_URN +from core.models import RequirementNode, Framework, StoredLibrary from iam.models import Folder from test_utils import EndpointTestsQueries, EndpointTestsUtils @@ -76,7 +77,7 @@ def test_import_requirement_nodes(self, test): test.client, "Requirement nodes", EndpointTestsUtils.get_endpoint_url("Requirement nodes"), - EndpointTestsUtils.get_object_urn("Framework"), + EndpointTestsUtils.get_referential_object_url_from_urn(test.client, TEST_FRAMEWORK_URN, StoredLibrary), [ "name", "description", diff --git a/backend/app_tests/api/test_utils.py b/backend/app_tests/api/test_utils.py index 701d409d2..3f571315d 100644 --- a/backend/app_tests/api/test_utils.py +++ b/backend/app_tests/api/test_utils.py @@ -1,3 +1,4 @@ +from django.db import models from knox.auth import AuthToken import pytest import json @@ -6,6 +7,7 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APIClient +from core.models import StoredLibrary from test_vars import * @@ -27,6 +29,13 @@ def get_object_urn(object_name: str, resolved: bool = True): urn = get_var(urn_varname) return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/" if resolved else eval(urn) + def get_referential_object_url_from_urn( + authenticated_client, urn: str, model: models.Model = StoredLibrary + ): + """Get the object URL from the URN""" + uuid = model.objects.filter(urn=urn).last().id + return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{uuid}/" + @pytest.mark.django_db def get_test_client_and_folder( authenticated_client, @@ -958,7 +967,7 @@ def import_object( user_perm_expected_status, user_perm_reason, ) = EndpointTestsUtils.expected_request_response( - "add", "library", scope, user_group, expected_status + "add", "loadedlibrary", scope, user_group, expected_status ) url = urn or EndpointTestsUtils.get_object_urn(verbose_name) @@ -1014,9 +1023,10 @@ def compare_results( reference.status_code == status.HTTP_200_OK ), "reference endpoint is not accessible" - for object in reference.json()["objects"]["framework"][ - object_name.lower().replace(" ", "_") - ][:count]: + content = json.loads(reference.json()["content"]) + for object in content["framework"][object_name.lower().replace(" ", "_")][ + :count + ]: comparelist = authenticated_client.get(compare_url) compare = dict() assert ( diff --git a/backend/app_tests/test_vars.py b/backend/app_tests/test_vars.py index 4cc64d9ca..4450f18ed 100644 --- a/backend/app_tests/test_vars.py +++ b/backend/app_tests/test_vars.py @@ -95,7 +95,7 @@ def get_var(varname: str) -> Any: def get_singular_name(plural_name: str) -> str: exceptions = { - "Libraries": "Library", + "Stored libraries": "Stored library", "Risk matrices": "Risk matrix", "Policies": "Policy", } diff --git a/backend/core/apps.py b/backend/core/apps.py index ed9edb367..0d545caec 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -26,6 +26,7 @@ "view_evidence", "view_framework", "view_loadedlibrary", + "view_storedlibrary", "view_user", ] @@ -238,7 +239,6 @@ "view_framework", "delete_framework", "view_requirementnode", - "view_requirementlevel", # Permits to see the object on api by an admin "view_storedlibrary", "add_storedlibrary", "delete_storedlibrary", From 61bccbc8eb01bd7fed204202bc1c6ca3bed629f1 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:35:33 +0200 Subject: [PATCH 35/86] Fix ruff pre-commit hook --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbe741ab8..7b7fa3454 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: hooks: # Run the linter. - id: ruff - args: [--fix backend] + args: [--fix, backend] # Run the formatter. - id: ruff-format args: [backend] From a9d6f406ac88c0a1c258c580552eb5fabc82190a Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:37:06 +0200 Subject: [PATCH 36/86] Disable ruff linting in pre-commit --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7b7fa3454..32e52755d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,8 +9,8 @@ repos: rev: v0.4.1 hooks: # Run the linter. - - id: ruff - args: [--fix, backend] + # - id: ruff + # args: [--fix, backend] # Run the formatter. - id: ruff-format args: [backend] From a2c149404fa67edd7c52d588fcdb4f0fbd963bb1 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:37:29 +0200 Subject: [PATCH 37/86] chore: Run formatter --- backend/app_tests/api/test_api_requirement_nodes.py | 4 +++- backend/core/apps.py | 4 +--- backend/core/tests/test_models.py | 1 - backend/core/urls.py | 1 - backend/library/serializers.py | 3 --- 5 files changed, 4 insertions(+), 9 deletions(-) diff --git a/backend/app_tests/api/test_api_requirement_nodes.py b/backend/app_tests/api/test_api_requirement_nodes.py index f97905b67..5bb9a0b20 100644 --- a/backend/app_tests/api/test_api_requirement_nodes.py +++ b/backend/app_tests/api/test_api_requirement_nodes.py @@ -77,7 +77,9 @@ def test_import_requirement_nodes(self, test): test.client, "Requirement nodes", EndpointTestsUtils.get_endpoint_url("Requirement nodes"), - EndpointTestsUtils.get_referential_object_url_from_urn(test.client, TEST_FRAMEWORK_URN, StoredLibrary), + EndpointTestsUtils.get_referential_object_url_from_urn( + test.client, TEST_FRAMEWORK_URN, StoredLibrary + ), [ "name", "description", diff --git a/backend/core/apps.py b/backend/core/apps.py index 0d545caec..b90f7359c 100644 --- a/backend/core/apps.py +++ b/backend/core/apps.py @@ -1,8 +1,6 @@ -import sys from django.apps import AppConfig -from django.db import connection from django.db.models.signals import post_migrate -from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL, LIBRARIES_PATH +from ciso_assistant.settings import CISO_ASSISTANT_SUPERUSER_EMAIL import os from django.core.management import call_command diff --git a/backend/core/tests/test_models.py b/backend/core/tests/test_models.py index cd795e0ac..e21387024 100644 --- a/backend/core/tests/test_models.py +++ b/backend/core/tests/test_models.py @@ -25,7 +25,6 @@ from django.contrib.auth import get_user_model from django.core.files.uploadedfile import SimpleUploadedFile from iam.models import Folder -from library.utils import import_library_view, get_library User = get_user_model() diff --git a/backend/core/urls.py b/backend/core/urls.py index 58e294906..2f8062ee6 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -1,5 +1,4 @@ from .views import * -from django.contrib.auth import views as auth_views from library.views import StoredLibraryViewSet, LoadedLibraryViewSet diff --git a/backend/library/serializers.py b/backend/library/serializers.py index 4c05c0667..7340e5b34 100644 --- a/backend/library/serializers.py +++ b/backend/library/serializers.py @@ -1,7 +1,4 @@ from core.models import StoredLibrary, LoadedLibrary -from core.serializers import ( - BaseModelSerializer, -) from rest_framework import serializers """class LibraryObjectSerializer(serializers.Serializer): From 4d6918bffe8ce308f60386b33b1f2dcb8fe74f9e Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:40:52 +0200 Subject: [PATCH 38/86] pre-commit: Exclude migrations files from format checking --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32e52755d..2ea348010 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: # Run the formatter. - id: ruff-format args: [backend] + exclude: (migrations) - repo: local hooks: - id: format-frontend From 0b6b76ac93d4cccff89cc6a0acb860ecf95b4cb5 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:41:51 +0200 Subject: [PATCH 39/86] chore: Run ruff format --- tools/aircyber/aircyber.py | 4 +++- tools/ccm/convert_ccm.py | 10 ++++++++-- tools/cis/convert_cis.py | 24 ++++++++++++++++++++---- tools/convert_framework.py | 14 ++++++++++---- tools/tisax/convert_tisax.py | 25 ++++++++++++++++++------- 5 files changed, 59 insertions(+), 18 deletions(-) diff --git a/tools/aircyber/aircyber.py b/tools/aircyber/aircyber.py index 72d3ad944..54fdabdfb 100644 --- a/tools/aircyber/aircyber.py +++ b/tools/aircyber/aircyber.py @@ -95,7 +95,9 @@ ws.append(["tab", "implementation_groups", "implementation_groups"]) ws1 = wb_output.create_sheet("controls") -ws1.append(["assessable", "depth", "ref_id", "name", "description", "implementation_groups"]) +ws1.append( + ["assessable", "depth", "ref_id", "name", "description", "implementation_groups"] +) for row in output_table: ws1.append(row) ws2 = wb_output.create_sheet("implementation_groups") diff --git a/tools/ccm/convert_ccm.py b/tools/ccm/convert_ccm.py index f27204481..0974ea84c 100644 --- a/tools/ccm/convert_ccm.py +++ b/tools/ccm/convert_ccm.py @@ -97,12 +97,18 @@ def pretify_content(content): ws1 = wb_output.create_sheet("controls") ws1.append( ["assessable", "depth", "ref_id", "name", "description", "implementation_groups"] - ) +) for row in output_table: ws1.append(row) ws2 = wb_output.create_sheet("implementation_groups") ws2.append(["ref_id", "name", "description"]) -ws2.append(["lite", "foundational", "foundational controls that should be implemented by any organization, regardless of their budget, maturity and risk profile"]) +ws2.append( + [ + "lite", + "foundational", + "foundational controls that should be implemented by any organization, regardless of their budget, maturity and risk profile", + ] +) ws2.append(["full", "systematic ", "systematic assessment of a cloud implementation"]) print("generate ", output_file_name) wb_output.save(output_file_name) diff --git a/tools/cis/convert_cis.py b/tools/cis/convert_cis.py index 945a10e89..458993cbb 100644 --- a/tools/cis/convert_cis.py +++ b/tools/cis/convert_cis.py @@ -44,7 +44,9 @@ else: safeguard_index += 1 safeguard = f"{control},{safeguard_index}" - implementation_groups = "IG1,IG2,IG3" if ig1 else "IG2,IG3" if ig2 else "IG3" + implementation_groups = ( + "IG1,IG2,IG3" if ig1 else "IG2,IG3" if ig2 else "IG3" + ) output_table.append( ("x", 2, safeguard, title, description, implementation_groups) ) @@ -73,14 +75,28 @@ ws.append(["tab", "implementation_groups", "implementation_groups"]) ws1 = wb_output.create_sheet("controls") -ws1.append(["assessable", "depth", "ref_id", "name", "description", "implementation_groups"]) +ws1.append( + ["assessable", "depth", "ref_id", "name", "description", "implementation_groups"] +) for row in output_table: ws1.append(row) ws2 = wb_output.create_sheet("implementation_groups") ws2.append(["ref_id", "name", "description"]) -ws2.append(["IG1", "Essential Cyber Hygiene", "Minimum standard of information security for all enterprises."]) -ws2.append(["IG2", "", "For enterprises managing IT infrastructure of multiple departments with differing risk profiles."]) +ws2.append( + [ + "IG1", + "Essential Cyber Hygiene", + "Minimum standard of information security for all enterprises.", + ] +) +ws2.append( + [ + "IG2", + "", + "For enterprises managing IT infrastructure of multiple departments with differing risk profiles.", + ] +) ws2.append(["IG3", "", "To secure sensitive and confidential data."]) print("generate ", output_file_name) diff --git a/tools/convert_framework.py b/tools/convert_framework.py index a64a1c901..313e96b6c 100644 --- a/tools/convert_framework.py +++ b/tools/convert_framework.py @@ -217,7 +217,7 @@ def read_header(row): if annotation: req_node["annotation"] = annotation if implementation_groups: - req_node["implementation_groups"] = implementation_groups.split(',') + req_node["implementation_groups"] = implementation_groups.split(",") threats = row[header["threats"]].value if "threats" in header else None reference_controls = ( row[header["reference_controls"]].value @@ -404,13 +404,19 @@ def read_header(row): "description": library_vars["framework_description"], } if "framework_min_score" in library_vars: - library["objects"]["framework"]["min_score"] = library_vars["framework_min_score"] + library["objects"]["framework"]["min_score"] = library_vars[ + "framework_min_score" + ] if "framework_max_score" in library_vars: - library["objects"]["framework"]["max_score"] = library_vars["framework_max_score"] + library["objects"]["framework"]["max_score"] = library_vars[ + "framework_max_score" + ] if scores_definition: library["objects"]["framework"]["scores_definition"] = scores_definition if implementation_groups_definition: - library["objects"]["framework"]["implementation_groups_definition"] = implementation_groups_definition + library["objects"]["framework"]["implementation_groups_definition"] = ( + implementation_groups_definition + ) library["objects"]["framework"]["requirement_nodes"] = requirement_nodes if has_reference_controls: diff --git a/tools/tisax/convert_tisax.py b/tools/tisax/convert_tisax.py index e010fdb87..b58e1faee 100644 --- a/tools/tisax/convert_tisax.py +++ b/tools/tisax/convert_tisax.py @@ -50,9 +50,9 @@ req_sga = None req_vehicle = None further_info = None - ex_normal=None - ex_high=None - ex_very_high=None + ex_normal = None + ex_high = None + ex_very_high = None if title == "Information Security": ( _, @@ -127,10 +127,19 @@ output_table.append(("", level, control_number, control_question, "")) output_table.append(("x", level + 1, "", "(must)", req_must, "must")) if req_should and req_should != "None": - output_table.append(("x", level + 1, "", "(should)", req_should, "should")) + output_table.append( + ("x", level + 1, "", "(should)", req_should, "should") + ) if req_high and req_high != "None": output_table.append( - ("x", level + 1, "", "(for high protection needs)", req_high, "high") + ( + "x", + level + 1, + "", + "(for high protection needs)", + req_high, + "high", + ) ) if req_very_high and req_very_high != "None": output_table.append( @@ -140,7 +149,7 @@ "", "(for very high protection needs)", req_very_high, - "very_high" + "very_high", ) ) if req_sga and req_sga != "None": @@ -194,7 +203,9 @@ ws.append(["tab", "implementation_groups", "implementation_groups"]) ws1 = wb_output.create_sheet("controls") -ws1.append(["assessable", "depth", "ref_id", "name", "description", "implementation_groups"]) +ws1.append( + ["assessable", "depth", "ref_id", "name", "description", "implementation_groups"] +) for row in output_table: ws1.append(row) From 3f87144aa7d2ae269194846b2ddc0347e740c005 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 14:42:04 +0200 Subject: [PATCH 40/86] chore: Run migrations --- ...re.py => 0012_alter_appliedcontrol_updated_at_and_more.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename backend/core/migrations/{0011_alter_appliedcontrol_updated_at_and_more.py => 0012_alter_appliedcontrol_updated_at_and_more.py} (98%) diff --git a/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py similarity index 98% rename from backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py rename to backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py index 909bb3232..87a6efc14 100644 --- a/backend/core/migrations/0011_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.4 on 2024-04-30 23:57 +# Generated by Django 5.0.4 on 2024-05-03 12:41 import django.db.models.deletion import iam.models @@ -8,7 +8,7 @@ class Migration(migrations.Migration): dependencies = [ - ("core", "0010_rename_score_definition_framework_scores_definition_and_more"), + ("core", "0011_auto_20240501_1342"), ("iam", "0003_alter_folder_updated_at_alter_role_updated_at_and_more"), ] From 6a0c87af9562f9301b45b05f79107717bbf1a9a2 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 16:15:42 +0200 Subject: [PATCH 41/86] Fix stack trace information exposure on library storage failure --- backend/library/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/library/views.py b/backend/library/views.py index 247e91d57..51ebfddd4 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -27,6 +27,10 @@ StoredLibrarySerializer, ) +import structlog + +logger = structlog.get_logger(__name__) + class StoredLibraryViewSet(BaseModelViewSet): parser_classes = [FileUploadParser] @@ -118,8 +122,9 @@ def upload_library(self, request): try: StoredLibrary.store_library_content(content) except ValueError as e: + logger.error("Failed to store library content", error=e) return HttpResponse( - json.dumps({"error": e}), + json.dumps({"error": "Failed to store library content"}), status=HTTP_422_UNPROCESSABLE_ENTITY, ) From 53d507b472a8ea97c1f034e9774c887f73abb916 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 16:18:45 +0200 Subject: [PATCH 42/86] Fix default django host in e2e tests script lcoalhost and 127.0.0.1 can not always be used interchangeably --- frontend/tests/e2e-tests.sh | 263 ++++++++++++++++++------------------ 1 file changed, 131 insertions(+), 132 deletions(-) diff --git a/frontend/tests/e2e-tests.sh b/frontend/tests/e2e-tests.sh index 44c5f0b99..cd37156cb 100755 --- a/frontend/tests/e2e-tests.sh +++ b/frontend/tests/e2e-tests.sh @@ -12,130 +12,129 @@ BACKEND_PORT=8173 MAILER_WEB_SERVER_PORT=8073 MAILER_SMTP_SERVER_PORT=1073 -for arg in "$@" -do - if [[ $arg == --port* ]]; then - if [[ "${arg#*=}" =~ ^[0-9]+$ ]]; then - BACKEND_PORT="${arg#*=}" - else - echo "Invalid format for --port argument. Please use --port=PORT" - exit 1 - fi - elif [[ $arg == --env* ]]; then - if [[ "${arg#*=}" =~ ^(.+)\/([^\/.]+)$ ]]; then - VENV_PATH="${arg#*=}" - else - echo "Invalid format for --env argument. Please use --env=PATH" - exit 1 - fi - elif [[ $arg == --mailer* ]]; then - MAILER_PORTS="${arg#*=}" - if [[ $MAILER_PORTS =~ ^[0-9]+/[0-9]+$ ]]; then - IFS='/' read -ra PORTS <<< "$MAILER_PORTS" - MAILER_SMTP_SERVER_PORT="${PORTS[0]}" - MAILER_WEB_SERVER_PORT="${PORTS[1]}" - SCRIPT_SHORT_ARGS+=("-m") - else - echo "Invalid format for --mailer argument. Please use --mailer=PORT/PORT" - exit 1 - fi - elif [[ $arg == --* ]]; then - SCRIPT_LONG_ARGS+=("$arg") - elif [[ $arg == -* ]]; then - SCRIPT_SHORT_ARGS+=("$arg") - elif [[ $arg != -* ]]; then - TEST_PATHS+=("$arg") - fi +for arg in "$@"; do + if [[ $arg == --port* ]]; then + if [[ "${arg#*=}" =~ ^[0-9]+$ ]]; then + BACKEND_PORT="${arg#*=}" + else + echo "Invalid format for --port argument. Please use --port=PORT" + exit 1 + fi + elif [[ $arg == --env* ]]; then + if [[ "${arg#*=}" =~ ^(.+)\/([^\/.]+)$ ]]; then + VENV_PATH="${arg#*=}" + else + echo "Invalid format for --env argument. Please use --env=PATH" + exit 1 + fi + elif [[ $arg == --mailer* ]]; then + MAILER_PORTS="${arg#*=}" + if [[ $MAILER_PORTS =~ ^[0-9]+/[0-9]+$ ]]; then + IFS='/' read -ra PORTS <<<"$MAILER_PORTS" + MAILER_SMTP_SERVER_PORT="${PORTS[0]}" + MAILER_WEB_SERVER_PORT="${PORTS[1]}" + SCRIPT_SHORT_ARGS+=("-m") + else + echo "Invalid format for --mailer argument. Please use --mailer=PORT/PORT" + exit 1 + fi + elif [[ $arg == --* ]]; then + SCRIPT_LONG_ARGS+=("$arg") + elif [[ $arg == -* ]]; then + SCRIPT_SHORT_ARGS+=("$arg") + elif [[ $arg != -* ]]; then + TEST_PATHS+=("$arg") + fi done if [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -h " ]] || [[ " ${SCRIPT_LONG_ARGS[@]} " =~ " --help " ]]; then - echo "Usage: e2e-tests.sh [options] [test_path]" - echo "Run the end-to-end tests for the CISO Assistant application." - echo "Options:" - echo " --browser=NAME Run the tests in the specified browser (chromium, firefox, webkit)" - echo " --env=PATH Path to the virtual environment to use for the tests (default: first one found in $VENV_PATH)" - echo " --global-timeout=MS Maximum time this test suite can run in milliseconds (default: unlimited)" - echo " --grep=SEARCH Only run tests matching this regular expression (default: \".*\")" - echo " --headed Run the tests in headful mode" - echo " -h Show this help message and exit" - echo " --list List all the tests" - echo " -m, --mailer=PORT/PORT Use an existing mailer service on the optionally defined ports (default: $MAILER_SMTP_SERVER_PORT/$MAILER_WEB_SERVER_PORT)" - echo " --port=PORT Run the backend server on the specified port (default: $BACKEND_PORT)" - echo " --project=NAME Run the tests in the specified project (chromium, firefox, webkit)" - echo " -q Quick mode: execute only the tests 1 time with no retries and only 1 project" - echo " --repeat-each=COUNT Run the tests the specified number of times (default: 1)" - echo " --retries=COUNT Set the number of retries for the tests" - echo " --timeout=MS Set the timeout for the tests in milliseconds" - echo " -v Show the output of the backend server" - echo -e " --workers=COUNT Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 1)" - echo " Be aware that increasing the number of workers may reduce tests accuracy and stability" - exit 0 + echo "Usage: e2e-tests.sh [options] [test_path]" + echo "Run the end-to-end tests for the CISO Assistant application." + echo "Options:" + echo " --browser=NAME Run the tests in the specified browser (chromium, firefox, webkit)" + echo " --env=PATH Path to the virtual environment to use for the tests (default: first one found in $VENV_PATH)" + echo " --global-timeout=MS Maximum time this test suite can run in milliseconds (default: unlimited)" + echo " --grep=SEARCH Only run tests matching this regular expression (default: \".*\")" + echo " --headed Run the tests in headful mode" + echo " -h Show this help message and exit" + echo " --list List all the tests" + echo " -m, --mailer=PORT/PORT Use an existing mailer service on the optionally defined ports (default: $MAILER_SMTP_SERVER_PORT/$MAILER_WEB_SERVER_PORT)" + echo " --port=PORT Run the backend server on the specified port (default: $BACKEND_PORT)" + echo " --project=NAME Run the tests in the specified project (chromium, firefox, webkit)" + echo " -q Quick mode: execute only the tests 1 time with no retries and only 1 project" + echo " --repeat-each=COUNT Run the tests the specified number of times (default: 1)" + echo " --retries=COUNT Set the number of retries for the tests" + echo " --timeout=MS Set the timeout for the tests in milliseconds" + echo " -v Show the output of the backend server" + echo -e " --workers=COUNT Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 1)" + echo " Be aware that increasing the number of workers may reduce tests accuracy and stability" + exit 0 fi ENV_CFG=$(find "$VENV_PATH" -name "pyvenv.cfg" -print -quit) if [[ ! -z $ENV_CFG ]]; then - VENV_PATH=$(dirname $ENV_CFG) - if [ -d "$VENV_PATH/bin" ] ; then - source "$VENV_PATH/bin/activate" - else - source "$VENV_PATH/Scripts/activate" - fi - echo "Using virtual environment at $VENV_PATH" + VENV_PATH=$(dirname $ENV_CFG) + if [ -d "$VENV_PATH/bin" ]; then + source "$VENV_PATH/bin/activate" + else + source "$VENV_PATH/Scripts/activate" + fi + echo "Using virtual environment at $VENV_PATH" else - echo "No virtual environment found at $VENV_PATH, using standard python environment instead." + echo "No virtual environment found at $VENV_PATH, using standard python environment instead." fi -if python -c "import socket;exit(0 if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost',$BACKEND_PORT)) else 1)" ; then - echo "The port $BACKEND_PORT is already in use!" - echo "Please stop the running process using the port or change the backend test server port using --port=PORT and try again." - exit 1 +if python -c "import socket;exit(0 if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',$BACKEND_PORT)) else 1)"; then + echo "The port $BACKEND_PORT is already in use!" + echo "Please stop the running process using the port or change the backend test server port using --port=PORT and try again." + exit 1 fi -for PORT in $MAILER_WEB_SERVER_PORT $MAILER_SMTP_SERVER_PORT ; do - if python -c "import socket;exit(0 if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost',$PORT)) else 1)" ; then - if [[ ! " ${SCRIPT_SHORT_ARGS[@]} " =~ " -m " ]] ; then - echo "The port $PORT is already in use!" - echo "Please stop the running process using the port or change the mailer port and try again." - exit 1 - fi - elif [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -m " ]] ; then - echo "No mailer service is running on port $PORT!" - echo "Please start a mailer service on port $PORT or change the mailer port using --mailer=PORT/PORT and try again." - echo "You can also use the isolated test mailer service by removing the -m option." - exit 1 - fi +for PORT in $MAILER_WEB_SERVER_PORT $MAILER_SMTP_SERVER_PORT; do + if python -c "import socket;exit(0 if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost',$PORT)) else 1)"; then + if [[ ! " ${SCRIPT_SHORT_ARGS[@]} " =~ " -m " ]]; then + echo "The port $PORT is already in use!" + echo "Please stop the running process using the port or change the mailer port and try again." + exit 1 + fi + elif [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -m " ]]; then + echo "No mailer service is running on port $PORT!" + echo "Please start a mailer service on port $PORT or change the mailer port using --mailer=PORT/PORT and try again." + echo "You can also use the isolated test mailer service by removing the -m option." + exit 1 + fi done cleanup() { - echo -e "\nCleaning up..." - if type deactivate >/dev/null 2>&1; then - deactivate - fi - if [ -n "$BACKEND_PID" ] ; then - kill $BACKEND_PID > /dev/null 2>&1 - echo "| backend server stopped" - fi - if [ -n "$MAILER_PID" ] ; then - docker stop $MAILER_PID > /dev/null 2>&1 - docker rm $MAILER_PID > /dev/null 2>&1 - echo "| mailer service stopped" - fi - if [ -f "$DB_DIR/$DB_NAME" ] ; then - rm "$DB_DIR/$DB_NAME" - echo "| test database deleted" - fi - if [ -d "$APP_DIR/frontend/tests/utils/.testhistory" ] ; then - rm -rf "$APP_DIR/frontend/tests/utils/.testhistory" - echo "| test data history removed" - fi - trap - SIGINT SIGTERM EXIT - echo "Cleanup done" - exit 0 + echo -e "\nCleaning up..." + if type deactivate >/dev/null 2>&1; then + deactivate + fi + if [ -n "$BACKEND_PID" ]; then + kill $BACKEND_PID >/dev/null 2>&1 + echo "| backend server stopped" + fi + if [ -n "$MAILER_PID" ]; then + docker stop $MAILER_PID >/dev/null 2>&1 + docker rm $MAILER_PID >/dev/null 2>&1 + echo "| mailer service stopped" + fi + if [ -f "$DB_DIR/$DB_NAME" ]; then + rm "$DB_DIR/$DB_NAME" + echo "| test database deleted" + fi + if [ -d "$APP_DIR/frontend/tests/utils/.testhistory" ]; then + rm -rf "$APP_DIR/frontend/tests/utils/.testhistory" + echo "| test data history removed" + fi + trap - SIGINT SIGTERM EXIT + echo "Cleanup done" + exit 0 } finish() { - echo "Test successfully completed!" - cleanup + echo "Test successfully completed!" + cleanup } trap cleanup SIGINT SIGTERM @@ -144,7 +143,7 @@ trap finish EXIT echo "Starting backend server..." unset POSTGRES_NAME POSTGRES_USER POSTGRES_PASSWORD export CISO_ASSISTANT_URL=http://localhost:4173 -export ALLOWED_HOSTS=localhost +export ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0 export DJANGO_DEBUG=True export DJANGO_SUPERUSER_EMAIL=admin@tests.com export DJANGO_SUPERUSER_PASSWORD=1234 @@ -159,50 +158,50 @@ cd $APP_DIR/backend/ python manage.py makemigrations python manage.py migrate python manage.py createsuperuser --noinput -if [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -v " ]] ; then - nohup python manage.py runserver $BACKEND_PORT > $APP_DIR/frontend/tests/utils/.testbackendoutput.out 2>&1 & - echo "You can view the backend server output at $APP_DIR/frontend/tests/utils/.testbackendoutput.out" +if [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -v " ]]; then + nohup python manage.py runserver $BACKEND_PORT >$APP_DIR/frontend/tests/utils/.testbackendoutput.out 2>&1 & + echo "You can view the backend server output at $APP_DIR/frontend/tests/utils/.testbackendoutput.out" else - nohup python manage.py runserver $BACKEND_PORT > /dev/null 2>&1 & + nohup python manage.py runserver $BACKEND_PORT >/dev/null 2>&1 & fi BACKEND_PID=$! echo "Test backend server started on port $BACKEND_PORT (PID: $BACKEND_PID)" -if [[ ! " ${SCRIPT_SHORT_ARGS[@]} " =~ " -m " ]] ; then - if command -v docker > /dev/null 2>&1 ; then - echo "Starting mailer service..." - MAILER_PID=$(docker run -d -p $MAILER_SMTP_SERVER_PORT:1025 -p $MAILER_WEB_SERVER_PORT:8025 mailhog/mailhog) - echo "Mailer service started on ports $MAILER_SMTP_SERVER_PORT/$MAILER_WEB_SERVER_PORT (Container ID: ${MAILER_PID:0:6})" - else - echo "Docker is not installed!" - echo "Please install Docker to use the isolated test mailer service or use -m to tell the tests to use an existing one." - exit 1 - fi +if [[ ! " ${SCRIPT_SHORT_ARGS[@]} " =~ " -m " ]]; then + if command -v docker >/dev/null 2>&1; then + echo "Starting mailer service..." + MAILER_PID=$(docker run -d -p $MAILER_SMTP_SERVER_PORT:1025 -p $MAILER_WEB_SERVER_PORT:8025 mailhog/mailhog) + echo "Mailer service started on ports $MAILER_SMTP_SERVER_PORT/$MAILER_WEB_SERVER_PORT (Container ID: ${MAILER_PID:0:6})" + else + echo "Docker is not installed!" + echo "Please install Docker to use the isolated test mailer service or use -m to tell the tests to use an existing one." + exit 1 + fi else - echo "Using an existing mailer service on ports $MAILER_SMTP_SERVER_PORT/$MAILER_WEB_SERVER_PORT" + echo "Using an existing mailer service on ports $MAILER_SMTP_SERVER_PORT/$MAILER_WEB_SERVER_PORT" fi echo "Starting playwright tests" export ORIGIN=http://localhost:4173 -export PUBLIC_BACKEND_API_URL=http://localhost:$BACKEND_PORT/api +export PUBLIC_BACKEND_API_URL=http://127.0.0.1:$BACKEND_PORT/api export MAILER_WEB_SERVER_PORT=$MAILER_WEB_SERVER_PORT cd $APP_DIR/frontend/ -if (( ${#TEST_PATHS[@]} == 0 )); then - echo "| running every functional test" +if ((${#TEST_PATHS[@]} == 0)); then + echo "| running every functional test" else - echo "| running tests: ${TEST_PATHS[@]}" + echo "| running tests: ${TEST_PATHS[@]}" fi -if (( ${#SCRIPT_LONG_ARGS[@]} == 0 )); then - echo "| without args" +if ((${#SCRIPT_LONG_ARGS[@]} == 0)); then + echo "| without args" else - echo "| with args: ${SCRIPT_LONG_ARGS[@]}" + echo "| with args: ${SCRIPT_LONG_ARGS[@]}" fi echo "==========================================================================================" -if [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -q " ]] ; then - npx playwright test ./tests/functional/"${TEST_PATHS[@]}" -x --project=chromium "${SCRIPT_LONG_ARGS[@]}" +if [[ " ${SCRIPT_SHORT_ARGS[@]} " =~ " -q " ]]; then + npx playwright test ./tests/functional/"${TEST_PATHS[@]}" -x --project=chromium "${SCRIPT_LONG_ARGS[@]}" else - npx playwright test ./tests/functional/"${TEST_PATHS[@]}" "${SCRIPT_LONG_ARGS[@]}" + npx playwright test ./tests/functional/"${TEST_PATHS[@]}" "${SCRIPT_LONG_ARGS[@]}" fi From 24937dcc1ef30dad3686925b10d841bca765a7d9 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 16:19:06 +0200 Subject: [PATCH 43/86] Pass library import functional test --- frontend/src/routes/(app)/libraries/+page.server.ts | 8 +++++++- frontend/tests/utils/test-utils.ts | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index b788e0c40..f45e081d4 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -61,7 +61,13 @@ export const load = (async ({ fetch }) => { }; }; - const storedLibrariesTable = makeLibrariesTable(storedLibraries, 'stored-libraries'); + const unimportedStoredLibraries = storedLibraries.filter((lib) => lib.is_imported === false); + const storedLibrariesTable = { + head: makeHeadData('stored-libraries'), + meta: { urlmodel: 'stored-libraries', ...unimportedStoredLibraries }, + body: tableSourceMapper(unimportedStoredLibraries, listViewFields['stored-libraries'].body) + }; + const loadedLibrariesTable = makeLibrariesTable(loadedLibraries, 'loaded-libraries'); const schema = z.object({ id: z.string() }); diff --git a/frontend/tests/utils/test-utils.ts b/frontend/tests/utils/test-utils.ts index 1a15f10ca..d9dfa8232 100644 --- a/frontend/tests/utils/test-utils.ts +++ b/frontend/tests/utils/test-utils.ts @@ -142,7 +142,7 @@ export const test = base.extend({ }, librariesPage: async ({ page }, use) => { - const lPage = new PageContent(page, '/loaded-ibraries', 'Loaded Libraries'); + const lPage = new PageContent(page, '/libraries', 'Libraries'); await use(lPage); }, @@ -621,7 +621,7 @@ export function getUniqueValue(value: string): string { } export function replaceValues(obj: any, searchValue: string, replaceValue: string) { - for (let key in obj) { + for (const key in obj) { if (typeof obj[key] === 'object') { replaceValues(obj[key], searchValue, replaceValue); } else if (typeof obj[key] === 'string') { From 4ae651c6f7e010fb2ab7bb9c3b6cd131dae4d601 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 18:48:03 +0200 Subject: [PATCH 44/86] Fix language tag functional test --- frontend/tests/functional/nav.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/tests/functional/nav.test.ts b/frontend/tests/functional/nav.test.ts index 4ed65d0fc..669481ce8 100644 --- a/frontend/tests/functional/nav.test.ts +++ b/frontend/tests/functional/nav.test.ts @@ -60,10 +60,13 @@ test('sidebar navigation tests', async ({ logedPage, analyticsPage, sideBar, pag await test.step('translation panel is working properly', async () => { await analyticsPage.goto(); - for await (const languageTag of availableLanguageTags.toSorted((a, b) => { - // English is always tested at last to ensure a clean state at the end - return a === 'en' ? 1 : b === 'en' ? -1 : a.localeCompare(b); - })) { + const locales = [...availableLanguageTags]; + const index = locales.indexOf('en'); + if (index !== -1) { + locales.splice(index, 1); + locales.push('en'); + } + for (const languageTag of locales) { await sideBar.moreButton.click(); await expect(sideBar.morePanel).not.toHaveAttribute('inert'); await expect(sideBar.languageSelect).toBeVisible(); From 586aea354e624e77f309eca8d1333aa8de242a5b Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Fri, 3 May 2024 18:48:13 +0200 Subject: [PATCH 45/86] Fix library import util --- frontend/tests/utils/page-content.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/tests/utils/page-content.ts b/frontend/tests/utils/page-content.ts index 4a50f1ace..f598cae8b 100644 --- a/frontend/tests/utils/page-content.ts +++ b/frontend/tests/utils/page-content.ts @@ -77,6 +77,13 @@ export class PageContent extends BasePage { return; } } + // If the library is not visible, it might have already been imported + if (await this.getRow(ref).isHidden()) { + await this.tab('Imported libraries').click(); + expect(this.tab('Imported libraries').getAttribute('aria-selected')).toBeTruthy(); + expect(this.getRow(ref)).toBeVisible(); + return; + } await this.importItemButton(ref, language === 'any' ? undefined : language).click(); await this.isToastVisible(`The library object has been successfully imported.+`, undefined, { timeout: 15000 From a84e3b015632e0aabf22fb5b084476a6b7036644 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Sun, 12 May 2024 16:39:32 +0200 Subject: [PATCH 46/86] Fix typing in crud.ts --- frontend/src/lib/utils/crud.ts | 44 +++++++++------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/frontend/src/lib/utils/crud.ts b/frontend/src/lib/utils/crud.ts index a932e0f64..2dec58a48 100644 --- a/frontend/src/lib/utils/crud.ts +++ b/frontend/src/lib/utils/crud.ts @@ -10,7 +10,9 @@ type GetOptionsParams = { suggestions?: any[]; label?: string; value?: string; - extra_fields: (string[] | string)[]; + extra_fields: string[]; + self?: Record; + selfSelect?: boolean; }; export function checkConstraints(constraints: { [key: string]: any }, foreignKeys: any) { @@ -22,7 +24,7 @@ export function checkConstraints(constraints: { [key: string]: any }, foreignKey return emptyConstraintsList; } -function getValue(object: { [key: string]: any }, keys: string[]) { +function getValue(object: { [key: string]: any }, keys: string | string[]) { if (typeof keys === 'string') { return object[keys]; } @@ -54,7 +56,7 @@ export const getOptions = ({ label: extra_fields.length > 0 ? extra_fields - .map((fields) => getValue(object, fields)) + .map((field) => getValue(object, field)) .map((string) => `${string}`) .join('/') + '/' + @@ -109,6 +111,8 @@ interface SelectField { export interface ModelMapEntry { name: string; + localName: string; + localNamePlural: string; verboseName: string; verboseNamePlural?: string; urlModel?: urlModel; @@ -117,7 +121,6 @@ export interface ModelMapEntry { reverseForeignKeyFields?: ForeignKeyField[]; selectFields?: SelectField[]; filters?: SelectField[]; - [key: string]: any; } type ModelMap = { @@ -129,7 +132,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'folder', localName: 'domain', localNamePlural: 'domains', - localFrGender: 'm', verboseName: 'Domain', verboseNamePlural: 'Domains', foreignKeyFields: [ @@ -141,7 +143,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'project', localName: 'project', localNamePlural: 'projects', - localFrGender: 'm', verboseName: 'Project', verboseNamePlural: 'Projects', foreignKeyFields: [{ field: 'folder', urlModel: 'folders', urlParams: 'content_type=DO' }], @@ -156,7 +157,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'riskmatrix', localName: 'riskMatrix', localNamePlural: 'riskMatrices', - localFrGender: 'f', verboseName: 'Risk matrix', verboseNamePlural: 'Risk matrices', foreignKeyFields: [{ field: 'folder', urlModel: 'folders' }] @@ -165,7 +165,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'riskassessment', localName: 'riskAssessment', localNamePlural: 'riskAssessments', - localFrGender: 'f', verboseName: 'Risk assessment', verboseNamePlural: 'Risk assessments', foreignKeyFields: [ @@ -180,11 +179,9 @@ export const URL_MODEL_MAP: ModelMap = { filters: [{ field: 'project' }, { field: 'auditor' }, { field: 'status' }] }, threats: { - ref_id: 'ref_id', name: 'threat', localName: 'threat', localNamePlural: 'threats', - localFrGender: 'f', verboseName: 'Threat', verboseNamePlural: 'Threats', foreignKeyFields: [{ field: 'folder', urlModel: 'folders' }] @@ -193,7 +190,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'riskscenario', localName: 'riskScenario', localNamePlural: 'riskScenarios', - localFrGender: 'm', verboseName: 'Risk scenario', verboseNamePlural: 'Risk scenarios', foreignKeyFields: [ @@ -205,14 +201,12 @@ export const URL_MODEL_MAP: ModelMap = { { field: 'risk_matrix', urlModel: 'risk-matrices' }, { field: 'auditor', urlModel: 'users' } ], - filters: [{ field: 'threats' }, { field: 'risk_assessment' }], - search: false + filters: [{ field: 'threats' }, { field: 'risk_assessment' }] }, 'applied-controls': { name: 'appliedcontrol', localName: 'appliedControl', localNamePlural: 'appliedControls', - localFrGender: 'f', verboseName: 'Applied control', verboseNamePlural: 'Applied controls', detailViewFields: [ @@ -248,7 +242,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'appliedcontrol', localName: 'policy', localNamePlural: 'policies', - localFrGender: 'f', verboseName: 'Policy', verboseNamePlural: 'Policies', foreignKeyFields: [ @@ -268,7 +261,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'riskacceptance', localName: 'riskAcceptance', localNamePlural: 'riskAcceptances', - localFrGender: 'f', verboseName: 'Risk acceptance', verboseNamePlural: 'Risk acceptances', foreignKeyFields: [ @@ -283,11 +275,9 @@ export const URL_MODEL_MAP: ModelMap = { filters: [{ field: 'risk_scenarios' }, { field: 'folder' }, { field: 'approver' }] }, 'reference-controls': { - ref_id: 'ref_id', name: 'referencecontrol', localName: 'referenceControl', localNamePlural: 'referenceControls', - localFrGender: 'f', verboseName: 'Reference control', verboseNamePlural: 'Reference controls', foreignKeyFields: [{ field: 'folder', urlModel: 'folders' }], @@ -298,7 +288,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'asset', localName: 'asset', localNamePlural: 'assets', - localFrGender: 'm', verboseName: 'Asset', verboseNamePlural: 'Assets', foreignKeyFields: [ @@ -312,7 +301,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'user', localName: 'user', localNamePlural: 'users', - localFrGender: 'm', verboseName: 'User', verboseNamePlural: 'Users', foreignKeyFields: [{ field: 'user_groups', urlModel: 'user-groups' }], @@ -322,7 +310,7 @@ export const URL_MODEL_MAP: ModelMap = { name: 'usergroup', localName: 'userGroup', localNamePlural: 'userGroups', - localFrGender: 'm', + verboseName: 'User group', verboseNamePlural: 'User groups', foreignKeyFields: [{ field: 'folder', urlModel: 'folders' }], @@ -332,18 +320,15 @@ export const URL_MODEL_MAP: ModelMap = { name: 'roleassignment', localName: 'roleAssignment', localNamePlural: 'roleAssignments', - localFrGender: 'f', verboseName: 'Role assignment', verboseNamePlural: 'Role assignments', foreignKeyFields: [], filters: [] }, frameworks: { - ref_id: 'ref_id', name: 'framework', localName: 'framework', localNamePlural: 'frameworks', - localFrGender: 'm', verboseName: 'Framework', verboseNamePlural: 'Frameworks', foreignKeyFields: [ @@ -357,7 +342,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'evidence', localName: 'evidence', localNamePlural: 'evidences', - localFrGender: 'f', verboseName: 'Evidence', verboseNamePlural: 'Evidences', foreignKeyFields: [ @@ -370,7 +354,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'complianceassessment', localName: 'complianceAssessment', localNamePlural: 'complianceAssessments', - localFrGender: 'f', verboseName: 'Compliance assessment', verboseNamePlural: 'Compliance assessments', foreignKeyFields: [ @@ -383,11 +366,9 @@ export const URL_MODEL_MAP: ModelMap = { filters: [{ field: 'status' }] }, requirements: { - ref_id: 'ref_id', name: 'requirement', localName: 'requirement', localNamePlural: 'requirements', - localFrGender: 'f', verboseName: 'Requirement', verboseNamePlural: 'Requirements' }, @@ -395,7 +376,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'requirementassessment', localName: 'requirementAssessment', localNamePlural: 'requirementAssessments', - localFrGender: 'f', verboseName: 'Requirement assessment', verboseNamePlural: 'Requirement assessments', selectFields: [{ field: 'status' }], @@ -409,7 +389,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'storedlibrary', localName: 'imported library', localNamePlural: 'imported libraries', - localFrGender: 'f', verboseName: 'Imported Library', verboseNamePlural: 'Imported Libraries' }, @@ -417,7 +396,6 @@ export const URL_MODEL_MAP: ModelMap = { name: 'loadedlibrary', localName: 'imported library', localNamePlural: 'imported libraries', - localFrGender: 'f', verboseName: 'Imported Library', verboseNamePlural: 'Imported Libraries' } @@ -562,9 +540,9 @@ export const urlParamModelSelectFields = (model: string): SelectField[] => { return URL_MODEL_MAP[model]?.selectFields || []; }; -export const getModelInfo = (model: string): ModelMapEntry => { +export const getModelInfo = (model: urlModel): ModelMapEntry => { const map = URL_MODEL_MAP[model] || {}; - map['urlModel' as urlModel] = model; + map['urlModel'] = model; return map; }; From f779700750cedc4660cab5296c12e2dda38e1a96 Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Sun, 12 May 2024 16:41:08 +0200 Subject: [PATCH 47/86] Remove 'search' attribute from model definitions It is not a useful abstraction for now. Setting the `search` prop when instantiating the ModelTable component is a more straightforward, less error-prone approach. --- frontend/src/routes/(app)/[model=urlmodel]/+page.svelte | 4 +--- .../src/routes/(app)/risk-assessments/[id=uuid]/+page.svelte | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/routes/(app)/[model=urlmodel]/+page.svelte b/frontend/src/routes/(app)/[model=urlmodel]/+page.svelte index 0d9f016b6..c9710c085 100644 --- a/frontend/src/routes/(app)/[model=urlmodel]/+page.svelte +++ b/frontend/src/routes/(app)/[model=urlmodel]/+page.svelte @@ -44,13 +44,11 @@ } modalStore.trigger(modal); } - - $: search = data.model?.search !== false; {#if data.table}
- +
{#if !['risk-matrices', 'frameworks', 'user-groups', 'role-assignments'].includes(data.URLModel)} {:catch err} From c02b536dba52c4a3de12eed1738f59d0d8679514 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Tue, 14 May 2024 17:45:46 +0200 Subject: [PATCH 63/86] Fix non-clickable libraries link in breadcrumb --- frontend/src/lib/utils/table.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/lib/utils/table.ts b/frontend/src/lib/utils/table.ts index 0428be0b4..9ecc38625 100644 --- a/frontend/src/lib/utils/table.ts +++ b/frontend/src/lib/utils/table.ts @@ -93,6 +93,10 @@ export const listViewFields = { body: ['ref_id', 'name', 'description', 'framework'], meta: ['id', 'urn'] }, + 'libraries': { + head: ['ref', 'name', 'description', 'language', 'overview'], + body: ['ref_id', 'name', 'description', 'locale', 'overview'] + }, 'stored-libraries': { head: ['ref', 'name', 'description', 'language', 'overview'], body: ['ref_id', 'name', 'description', 'locale', 'overview'] From 61a5b20d585a1202c76c17c4dfd8ec58af846d75 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Tue, 14 May 2024 17:50:02 +0200 Subject: [PATCH 64/86] Improve library delete message --- frontend/messages/en.json | 1 + frontend/messages/fr.json | 1 + frontend/messages/pt.json | 1 + frontend/src/routes/(app)/libraries/+page.server.ts | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 3470d1cf6..be90cc733 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -459,6 +459,7 @@ "successfullyUpdatedObject": "The {object} object: {name} has been successfully updated", "successfullySavedObject": "The {object} object has been successfully saved", "successfullyDeletedObject": "The {object} object has been successfully deleted", + "successfullyDeletedLibrary": "The library has been successfully deleted", "successfullyCreatedUser": "User successfully created. An email was sent to set the password.", "successfullyUpdatedUser": "The user: {email} has been successfully updated", "successfullyValidatedObject": "The {object} object has been successfully validated", diff --git a/frontend/messages/fr.json b/frontend/messages/fr.json index 03f6cf092..be6a61b44 100644 --- a/frontend/messages/fr.json +++ b/frontend/messages/fr.json @@ -459,6 +459,7 @@ "successfullyUpdatedObject": "L'objet {object}: {name} a été mis à jour avec succès", "successfullySavedObject": "L'objet {object} a été enregistré avec succès", "successfullyDeletedObject": "L'objet {object} a été supprimé avec succès", + "successfullyDeletedLibrary": "La librairie a été supprimée avec succès", "successfullyCreatedUser": "Utilisateur créé avec succès. Un email a été envoyé pour définir le mot de passe.", "successfullyUpdatedUser": "L'utilisateur: {email} a été mis à jour avec succès", "successfullyValidatedObject": "L'objet {object} a été validé avec succès", diff --git a/frontend/messages/pt.json b/frontend/messages/pt.json index 009549881..6739c88eb 100644 --- a/frontend/messages/pt.json +++ b/frontend/messages/pt.json @@ -459,6 +459,7 @@ "successfullyUpdatedObject": "O objeto {object}: {name} foi atualizado com sucesso", "successfullySavedObject": "O objeto {object} foi salvo com sucesso", "successfullyDeletedObject": "O objeto {object} foi excluído com sucesso", + "successfullyDeletedLibrary": "A biblioteca foi excluída com sucesso", "successfullyCreatedUser": "Usuário criado com sucesso. Um e-mail foi enviado para definir a senha.", "successfullyUpdatedUser": "O usuário: {email} foi atualizado com sucesso", "successfullyValidatedObject": "O objeto {object} foi validado com sucesso", diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index 4d5551dbf..653301b46 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -140,7 +140,7 @@ export const actions: Actions = { return fail(400, { form: deleteForm }); } setFlash( - { type: 'success', message: m.successfullyDeletedObject({ object: 'library' }) }, + { type: 'success', message: m.successfullyDeletedLibrary() }, event ); } From c5ad388610cf39dd9c42f74b34e57d92a8d1db8f Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Tue, 14 May 2024 19:46:15 +0200 Subject: [PATCH 65/86] Fix migration issue by renaming library to loadedlibrary --- ...lter_appliedcontrol_updated_at_and_more.py | 194 ++++++++---------- backend/core/models.py | 2 +- 2 files changed, 90 insertions(+), 106 deletions(-) diff --git a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py index 4d944476e..217dcd722 100644 --- a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py @@ -1,10 +1,70 @@ # Generated by Django 5.0.4 on 2024-05-03 12:41 +# loadedlibrary updates added manually import django.db.models.deletion import iam.models import uuid from django.db import migrations, models +BUILTIN_LIBRARY_URNS = set([ + "urn:intuitem:risk:library:nis2-directive", + "urn:intuitem:risk:library:cmmc-2.0", + "urn:intuitem:risk:library:pcidss-4_0", + "urn:intuitem:risk:library:nist-ssdf-1.1", + "urn:intuitem:risk:library:rgs-v2.0", + "urn:intuitem:risk:library:doc-pol", + "urn:intuitem:risk:library:dora", + "urn:intuitem:risk:library:3cf-v2", + "urn:intuitem:risk:library:owasp-top-10-web", + "urn:intuitem:risk:library:hds-v2023-a", + "urn:ackwa:risk:library:pgssi-s-1.0", + "urn:intuitem:risk:library:gdpr-checklist", + "urn:intuitem:risk:library:anssi-guide-hygiene", + "urn:intuitem:risk:library:iso27001-2022", + "urn:intuitem:risk:library:mitre-attack-v14", + "urn:protocolpaladin:risk:library:matrice-des-risques-critiques-5x5", + "urn:intuitem:risk:library:risk-matrix-3x3-mult", + "urn:intuitem:risk:library:fedramp-rev5", + "urn:intuitem:risk:library:nist-csf-1.1", + "urn:intuitem:risk:library:critical_risk_matrix_3x3", + "urn:intuitem:risk:library:nist-800-171-rev2", + "urn:intuitem:risk:library:ecc-1", + "urn:intuitem:risk:library:secnumcloud-3.2-annexe-2", + "urn:intuitem:risk:library:secnumcloud-3.2", + "urn:intuitem:risk:library:3cf-ed1-v1", + "urn:intuitem:risk:library:fadp", + "urn:intuitem:risk:library:tisax-v6.0.2", + "urn:intuitem:risk:library:owasp-asvs-4.0.3", + "urn:protocolpaladin:risk:library:anssi-recommandations-configuration-systeme-gnu-linux", + "urn:intuitem:risk:library:lpm-oiv-2019", + "urn:intuitem:risk:library:aircyber-v1.5.2", + "urn:intuitem:risk:library:nist-ai-rmf-1.0", + "urn:intuitem:risk:library:dfs-500-2023-11", + "urn:intuitem:risk:library:nist-csf-2.0", + "urn:intuitem:risk:library:anssi-nis-rules", + "urn:intuitem:risk:library:risk-matrix-5x5-sensitive", + "urn:intuitem:risk:library:iso27001-2022-fr", + "urn:intuitem:risk:library:pspf", + "urn:intuitem:risk:library:nist-privacy-1.0", + "urn:intuitem:risk:library:ccb-cff-2023-03-01", + "urn:intuitem:risk:library:cra-proposal-annexes", + "urn:ackwa:risk:library:risk-matrix-4x4-pgssi-s-1.0", + "urn:intuitem:risk:library:essential-eight", + "urn:intuitem:risk:library:nist-sp-800-66-rev2", + "urn:intuitem:risk:library:critical_risk_matrix_5x5", + "urn:protocolpaladin:risk:library:matrice-des-risques-critiques-3x3", + "urn:intuitem:risk:library:nist-sp-800-53-rev5", + "urn:intuitem:risk:library:tiber-eu-2018", + "urn:intuitem:risk:library:anssi-genai-security-recommendations-1.0", + "urn:intuitem:risk:library:soc2-2017" +]) + +def adapt_libraries(apps, schema_editor): + LoadedLibrary = apps.get_model("core", "LoadedLibrary") + for library in LoadedLibrary.objects.all() : + library.builtin = library.urn in BUILTIN_LIBRARY_URNS # There is no perfect way to verify is a loaded custom library is builtin or not + # There is no way to generate the objects_meta dictionary without reading all files from ./backend/library/libraries, but we can generate the missing objects_meta values at the same time we generate the StoredLibrary objects. + library.save() class Migration(migrations.Migration): dependencies = [ @@ -83,108 +143,34 @@ class Migration(migrations.Migration): name="updated_at", field=models.DateTimeField(auto_now=True, verbose_name="Updated at"), ), - migrations.CreateModel( - name="LoadedLibrary", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, - editable=False, - primary_key=True, - serialize=False, - ), - ), - ( - "created_at", - models.DateTimeField(auto_now_add=True, verbose_name="Created at"), - ), - ( - "updated_at", - models.DateTimeField(auto_now=True, verbose_name="Updated at"), - ), - ( - "is_published", - models.BooleanField(default=False, verbose_name="published"), - ), - ( - "ref_id", - models.CharField( - blank=True, - max_length=100, - null=True, - verbose_name="Reference ID", - ), - ), - ( - "locale", - models.CharField( - default="en", max_length=100, verbose_name="Locale" - ), - ), - ( - "default_locale", - models.BooleanField(default=True, verbose_name="Default locale"), - ), - ( - "provider", - models.CharField( - blank=True, max_length=200, null=True, verbose_name="Provider" - ), - ), - ( - "name", - models.CharField(max_length=200, null=True, verbose_name="Name"), - ), - ( - "description", - models.TextField(blank=True, null=True, verbose_name="Description"), - ), - ( - "annotation", - models.TextField(blank=True, null=True, verbose_name="Annotation"), - ), - ( - "urn", - models.CharField( - blank=True, max_length=100, null=True, verbose_name="URN" - ), - ), - ( - "copyright", - models.CharField( - blank=True, max_length=4096, null=True, verbose_name="Copyright" - ), - ), - ("version", models.IntegerField(verbose_name="Version")), - ( - "packager", - models.CharField( - blank=True, - help_text="Packager of the library", - max_length=100, - null=True, - verbose_name="Packager", - ), - ), - ("builtin", models.BooleanField(default=False)), - ("objects_meta", models.JSONField()), - ( - "dependencies", - models.ManyToManyField( - blank=True, to="core.loadedlibrary", verbose_name="Dependencies" - ), - ), - ( - "folder", - models.ForeignKey( - default=iam.models.Folder.get_root_folder, - on_delete=django.db.models.deletion.CASCADE, - related_name="%(class)s_folder", - to="iam.folder", - ), - ), - ], + migrations.RenameModel("Library", "LoadedLibrary"), + migrations.AlterField( + model_name="loadedlibrary", + name="provider", + field=models.CharField(blank=True, max_length=200, null=True, verbose_name="Provider") + ), + migrations.AlterField( + model_name="loadedlibrary", + name="urn", + field=models.CharField(blank=True, max_length=100, null=True, verbose_name="URN") + ), + migrations.AlterField( + model_name="loadedlibrary", + name="dependencies", + field=models.ManyToManyField(blank=True, to="core.loadedlibrary", verbose_name="Dependencies"), + ), + migrations.AddField( + model_name="loadedlibrary", + name="builtin", + field=models.BooleanField(default=False) + ), + migrations.AddField( + model_name="loadedlibrary", + name="objects_meta", + field=models.JSONField(default=dict) + ), + migrations.AlterModelOptions( + name="loadedlibrary", options={ "abstract": False, "unique_together": {("urn", "locale", "version")}, @@ -340,7 +326,5 @@ class Migration(migrations.Migration): "unique_together": {("urn", "locale", "version")}, }, ), - migrations.DeleteModel( - name="Library", - ), + migrations.RunPython(adapt_libraries), ] diff --git a/backend/core/models.py b/backend/core/models.py index b19276420..680d1a29d 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -103,7 +103,7 @@ class Meta: verbose_name=_("Packager"), ) builtin = models.BooleanField(default=False) - objects_meta = models.JSONField() + objects_meta = models.JSONField(default=dict) dependencies = models.JSONField( null=True ) # models.CharField(blank=False,null=True,max_length=16384) From 84578be68fe9d75da16fb3b84d0226ebdba1a75c Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Tue, 14 May 2024 19:48:09 +0200 Subject: [PATCH 66/86] Fix the delete button not showing for loaded builtin libarries --- .../src/lib/components/ModelTable/ModelTable.svelte | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index 60fda2a6f..9e4c49e91 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -132,9 +132,11 @@ $: model = source.meta?.urlmodel ? URL_MODEL_MAP[source.meta.urlmodel] : URL_MODEL_MAP[URLModel]; $: source, handler.setRows(data); - $: preventDelete = (row: TableSource) => - row.meta.builtin || - (Object.hasOwn(row.meta, 'reference_count') && row.meta.reference_count > 0); + + const actionsURLModel = source.meta.urlmodel ?? URLModel + const preventDelete = (row: TableSource) => + (row.meta.builtin && actionsURLModel !== 'loaded-libraries') || + (Object.hasOwn(row.meta, 'reference_count') && row.meta.reference_count > 0)
@@ -267,7 +269,7 @@ {@const actionsComponent = field_component_map['actions']} {@const actionsURLModel = source.meta.urlmodel ?? URLModel} Date: Tue, 14 May 2024 20:47:06 +0200 Subject: [PATCH 67/86] Formatter --- ...lter_appliedcontrol_updated_at_and_more.py | 130 ++++++++++-------- .../components/ModelTable/ModelTable.svelte | 4 +- frontend/src/lib/utils/table.ts | 2 +- .../routes/(app)/libraries/+page.server.ts | 5 +- 4 files changed, 75 insertions(+), 66 deletions(-) diff --git a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py index 217dcd722..3c3a4ed38 100644 --- a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py @@ -6,66 +6,72 @@ import uuid from django.db import migrations, models -BUILTIN_LIBRARY_URNS = set([ - "urn:intuitem:risk:library:nis2-directive", - "urn:intuitem:risk:library:cmmc-2.0", - "urn:intuitem:risk:library:pcidss-4_0", - "urn:intuitem:risk:library:nist-ssdf-1.1", - "urn:intuitem:risk:library:rgs-v2.0", - "urn:intuitem:risk:library:doc-pol", - "urn:intuitem:risk:library:dora", - "urn:intuitem:risk:library:3cf-v2", - "urn:intuitem:risk:library:owasp-top-10-web", - "urn:intuitem:risk:library:hds-v2023-a", - "urn:ackwa:risk:library:pgssi-s-1.0", - "urn:intuitem:risk:library:gdpr-checklist", - "urn:intuitem:risk:library:anssi-guide-hygiene", - "urn:intuitem:risk:library:iso27001-2022", - "urn:intuitem:risk:library:mitre-attack-v14", - "urn:protocolpaladin:risk:library:matrice-des-risques-critiques-5x5", - "urn:intuitem:risk:library:risk-matrix-3x3-mult", - "urn:intuitem:risk:library:fedramp-rev5", - "urn:intuitem:risk:library:nist-csf-1.1", - "urn:intuitem:risk:library:critical_risk_matrix_3x3", - "urn:intuitem:risk:library:nist-800-171-rev2", - "urn:intuitem:risk:library:ecc-1", - "urn:intuitem:risk:library:secnumcloud-3.2-annexe-2", - "urn:intuitem:risk:library:secnumcloud-3.2", - "urn:intuitem:risk:library:3cf-ed1-v1", - "urn:intuitem:risk:library:fadp", - "urn:intuitem:risk:library:tisax-v6.0.2", - "urn:intuitem:risk:library:owasp-asvs-4.0.3", - "urn:protocolpaladin:risk:library:anssi-recommandations-configuration-systeme-gnu-linux", - "urn:intuitem:risk:library:lpm-oiv-2019", - "urn:intuitem:risk:library:aircyber-v1.5.2", - "urn:intuitem:risk:library:nist-ai-rmf-1.0", - "urn:intuitem:risk:library:dfs-500-2023-11", - "urn:intuitem:risk:library:nist-csf-2.0", - "urn:intuitem:risk:library:anssi-nis-rules", - "urn:intuitem:risk:library:risk-matrix-5x5-sensitive", - "urn:intuitem:risk:library:iso27001-2022-fr", - "urn:intuitem:risk:library:pspf", - "urn:intuitem:risk:library:nist-privacy-1.0", - "urn:intuitem:risk:library:ccb-cff-2023-03-01", - "urn:intuitem:risk:library:cra-proposal-annexes", - "urn:ackwa:risk:library:risk-matrix-4x4-pgssi-s-1.0", - "urn:intuitem:risk:library:essential-eight", - "urn:intuitem:risk:library:nist-sp-800-66-rev2", - "urn:intuitem:risk:library:critical_risk_matrix_5x5", - "urn:protocolpaladin:risk:library:matrice-des-risques-critiques-3x3", - "urn:intuitem:risk:library:nist-sp-800-53-rev5", - "urn:intuitem:risk:library:tiber-eu-2018", - "urn:intuitem:risk:library:anssi-genai-security-recommendations-1.0", - "urn:intuitem:risk:library:soc2-2017" -]) +BUILTIN_LIBRARY_URNS = set( + [ + "urn:intuitem:risk:library:nis2-directive", + "urn:intuitem:risk:library:cmmc-2.0", + "urn:intuitem:risk:library:pcidss-4_0", + "urn:intuitem:risk:library:nist-ssdf-1.1", + "urn:intuitem:risk:library:rgs-v2.0", + "urn:intuitem:risk:library:doc-pol", + "urn:intuitem:risk:library:dora", + "urn:intuitem:risk:library:3cf-v2", + "urn:intuitem:risk:library:owasp-top-10-web", + "urn:intuitem:risk:library:hds-v2023-a", + "urn:ackwa:risk:library:pgssi-s-1.0", + "urn:intuitem:risk:library:gdpr-checklist", + "urn:intuitem:risk:library:anssi-guide-hygiene", + "urn:intuitem:risk:library:iso27001-2022", + "urn:intuitem:risk:library:mitre-attack-v14", + "urn:protocolpaladin:risk:library:matrice-des-risques-critiques-5x5", + "urn:intuitem:risk:library:risk-matrix-3x3-mult", + "urn:intuitem:risk:library:fedramp-rev5", + "urn:intuitem:risk:library:nist-csf-1.1", + "urn:intuitem:risk:library:critical_risk_matrix_3x3", + "urn:intuitem:risk:library:nist-800-171-rev2", + "urn:intuitem:risk:library:ecc-1", + "urn:intuitem:risk:library:secnumcloud-3.2-annexe-2", + "urn:intuitem:risk:library:secnumcloud-3.2", + "urn:intuitem:risk:library:3cf-ed1-v1", + "urn:intuitem:risk:library:fadp", + "urn:intuitem:risk:library:tisax-v6.0.2", + "urn:intuitem:risk:library:owasp-asvs-4.0.3", + "urn:protocolpaladin:risk:library:anssi-recommandations-configuration-systeme-gnu-linux", + "urn:intuitem:risk:library:lpm-oiv-2019", + "urn:intuitem:risk:library:aircyber-v1.5.2", + "urn:intuitem:risk:library:nist-ai-rmf-1.0", + "urn:intuitem:risk:library:dfs-500-2023-11", + "urn:intuitem:risk:library:nist-csf-2.0", + "urn:intuitem:risk:library:anssi-nis-rules", + "urn:intuitem:risk:library:risk-matrix-5x5-sensitive", + "urn:intuitem:risk:library:iso27001-2022-fr", + "urn:intuitem:risk:library:pspf", + "urn:intuitem:risk:library:nist-privacy-1.0", + "urn:intuitem:risk:library:ccb-cff-2023-03-01", + "urn:intuitem:risk:library:cra-proposal-annexes", + "urn:ackwa:risk:library:risk-matrix-4x4-pgssi-s-1.0", + "urn:intuitem:risk:library:essential-eight", + "urn:intuitem:risk:library:nist-sp-800-66-rev2", + "urn:intuitem:risk:library:critical_risk_matrix_5x5", + "urn:protocolpaladin:risk:library:matrice-des-risques-critiques-3x3", + "urn:intuitem:risk:library:nist-sp-800-53-rev5", + "urn:intuitem:risk:library:tiber-eu-2018", + "urn:intuitem:risk:library:anssi-genai-security-recommendations-1.0", + "urn:intuitem:risk:library:soc2-2017", + ] +) + def adapt_libraries(apps, schema_editor): LoadedLibrary = apps.get_model("core", "LoadedLibrary") - for library in LoadedLibrary.objects.all() : - library.builtin = library.urn in BUILTIN_LIBRARY_URNS # There is no perfect way to verify is a loaded custom library is builtin or not + for library in LoadedLibrary.objects.all(): + library.builtin = ( + library.urn in BUILTIN_LIBRARY_URNS + ) # There is no perfect way to verify is a loaded custom library is builtin or not # There is no way to generate the objects_meta dictionary without reading all files from ./backend/library/libraries, but we can generate the missing objects_meta values at the same time we generate the StoredLibrary objects. library.save() + class Migration(migrations.Migration): dependencies = [ ("core", "0011_auto_20240501_1342"), @@ -147,27 +153,33 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="loadedlibrary", name="provider", - field=models.CharField(blank=True, max_length=200, null=True, verbose_name="Provider") + field=models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider" + ), ), migrations.AlterField( model_name="loadedlibrary", name="urn", - field=models.CharField(blank=True, max_length=100, null=True, verbose_name="URN") + field=models.CharField( + blank=True, max_length=100, null=True, verbose_name="URN" + ), ), migrations.AlterField( model_name="loadedlibrary", name="dependencies", - field=models.ManyToManyField(blank=True, to="core.loadedlibrary", verbose_name="Dependencies"), + field=models.ManyToManyField( + blank=True, to="core.loadedlibrary", verbose_name="Dependencies" + ), ), migrations.AddField( model_name="loadedlibrary", name="builtin", - field=models.BooleanField(default=False) + field=models.BooleanField(default=False), ), migrations.AddField( model_name="loadedlibrary", name="objects_meta", - field=models.JSONField(default=dict) + field=models.JSONField(default=dict), ), migrations.AlterModelOptions( name="loadedlibrary", diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index 9e4c49e91..f92d8caaf 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -133,10 +133,10 @@ $: model = source.meta?.urlmodel ? URL_MODEL_MAP[source.meta.urlmodel] : URL_MODEL_MAP[URLModel]; $: source, handler.setRows(data); - const actionsURLModel = source.meta.urlmodel ?? URLModel + const actionsURLModel = source.meta.urlmodel ?? URLModel; const preventDelete = (row: TableSource) => (row.meta.builtin && actionsURLModel !== 'loaded-libraries') || - (Object.hasOwn(row.meta, 'reference_count') && row.meta.reference_count > 0) + (Object.hasOwn(row.meta, 'reference_count') && row.meta.reference_count > 0);
diff --git a/frontend/src/lib/utils/table.ts b/frontend/src/lib/utils/table.ts index 9ecc38625..23d86e726 100644 --- a/frontend/src/lib/utils/table.ts +++ b/frontend/src/lib/utils/table.ts @@ -93,7 +93,7 @@ export const listViewFields = { body: ['ref_id', 'name', 'description', 'framework'], meta: ['id', 'urn'] }, - 'libraries': { + libraries: { head: ['ref', 'name', 'description', 'language', 'overview'], body: ['ref_id', 'name', 'description', 'locale', 'overview'] }, diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index 653301b46..cd9c79a5b 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -139,10 +139,7 @@ export const actions: Actions = { } return fail(400, { form: deleteForm }); } - setFlash( - { type: 'success', message: m.successfullyDeletedLibrary() }, - event - ); + setFlash({ type: 'success', message: m.successfullyDeletedLibrary() }, event); } return { deleteForm }; } From debaa8f5afcbac422bfd45dea9a472a7dc396032 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 11:03:07 +0200 Subject: [PATCH 68/86] Set the is_loaded attribute at the initialization of stored libraries --- backend/core/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/core/models.py b/backend/core/models.py index 680d1a29d..b76629b47 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -148,6 +148,11 @@ def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None" urn = library_data["urn"] locale = library_data.get("locale", "en") version = int(library_data["version"]) + is_loaded = LoadedLibrary.objects.filter( + urn=urn, + locale=locale, + version=version + ).exists() library_matches = [*StoredLibrary.objects.filter(urn=urn, locale=locale)] if any(library.version >= version for library in library_matches): @@ -186,6 +191,7 @@ def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None" packager=library_data.get("packager"), objects_meta=objects_meta, dependencies=dependencies, + is_loaded=is_loaded, builtin=library_data.get( "builtin", False ), # We have to add a "builtin: true" line to every builtin library file. From 42731aa4376c9970be72d3f766abe18c3032d5e2 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 11:20:39 +0200 Subject: [PATCH 69/86] Set the objects_meta attribute at the migration of the library model to the loadedlibrary model --- .../0012_alter_appliedcontrol_updated_at_and_more.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py index 3c3a4ed38..23137413d 100644 --- a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py @@ -69,6 +69,13 @@ def adapt_libraries(apps, schema_editor): library.urn in BUILTIN_LIBRARY_URNS ) # There is no perfect way to verify is a loaded custom library is builtin or not # There is no way to generate the objects_meta dictionary without reading all files from ./backend/library/libraries, but we can generate the missing objects_meta values at the same time we generate the StoredLibrary objects. + + library.objects_meta = { + "frameworks": library.frameworks.count(), + "threats": library.threats.count(), + "reference_controls": library.reference_controls.count(), + "risk_matrix": library.risk_matrices.count() + } library.save() From fa6e99c788214094e218ee055ef749ef8a9105ea Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 11:30:43 +0200 Subject: [PATCH 70/86] Display stored libraries that have been loaded when they are not builtin --- frontend/src/routes/(app)/libraries/+page.server.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index cd9c79a5b..181b3dfe1 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -61,11 +61,11 @@ export const load = (async ({ fetch }) => { }; }; - const unloadedStoredLibraries = storedLibraries.filter((lib) => lib.is_loaded === false); + const visibleStoredLibraries = storedLibraries.filter(lib => !(lib.is_loaded && lib.builtin)); const storedLibrariesTable = { head: makeHeadData('stored-libraries'), - meta: { urlmodel: 'stored-libraries', ...unloadedStoredLibraries }, - body: tableSourceMapper(unloadedStoredLibraries, listViewFields['stored-libraries'].body) + meta: { urlmodel: 'stored-libraries', ...visibleStoredLibraries }, + body: tableSourceMapper(visibleStoredLibraries, listViewFields['stored-libraries'].body) }; const loadedLibrariesTable = makeLibrariesTable(loadedLibraries, 'loaded-libraries'); From 08d0ba5c8bdcf9afa1630c11b8529b84a7264fcf Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 14:18:39 +0200 Subject: [PATCH 71/86] Fix failing detail views --- backend/library/views.py | 68 ++++++++++++++----- .../components/ModelTable/ModelTable.svelte | 10 +-- .../src/routes/(app)/libraries/+page.svelte | 1 + .../(app)/libraries/[id=urn]/+server.ts | 25 +++---- .../(app)/libraries/[id=urn]/tree/+server.ts | 21 ++---- 5 files changed, 72 insertions(+), 53 deletions(-) diff --git a/backend/library/views.py b/backend/library/views.py index 54aab536c..a7c1391b2 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -3,7 +3,9 @@ from rest_framework import viewsets, status from rest_framework.status import ( HTTP_200_OK, + HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST, + HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND, HTTP_422_UNPROCESSABLE_ENTITY, ) @@ -48,9 +50,31 @@ def get_serializer_class(self): return StoredLibrarySerializer return StoredLibraryDetailedSerializer + def retrieve(self, request, *args, pk, **kwargs): + if "view_storedlibrary" not in request.user.permissions: + return Response(status=HTTP_403_FORBIDDEN) + try: + lib = StoredLibrary.objects.get( + urn=pk + ) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist + except: + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) + data = StoredLibrarySerializer(lib).data + return Response(data) + + def content(self, request, pk): + try : + lib = StoredLibrary.objects.get(urn=pk) + except : + return Response("Library not found.", status=HTTP_404_NOT_FOUND) + return Response(lib.content) + @action(detail=True, methods=["get"]) def content(self, request, pk): - lib = StoredLibrary.objects.get(id=pk) + try : + lib = StoredLibrary.objects.get(urn=pk) + except : + return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib.content) def destroy(self, request, *args, pk, **kwargs): @@ -59,15 +83,15 @@ def destroy(self, request, *args, pk, **kwargs): perm=Permission.objects.get(codename="delete_storedlibrary"), folder=Folder.get_root_folder(), ): - return Response(status=status.HTTP_403_FORBIDDEN) + return Response(status=HTTP_403_FORBIDDEN) try: lib = StoredLibrary.objects.get(urn=pk) except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) lib.delete() - return Response(status=status.HTTP_204_NO_CONTENT) + return Response(status=HTTP_204_NO_CONTENT) @action(detail=True, methods=["get"], url_path="import") def import_library(self, request, pk): @@ -76,20 +100,20 @@ def import_library(self, request, pk): perm=Permission.objects.get(codename="add_loadedlibrary"), folder=Folder.get_root_folder(), ): - return Response(status=status.HTTP_403_FORBIDDEN) + return Response(status=HTTP_403_FORBIDDEN) try: library = StoredLibrary.objects.get( urn=pk ) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) try: error_msg = library.load() if error_msg is not None: return Response( {"status": "error", "error": error_msg}, - status=status.HTTP_400_BAD_REQUEST, + status=HTTP_400_BAD_REQUEST, ) # This can cause translation issues return Response({"status": "success"}) except Exception: @@ -103,9 +127,9 @@ def import_library(self, request, pk): @action(detail=True, methods=["get"]) def tree(self, request, pk): try: - lib = StoredLibrary.objects.get(id=pk) + lib = StoredLibrary.objects.get(urn=pk) except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) library_objects = json.loads(lib.content) # We may need caching for this if not (framework := library_objects.get("framework")): @@ -113,6 +137,7 @@ def tree(self, request, pk): data="This library does not include a framework.", status=HTTP_400_BAD_REQUEST, ) + preview = preview_library(framework) return Response( get_sorted_requirement_nodes(preview.get("requirement_nodes"), None) @@ -165,7 +190,7 @@ class LoadedLibraryViewSet(viewsets.ModelViewSet): def list(self, request, *args, **kwargs): if "view_storedlibrary" not in request.user.permissions: - return Response(status=status.HTTP_403_FORBIDDEN) + return Response(status=HTTP_403_FORBIDDEN) loaded_libraries = [ { @@ -191,13 +216,13 @@ def list(self, request, *args, **kwargs): def retrieve(self, request, *args, pk, **kwargs): if "view_loadedlibrary" not in request.user.permissions: - return Response(status=status.HTTP_403_FORBIDDEN) + return Response(status=HTTP_403_FORBIDDEN) try: lib = LoadedLibrary.objects.get( urn=pk ) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) data = LoadedLibraryDetailedSerializer(lib).data data["objects"] = lib._objects return Response(data) @@ -208,30 +233,39 @@ def destroy(self, request, *args, pk, **kwargs): perm=Permission.objects.get(codename="delete_loadedlibrary"), folder=Folder.get_root_folder(), ): - return Response(status=status.HTTP_403_FORBIDDEN) + return Response(status=HTTP_403_FORBIDDEN) try: lib = LoadedLibrary.objects.get(urn=pk) except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) if lib.reference_count != 0: return Response( data="Library cannot be deleted because it has references.", - status=status.HTTP_400_BAD_REQUEST, + status=HTTP_400_BAD_REQUEST, ) lib.delete() - return Response(status=status.HTTP_204_NO_CONTENT) + return Response(status=HTTP_204_NO_CONTENT) + + @action(detail=True, methods=["get"]) + def content(self, request, pk): + try : + lib = LoadedLibrary.objects.get(urn=pk) + except : + return Response("Library not found.", status=HTTP_404_NOT_FOUND) + return Response(lib._objects) @action(detail=True, methods=["get"]) def tree( self, request, pk ): # We must ensure that users that are not allowed to read the content of libraries can't have any access to them either from the /api/{URLModel/{library_urn}/tree view or the /api/{URLModel}/{library_urn} view. + try: lib = LoadedLibrary.objects.get(urn=pk) except: - return Response(data="Library not found.", status=status.HTTP_404_NOT_FOUND) + return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) if lib.frameworks.count() == 0: return Response( diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index f92d8caaf..d038bb30c 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -72,7 +72,7 @@ const rowMetaData = $rows[rowIndex].meta; /** @event {rowMetaData} selected - Fires when a table row is clicked. */ if (!rowMetaData[identifierField] || !URLModel) return; - goto(`/${URLModel}/${rowMetaData[identifierField]}`); + goto(`/${URLModel}/${rowMetaData[identifierField]}${detailQueryParameter}`); } // Row Keydown Handler @@ -84,10 +84,10 @@ } export let identifierField = 'id'; - export let deleteForm: SuperValidated | undefined = undefined; - export let URLModel: urlModel | undefined = undefined; + export let detailQueryParameter: string | undefined ; + detailQueryParameter = detailQueryParameter ? `?${detailQueryParameter}` : ""; const user = $page.data.user; @@ -133,7 +133,7 @@ $: model = source.meta?.urlmodel ? URL_MODEL_MAP[source.meta.urlmodel] : URL_MODEL_MAP[URLModel]; $: source, handler.setRows(data); - const actionsURLModel = source.meta.urlmodel ?? URLModel; + const actionsURLModel = source.meta?.urlmodel ?? URLModel; const preventDelete = (row: TableSource) => (row.meta.builtin && actionsURLModel !== 'loaded-libraries') || (Object.hasOwn(row.meta, 'reference_count') && row.meta.reference_count > 0); @@ -272,7 +272,7 @@ deleteForm={deleteForm} {model} URLModel={actionsURLModel} - detailURL={`/${actionsURLModel}/${row.meta[identifierField]}`} + detailURL={`/${actionsURLModel}/${row.meta[identifierField]}${detailQueryParameter}`} editURL={!(row.meta.builtin || row.meta.urn) ? `/${actionsURLModel}/${row.meta[identifierField]}/edit?next=${$page.url.pathname}` : undefined} {row} hasBody={$$slots.actionsBody} diff --git a/frontend/src/routes/(app)/libraries/+page.svelte b/frontend/src/routes/(app)/libraries/+page.svelte index aba78fa54..6549adcab 100644 --- a/frontend/src/routes/(app)/libraries/+page.svelte +++ b/frontend/src/routes/(app)/libraries/+page.svelte @@ -46,6 +46,7 @@ identifierField="urn" pagination={false} deleteForm={data.deleteForm} + detailQueryParameter="loaded" /> {/if} diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index 28c9594bf..1879fe21c 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -6,23 +6,20 @@ import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, url, params }) => { const isLoaded = url.searchParams.has('loaded'); const URLModel = isLoaded ? 'loaded-libraries' : 'stored-libraries'; - const endpoint = `${BASE_API_URL}/${URLModel}/?urn=${params.id}`; + const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/`; + const contentEndpoint = `${BASE_API_URL}/${URLModel}/${params.id}/content`; - const res = await fetch(endpoint); - if (!res.ok) { - error(res.status as NumericRange<400, 599>, await res.json()); - } - const data = await res - .json() - .then((res) => res.results) - .then((res) => res.reduce((acc, curr) => (acc.version > curr.version ? acc : curr))); // Get the latest version of the library + const [res,contentRes] = await Promise.all([ + fetch(endpoint), + fetch(contentEndpoint) + ]); - const uuid = data.id; - const contentEndpoint = `${BASE_API_URL}/${URLModel}/${uuid}/content`; - const contentRes = await fetch(contentEndpoint); - if (!contentRes.ok) { + if (!res.ok) + error(res.status as NumericRange<400, 599>, await res.json()); + if (!contentRes.ok) error(contentRes.status as NumericRange<400, 599>, await contentRes.json()); - } + + const data = await res.json() const content = await contentRes.json(); data.objects = content; if (!isLoaded) { diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index 5d2119ae1..d9c142e66 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -5,26 +5,13 @@ import type { RequestHandler } from './$types'; export const GET: RequestHandler = async ({ fetch, params, url }) => { const URLModel = url.searchParams.has('loaded') ? 'loaded-libraries' : 'stored-libraries'; - const endpoint = `${BASE_API_URL}/${URLModel}/?urn=${params.id}`; - + const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/tree`; const res = await fetch(endpoint); - if (!res.ok) { - error(res.status as NumericRange<400, 599>, await res.json()); - } - const data = await res - .json() - .then((res) => res.results) - .then((res) => res.reduce((acc, curr) => (acc.version > curr.version ? acc : curr))); // Get the latest version of the library - const uuid = data.id; - const treeEndpoint = `${BASE_API_URL}/${URLModel}/${uuid}/tree`; - const treeRes = await fetch(treeEndpoint); - if (!treeRes.ok) { - error(treeRes.status as NumericRange<400, 599>, await treeRes.json()); - } - - const tree = await treeRes.json(); + if (!res.ok) + error(res.status as NumericRange<400, 599>, await res.json()); + const tree = await res.json(); return new Response(JSON.stringify(tree), { headers: { 'Content-Type': 'application/json' From 27b3e93850dbe082e2f5aced3d5fa67ea2e913ed Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 14:26:43 +0200 Subject: [PATCH 72/86] Formatter --- ...lter_appliedcontrol_updated_at_and_more.py | 2 +- backend/core/models.py | 4 +-- backend/library/views.py | 13 ++++---- tools/add_builtin.py | 30 ++++++++----------- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py index 23137413d..ee9779bb0 100644 --- a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py @@ -74,7 +74,7 @@ def adapt_libraries(apps, schema_editor): "frameworks": library.frameworks.count(), "threats": library.threats.count(), "reference_controls": library.reference_controls.count(), - "risk_matrix": library.risk_matrices.count() + "risk_matrix": library.risk_matrices.count(), } library.save() diff --git a/backend/core/models.py b/backend/core/models.py index b76629b47..29dc2d536 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -149,9 +149,7 @@ def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None" locale = library_data.get("locale", "en") version = int(library_data["version"]) is_loaded = LoadedLibrary.objects.filter( - urn=urn, - locale=locale, - version=version + urn=urn, locale=locale, version=version ).exists() library_matches = [*StoredLibrary.objects.filter(urn=urn, locale=locale)] diff --git a/backend/library/views.py b/backend/library/views.py index a7c1391b2..88ce401ee 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -63,17 +63,17 @@ def retrieve(self, request, *args, pk, **kwargs): return Response(data) def content(self, request, pk): - try : + try: lib = StoredLibrary.objects.get(urn=pk) - except : + except: return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib.content) @action(detail=True, methods=["get"]) def content(self, request, pk): - try : + try: lib = StoredLibrary.objects.get(urn=pk) - except : + except: return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib.content) @@ -251,9 +251,9 @@ def destroy(self, request, *args, pk, **kwargs): @action(detail=True, methods=["get"]) def content(self, request, pk): - try : + try: lib = LoadedLibrary.objects.get(urn=pk) - except : + except: return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib._objects) @@ -261,7 +261,6 @@ def content(self, request, pk): def tree( self, request, pk ): # We must ensure that users that are not allowed to read the content of libraries can't have any access to them either from the /api/{URLModel/{library_urn}/tree view or the /api/{URLModel}/{library_urn} view. - try: lib = LoadedLibrary.objects.get(urn=pk) except: diff --git a/tools/add_builtin.py b/tools/add_builtin.py index e69afa3a5..cfb10e58e 100644 --- a/tools/add_builtin.py +++ b/tools/add_builtin.py @@ -3,24 +3,18 @@ # This is a temporary tool that adds "builtin: true" to all libraries stored as yaml files under ./backend/library/libraries. current_path = "/".join(__file__.split("/")[:-1]) -library_path = os.path.join(current_path,"..","backend","library","libraries") -fnames = [ - fname for fname in os.listdir(library_path) - if fname.endswith(".yaml") -] +library_path = os.path.join(current_path, "..", "backend", "library", "libraries") +fnames = [fname for fname in os.listdir(library_path) if fname.endswith(".yaml")] -for fname in fnames : - lib = os.path.join(library_path,fname) +for fname in fnames: + lib = os.path.join(library_path, fname) - with open(lib,"r",encoding="utf-8") as f: - content = f.read() - lines = content.split("\n") + with open(lib, "r", encoding="utf-8") as f: + content = f.read() + lines = content.split("\n") - if not any( - re.match(r"^builtin.*:.*true",line) is not None - for line in lines - ) : - lines.insert(1,"builtin: true") - with open(lib,"w",encoding="utf-8") as f : - f.write("\n".join(lines)) - print(f"Library '{fname}' updated.") + if not any(re.match(r"^builtin.*:.*true", line) is not None for line in lines): + lines.insert(1, "builtin: true") + with open(lib, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + print(f"Library '{fname}' updated.") From 1381c9828dcd20af9087d162fb41d05d35fe9bd2 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 15:28:03 +0200 Subject: [PATCH 73/86] Prettier --- .../src/lib/components/ModelTable/ModelTable.svelte | 4 ++-- frontend/src/routes/(app)/libraries/+page.server.ts | 2 +- .../src/routes/(app)/libraries/[id=urn]/+server.ts | 13 ++++--------- .../routes/(app)/libraries/[id=urn]/tree/+server.ts | 3 +-- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/frontend/src/lib/components/ModelTable/ModelTable.svelte b/frontend/src/lib/components/ModelTable/ModelTable.svelte index d038bb30c..268ddd2d3 100644 --- a/frontend/src/lib/components/ModelTable/ModelTable.svelte +++ b/frontend/src/lib/components/ModelTable/ModelTable.svelte @@ -86,8 +86,8 @@ export let identifierField = 'id'; export let deleteForm: SuperValidated | undefined = undefined; export let URLModel: urlModel | undefined = undefined; - export let detailQueryParameter: string | undefined ; - detailQueryParameter = detailQueryParameter ? `?${detailQueryParameter}` : ""; + export let detailQueryParameter: string | undefined; + detailQueryParameter = detailQueryParameter ? `?${detailQueryParameter}` : ''; const user = $page.data.user; diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index 181b3dfe1..51d4e31f2 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -61,7 +61,7 @@ export const load = (async ({ fetch }) => { }; }; - const visibleStoredLibraries = storedLibraries.filter(lib => !(lib.is_loaded && lib.builtin)); + const visibleStoredLibraries = storedLibraries.filter((lib) => !(lib.is_loaded && lib.builtin)); const storedLibrariesTable = { head: makeHeadData('stored-libraries'), meta: { urlmodel: 'stored-libraries', ...visibleStoredLibraries }, diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts index 1879fe21c..a51bb86f7 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+server.ts @@ -9,17 +9,12 @@ export const GET: RequestHandler = async ({ fetch, url, params }) => { const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/`; const contentEndpoint = `${BASE_API_URL}/${URLModel}/${params.id}/content`; - const [res,contentRes] = await Promise.all([ - fetch(endpoint), - fetch(contentEndpoint) - ]); + const [res, contentRes] = await Promise.all([fetch(endpoint), fetch(contentEndpoint)]); - if (!res.ok) - error(res.status as NumericRange<400, 599>, await res.json()); - if (!contentRes.ok) - error(contentRes.status as NumericRange<400, 599>, await contentRes.json()); + if (!res.ok) error(res.status as NumericRange<400, 599>, await res.json()); + if (!contentRes.ok) error(contentRes.status as NumericRange<400, 599>, await contentRes.json()); - const data = await res.json() + const data = await res.json(); const content = await contentRes.json(); data.objects = content; if (!isLoaded) { diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts index d9c142e66..f2e2f9421 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts +++ b/frontend/src/routes/(app)/libraries/[id=urn]/tree/+server.ts @@ -8,8 +8,7 @@ export const GET: RequestHandler = async ({ fetch, params, url }) => { const endpoint = `${BASE_API_URL}/${URLModel}/${params.id}/tree`; const res = await fetch(endpoint); - if (!res.ok) - error(res.status as NumericRange<400, 599>, await res.json()); + if (!res.ok) error(res.status as NumericRange<400, 599>, await res.json()); const tree = await res.json(); return new Response(JSON.stringify(tree), { From 466e466da870cb2a417a42ce94ae1bf755c555a3 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 16:18:21 +0200 Subject: [PATCH 74/86] Fix backend unit tests --- backend/app_tests/api/test_api_libraries.py | 22 +++++++++----- .../api/test_api_requirement_nodes.py | 4 +-- backend/app_tests/api/test_utils.py | 13 ++++++-- backend/library/views.py | 30 ++++++++++++------- 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/backend/app_tests/api/test_api_libraries.py b/backend/app_tests/api/test_api_libraries.py index df16ab0f3..58a09b591 100644 --- a/backend/app_tests/api/test_api_libraries.py +++ b/backend/app_tests/api/test_api_libraries.py @@ -60,11 +60,14 @@ def test_import_frameworks(self, test): # Uses the API endpoint to get library details with the admin client lib_detail_response = test.admin_client.get( - EndpointTestsUtils.get_referential_object_url_from_urn( - test.client, TEST_FRAMEWORK_URN, StoredLibrary + EndpointTestsUtils.get_stored_library_content( + test.client, TEST_FRAMEWORK_URN ) - ).json()["content"] - lib_detail_response = json.loads(lib_detail_response)["framework"] + ) + lib_detail_response = lib_detail_response.content + lib_detail_response = json.loads(lib_detail_response) + lib_detail_response = json.loads(lib_detail_response) + lib_detail_response = lib_detail_response["framework"] # Asserts that the library is not already loaded assert ( @@ -126,11 +129,14 @@ def test_import_risk_matrix(self, test): # Uses the API endpoint to get library details with the admin client lib_detail_response = test.admin_client.get( - EndpointTestsUtils.get_referential_object_url_from_urn( - test.client, TEST_RISK_MATRIX_URN, StoredLibrary + EndpointTestsUtils.get_stored_library_content( + test.client, TEST_RISK_MATRIX_URN ) - ).json()["content"] - lib_detail_response = json.loads(lib_detail_response)["risk_matrix"][0] + ) + lib_detail_response = lib_detail_response.content + lib_detail_response = json.loads(lib_detail_response) + lib_detail_response = json.loads(lib_detail_response) + lib_detail_response = lib_detail_response["risk_matrix"][0] # Asserts that the library is not already loaded assert ( diff --git a/backend/app_tests/api/test_api_requirement_nodes.py b/backend/app_tests/api/test_api_requirement_nodes.py index 5bb9a0b20..611214503 100644 --- a/backend/app_tests/api/test_api_requirement_nodes.py +++ b/backend/app_tests/api/test_api_requirement_nodes.py @@ -77,9 +77,7 @@ def test_import_requirement_nodes(self, test): test.client, "Requirement nodes", EndpointTestsUtils.get_endpoint_url("Requirement nodes"), - EndpointTestsUtils.get_referential_object_url_from_urn( - test.client, TEST_FRAMEWORK_URN, StoredLibrary - ), + EndpointTestsUtils.get_stored_library_content(test.client, TEST_FRAMEWORK_URN), [ "name", "description", diff --git a/backend/app_tests/api/test_utils.py b/backend/app_tests/api/test_utils.py index 3f571315d..34c603de7 100644 --- a/backend/app_tests/api/test_utils.py +++ b/backend/app_tests/api/test_utils.py @@ -33,8 +33,13 @@ def get_referential_object_url_from_urn( authenticated_client, urn: str, model: models.Model = StoredLibrary ): """Get the object URL from the URN""" - uuid = model.objects.filter(urn=urn).last().id - return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{uuid}/" + return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/" + + def get_stored_library_content( + authenticated_client,urn: str + ) -> str : + """Return an URL to fetch the content of a stored library""" + return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/content/" @pytest.mark.django_db def get_test_client_and_folder( @@ -1023,7 +1028,9 @@ def compare_results( reference.status_code == status.HTTP_200_OK ), "reference endpoint is not accessible" - content = json.loads(reference.json()["content"]) + content = json.loads(reference.content) + content = json.loads(content) + for object in content["framework"][object_name.lower().replace(" ", "_")][ :count ]: diff --git a/backend/library/views.py b/backend/library/views.py index 88ce401ee..83b4ea543 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -54,8 +54,9 @@ def retrieve(self, request, *args, pk, **kwargs): if "view_storedlibrary" not in request.user.permissions: return Response(status=HTTP_403_FORBIDDEN) try: + key = "urn" if pk.startswith("urn:") else "id" lib = StoredLibrary.objects.get( - urn=pk + **{key: pk} ) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) @@ -64,7 +65,8 @@ def retrieve(self, request, *args, pk, **kwargs): def content(self, request, pk): try: - lib = StoredLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = StoredLibrary.objects.get(**{key: pk}) except: return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib.content) @@ -72,7 +74,8 @@ def content(self, request, pk): @action(detail=True, methods=["get"]) def content(self, request, pk): try: - lib = StoredLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = StoredLibrary.objects.get(**{key: pk}) except: return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib.content) @@ -86,7 +89,8 @@ def destroy(self, request, *args, pk, **kwargs): return Response(status=HTTP_403_FORBIDDEN) try: - lib = StoredLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = StoredLibrary.objects.get(**{key: pk}) except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) @@ -102,8 +106,9 @@ def import_library(self, request, pk): ): return Response(status=HTTP_403_FORBIDDEN) try: + key = "urn" if pk.startswith("urn:") else "id" library = StoredLibrary.objects.get( - urn=pk + **{key: pk} ) # This is only fetching the lib by URN without caring about the locale or the version, this must change in the future. except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) @@ -127,7 +132,8 @@ def import_library(self, request, pk): @action(detail=True, methods=["get"]) def tree(self, request, pk): try: - lib = StoredLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = StoredLibrary.objects.get(**{key: pk}) except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) @@ -218,8 +224,9 @@ def retrieve(self, request, *args, pk, **kwargs): if "view_loadedlibrary" not in request.user.permissions: return Response(status=HTTP_403_FORBIDDEN) try: + key = "urn" if pk.startswith("urn:") else "id" lib = LoadedLibrary.objects.get( - urn=pk + **{key: pk} ) # There is no "locale" value involved in the fetch + we have to handle the exception if the pk urn doesn't exist except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) @@ -236,7 +243,8 @@ def destroy(self, request, *args, pk, **kwargs): return Response(status=HTTP_403_FORBIDDEN) try: - lib = LoadedLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = LoadedLibrary.objects.get(**{key: pk}) except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) @@ -252,7 +260,8 @@ def destroy(self, request, *args, pk, **kwargs): @action(detail=True, methods=["get"]) def content(self, request, pk): try: - lib = LoadedLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = LoadedLibrary.objects.get(**{key: pk}) except: return Response("Library not found.", status=HTTP_404_NOT_FOUND) return Response(lib._objects) @@ -262,7 +271,8 @@ def tree( self, request, pk ): # We must ensure that users that are not allowed to read the content of libraries can't have any access to them either from the /api/{URLModel/{library_urn}/tree view or the /api/{URLModel}/{library_urn} view. try: - lib = LoadedLibrary.objects.get(urn=pk) + key = "urn" if pk.startswith("urn:") else "id" + lib = LoadedLibrary.objects.get(**{key: pk}) except: return Response(data="Library not found.", status=HTTP_404_NOT_FOUND) From 5ae1662c2922721ecbd6d37d642a878d532e0af6 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 16:22:21 +0200 Subject: [PATCH 75/86] Formatter --- backend/app_tests/api/test_api_requirement_nodes.py | 4 +++- backend/app_tests/api/test_utils.py | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/app_tests/api/test_api_requirement_nodes.py b/backend/app_tests/api/test_api_requirement_nodes.py index 611214503..19bfaf76c 100644 --- a/backend/app_tests/api/test_api_requirement_nodes.py +++ b/backend/app_tests/api/test_api_requirement_nodes.py @@ -77,7 +77,9 @@ def test_import_requirement_nodes(self, test): test.client, "Requirement nodes", EndpointTestsUtils.get_endpoint_url("Requirement nodes"), - EndpointTestsUtils.get_stored_library_content(test.client, TEST_FRAMEWORK_URN), + EndpointTestsUtils.get_stored_library_content( + test.client, TEST_FRAMEWORK_URN + ), [ "name", "description", diff --git a/backend/app_tests/api/test_utils.py b/backend/app_tests/api/test_utils.py index 34c603de7..0d8ba48ab 100644 --- a/backend/app_tests/api/test_utils.py +++ b/backend/app_tests/api/test_utils.py @@ -35,9 +35,7 @@ def get_referential_object_url_from_urn( """Get the object URL from the URN""" return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/" - def get_stored_library_content( - authenticated_client,urn: str - ) -> str : + def get_stored_library_content(authenticated_client, urn: str) -> str: """Return an URL to fetch the content of a stored library""" return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/content/" From f3e56f5bc0a06fbd782b692fc24f09c69c2f80e0 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 17:09:21 +0200 Subject: [PATCH 76/86] Update data-model --- documentation/architecture/data-model.md | 44 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/documentation/architecture/data-model.md b/documentation/architecture/data-model.md index d6e6787dc..5a27bb067 100644 --- a/documentation/architecture/data-model.md +++ b/documentation/architecture/data-model.md @@ -35,7 +35,8 @@ erDiagram DOMAIN ||--o{ COMPLIANCE_ASSESSMENT_REVIEW: contains ROOT_FOLDER ||--o{ FRAMEWORK : contains ROOT_FOLDER ||--o{ REFERENCE_CONTROL : contains - ROOT_FOLDER ||--o{ LIBRARY : contains + ROOT_FOLDER ||--o{ STORED_LIBRARY : contains + ROOT_FOLDER ||--o{ LOADED_LIBRARY : contains ROOT_FOLDER ||--o{ USER : contains ROOT_FOLDER ||--o{ USER_GROUP : contains ROOT_FOLDER ||--o{ ROLE : contains @@ -58,12 +59,12 @@ erDiagram ```mermaid erDiagram - LIBRARY |o--o{ REFERENCE_CONTROL: contains - LIBRARY |o--o{ THREAT : contains - LIBRARY ||--o{ FRAMEWORK : contains - LIBRARY ||--o{ RISK_MATRIX : contains - LIBRARY ||--o{ MAPPING : contains - LIBRARY2 }o--o{ LIBRARY : depends_on + LOADED_LIBRARY |o--o{ REFERENCE_CONTROL: contains + LOADED_LIBRARY |o--o{ THREAT : contains + LOADED_LIBRARY ||--o{ FRAMEWORK : contains + LOADED_LIBRARY ||--o{ RISK_MATRIX : contains + LOADED_LIBRARY ||--o{ MAPPING : contains + LOADED_LIBRARY2 }o--o{ LOADED_LIBRARY : depends_on ``` ### General data model @@ -419,13 +420,15 @@ NameDescriptionMixin <|-- RiskScenario AbstractBaseModel <|-- NameDescriptionMixin NameDescriptionMixin <|-- ReferentialObjectMixin FolderMixin <|-- ReferentialObjectMixin -ReferentialObjectMixin <|-- Library ReferentialObjectMixin <|-- Threat ReferentialObjectMixin <|-- ReferenceControl ReferentialObjectMixin <|-- RiskMatrix ReferentialObjectMixin <|-- Framework ReferentialObjectMixin <|-- RequirementNode ReferentialObjectMixin <|-- Mapping +ReferentialObjectMixin <|-- LibraryMixin +LibraryMixin <|-- StoredLibrary +LibraryMixin <|-- LoadedLibrary NameDescriptionMixin <|-- Assessment FolderMixin <|-- Project NameDescriptionMixin <|-- Project @@ -455,23 +458,36 @@ namespace ReferentialObjects { +display_long() str } - class Library { + class LibraryMixin { +CharField copyright +IntegerField version +CharField provider +CharField packager - +Library[] dependencies + +JsonField dependencies + +BooleanField builtin + +JSONField objects_meta + } + + class StoredLibrary { + +BooleanField is_obsolete + +BooleanField is_loaded + +CharField hash_checksum + +TextField content + } + + class LoadedLibrary { + +LoadedLibrary[] dependencies +reference_count() int } class Threat { - +Library library + +LoadedLibrary library +is_deletable() bool +frameworks() Framework[] } class ReferenceControl { - +Library library + +LoadedLibrary library +CharField category +JSONField typical_evidence +is_deletable() bool @@ -479,7 +495,7 @@ namespace ReferentialObjects { } class RiskMatrix { - +Library library + +LoadedLibrary library +JSONField json_definition +BooleanField is_enabled +CharField provider @@ -492,7 +508,7 @@ namespace ReferentialObjects { } class Framework { - +Library library + +LoadedLibrary library +int get_next_order_id(obj_type, _parent_urn) +is_deletable() bool } From 0cac3a3c05223a970b6e6896b4307065004a2f98 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 17:27:22 +0200 Subject: [PATCH 77/86] Add the deutsch language flag display --- frontend/src/lib/utils/constants.ts | 3 ++- frontend/src/lib/utils/locales.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/utils/constants.ts b/frontend/src/lib/utils/constants.ts index 7062fbcdc..15a6966b0 100644 --- a/frontend/src/lib/utils/constants.ts +++ b/frontend/src/lib/utils/constants.ts @@ -39,7 +39,8 @@ export const URN_REGEX = export const LOCALE_DISPLAY_MAP = { en: '🇬🇧 English', - fr: '🇫🇷 Français' + fr: '🇫🇷 Français', + de: '🇩🇪 Deutsch' }; export const ISO_8601_REGEX = diff --git a/frontend/src/lib/utils/locales.ts b/frontend/src/lib/utils/locales.ts index b7c967bd8..eb01df081 100644 --- a/frontend/src/lib/utils/locales.ts +++ b/frontend/src/lib/utils/locales.ts @@ -9,6 +9,10 @@ export const LOCALE_MAP = { name: 'french', flag: '🇫🇷' }, + de: { + name: 'german', + flag: '🇩🇪' + }, ar: { name: 'arabic', flag: '🇸🇦' From d807671239445754b8d3f4430d44a587cd66f143 Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 17:54:28 +0200 Subject: [PATCH 78/86] Keep builtin library in the library store after loading them --- frontend/src/routes/(app)/libraries/+page.server.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/src/routes/(app)/libraries/+page.server.ts b/frontend/src/routes/(app)/libraries/+page.server.ts index 51d4e31f2..d5c6105c4 100644 --- a/frontend/src/routes/(app)/libraries/+page.server.ts +++ b/frontend/src/routes/(app)/libraries/+page.server.ts @@ -61,11 +61,10 @@ export const load = (async ({ fetch }) => { }; }; - const visibleStoredLibraries = storedLibraries.filter((lib) => !(lib.is_loaded && lib.builtin)); const storedLibrariesTable = { head: makeHeadData('stored-libraries'), - meta: { urlmodel: 'stored-libraries', ...visibleStoredLibraries }, - body: tableSourceMapper(visibleStoredLibraries, listViewFields['stored-libraries'].body) + meta: { urlmodel: 'stored-libraries', ...storedLibraries }, + body: tableSourceMapper(storedLibraries, listViewFields['stored-libraries'].body) }; const loadedLibrariesTable = makeLibrariesTable(loadedLibraries, 'loaded-libraries'); From 9a1344768cd8939018ba00888b6796470e52836f Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 17:58:41 +0200 Subject: [PATCH 79/86] Remove the is_obsolete attribute from the StoredLibrary model --- ...012_alter_appliedcontrol_updated_at_and_more.py | 1 - backend/core/models.py | 14 -------------- backend/library/utils.py | 2 +- backend/library/views.py | 2 +- documentation/architecture/data-model.md | 1 - 5 files changed, 2 insertions(+), 18 deletions(-) diff --git a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py index ee9779bb0..d7e506604 100644 --- a/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py +++ b/backend/core/migrations/0012_alter_appliedcontrol_updated_at_and_more.py @@ -326,7 +326,6 @@ class Migration(migrations.Migration): ("builtin", models.BooleanField(default=False)), ("objects_meta", models.JSONField()), ("dependencies", models.JSONField(null=True)), - ("is_obsolete", models.BooleanField(default=False)), ("is_loaded", models.BooleanField(default=False)), ("hash_checksum", models.CharField(max_length=64)), ("content", models.TextField()), diff --git a/backend/core/models.py b/backend/core/models.py index 29dc2d536..385a1e15b 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -110,7 +110,6 @@ class Meta: class StoredLibrary(LibraryMixin): - is_obsolete = models.BooleanField(default=False) is_loaded = models.BooleanField(default=False) hash_checksum = models.CharField(max_length=64) content = models.TextField() @@ -152,19 +151,6 @@ def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None" urn=urn, locale=locale, version=version ).exists() - library_matches = [*StoredLibrary.objects.filter(urn=urn, locale=locale)] - if any(library.version >= version for library in library_matches): - # The library isn't stored if it's obsolete due to be a too old version of itself. - err = "A library with the urn '{}', a locale '{}' with a superior superior or equal to {} is already stored in the database.".format( - urn, locale, version - ) - logger.error("Error while loading library content", error=err) - raise ValueError(err) - - for library in library_matches: - library.is_obsolete = True - library.save() # If a user delete a library from the library store we must set the is_obsolete value of its most recent obsolete version to False. - objects_meta = { key: len(value) for key, value in library_data["objects"].items() } diff --git a/backend/library/utils.py b/backend/library/utils.py index cff340b74..a9be619fd 100644 --- a/backend/library/utils.py +++ b/backend/library/utils.py @@ -575,7 +575,7 @@ def check_and_import_dependencies(self): if not LoadedLibrary.objects.filter(urn=dependency_urn).exists(): # import_library_view(get_library(dependency)) dependency = StoredLibrary.objects.get( - urn=dependency_urn, is_obsolete=False + urn=dependency_urn ) # We only fetch by URN without thinking about what locale, that may be a problem in the future. error_msg = dependency.load() if error_msg is not None: diff --git a/backend/library/views.py b/backend/library/views.py index 83b4ea543..6ea57133f 100644 --- a/backend/library/views.py +++ b/backend/library/views.py @@ -40,7 +40,7 @@ class StoredLibraryViewSet(BaseModelViewSet): # solve issue with URN containing dot, see https://stackoverflow.com/questions/27963899/django-rest-framework-using-dot-in-url lookup_value_regex = r"[\w.:-]+" model = StoredLibrary - queryset = StoredLibrary.objects.filter(is_obsolete=False) + queryset = StoredLibrary.objects.all() filterset_fields = ["urn", "locale", "version", "packager", "provider"] search_fields = ["name", "description", "urn"] diff --git a/documentation/architecture/data-model.md b/documentation/architecture/data-model.md index 5a27bb067..6ff4375cb 100644 --- a/documentation/architecture/data-model.md +++ b/documentation/architecture/data-model.md @@ -469,7 +469,6 @@ namespace ReferentialObjects { } class StoredLibrary { - +BooleanField is_obsolete +BooleanField is_loaded +CharField hash_checksum +TextField content From 7af9afb68ff20e7118f113c1f4cacf2f195e281b Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 18:34:38 +0200 Subject: [PATCH 80/86] Make the StoredLibrary builtin attribute non-customizable --- backend/core/models.py | 10 ++++------ backend/library/libraries/3cf-ed1-v1.yaml | 1 - backend/library/libraries/3cf-v2.yaml | 1 - .../library/libraries/aircyber-v1.5.2.yaml | 1 - ...si-genai-security-recommendations-1.0.yaml | 1 - .../libraries/anssi-guide-hygiene.yaml | 1 - .../library/libraries/anssi-nis-rules.yaml | 1 - ...tions-configuration-systeme-gnu-linux.yaml | 1 - .../library/libraries/ccb-cff-2023-03-01.yaml | 1 - backend/library/libraries/cmmc-2.0.yaml | 1 - .../libraries/cra-proposal-annexes.yaml | 1 - .../libraries/critical_risk_matrix_3x3.yaml | 1 - .../libraries/critical_risk_matrix_5x5.yaml | 1 - .../library/libraries/dfs-500-2023-11.yaml | 1 - backend/library/libraries/doc-pol.yaml | 1 - backend/library/libraries/dora.yaml | 1 - backend/library/libraries/ecc-1.yaml | 1 - .../library/libraries/essential-eight.yaml | 1 - backend/library/libraries/fadp.yaml | 1 - backend/library/libraries/fedramp-rev5.yaml | 1 - backend/library/libraries/gdpr-checklist.yaml | 1 - backend/library/libraries/hds-v2023-a.yaml | 1 - .../library/libraries/iso27001-2022-fr.yaml | 1 - backend/library/libraries/iso27001-2022.yaml | 1 - backend/library/libraries/lpm-oiv-2019.yaml | 1 - .../matrice-des-risques-critiques-3x3.yaml | 1 - .../matrice-des-risques-critiques-5x5.yaml | 1 - .../library/libraries/mitre-attack-v14.yaml | 1 - backend/library/libraries/nis2-directive.yaml | 1 - .../library/libraries/nist-800-171-rev2.yaml | 1 - .../library/libraries/nist-ai-rmf-1.0.yaml | 1 - backend/library/libraries/nist-csf-1.1.yaml | 1 - backend/library/libraries/nist-csf-2.0.yaml | 1 - .../library/libraries/nist-privacy-1.0.yaml | 1 - .../libraries/nist-sp-800-53-rev5.yaml | 1 - .../libraries/nist-sp-800-66-rev2.yaml | 1 - backend/library/libraries/nist-ssdf-1.1.yaml | 1 - .../library/libraries/owasp-asvs-4.0.3.yaml | 1 - .../library/libraries/owasp-top-10-web.yaml | 1 - backend/library/libraries/pcidss-4_0.yaml | 1 - backend/library/libraries/pgssi-s-1.0.yaml | 1 - backend/library/libraries/pspf.yaml | 1 - backend/library/libraries/rgs-v2.0.yaml | 1 - .../libraries/risk-matrix-3x3-mult.yaml | 1 - .../risk-matrix-4x4-pgssi-s-1.0.yaml | 1 - .../libraries/risk-matrix-5x5-sensitive.yaml | 1 - .../libraries/secnumcloud-3.2-annexe-2.yaml | 1 - .../library/libraries/secnumcloud-3.2.yaml | 1 - backend/library/libraries/soc2-2017.yaml | 1 - backend/library/libraries/tiber-eu-2018.yaml | 1 - backend/library/libraries/tisax-v6.0.2.yaml | 1 - tools/add_builtin.py | 20 ------------------- 52 files changed, 4 insertions(+), 76 deletions(-) delete mode 100644 tools/add_builtin.py diff --git a/backend/core/models.py b/backend/core/models.py index 385a1e15b..ea87c84d0 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -125,7 +125,7 @@ def __init_class__(cls): ) @classmethod - def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None": + def store_library_content(cls, library_content: bytes,builtin: bool=False) -> "StoredLibrary | None": hash_checksum = sha256(library_content) if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET: return None # We do not store the library if its hash checksum is in the database. @@ -176,18 +176,16 @@ def store_library_content(cls, library_content: bytes) -> "StoredLibrary | None" objects_meta=objects_meta, dependencies=dependencies, is_loaded=is_loaded, - builtin=library_data.get( - "builtin", False - ), # We have to add a "builtin: true" line to every builtin library file. + builtin=builtin, # We have to add a "builtin: true" line to every builtin library file. hash_checksum=hash_checksum, content=library_objects, ) @classmethod - def store_library_file(cls, fname: Path) -> "StoredLibrary | None": + def store_library_file(cls, fname: Path,builtin: bool=False) -> "StoredLibrary | None": with open(fname, "rb") as f: library_content = f.read() - return StoredLibrary.store_library_content(library_content) + return StoredLibrary.store_library_content(library_content,builtin) def load(self) -> Union[str, None]: from library.utils import LibraryImporter diff --git a/backend/library/libraries/3cf-ed1-v1.yaml b/backend/library/libraries/3cf-ed1-v1.yaml index 5ccb5fca2..0bde6b4b2 100644 --- a/backend/library/libraries/3cf-ed1-v1.yaml +++ b/backend/library/libraries/3cf-ed1-v1.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:3cf-ed1-v1 -builtin: true locale: fr ref_id: 3CF-ed1-v1 name: "Cadre de Conformit\xE9 Cyber France (3CF) pour l'aviation civile" diff --git a/backend/library/libraries/3cf-v2.yaml b/backend/library/libraries/3cf-v2.yaml index ca0024666..4ed3e381b 100644 --- a/backend/library/libraries/3cf-v2.yaml +++ b/backend/library/libraries/3cf-v2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:3cf-v2 -builtin: true locale: fr ref_id: 3CF-v2 name: "Cadre de Conformit\xE9 Cyber France (3CF) pour l'aviation civile - v2" diff --git a/backend/library/libraries/aircyber-v1.5.2.yaml b/backend/library/libraries/aircyber-v1.5.2.yaml index f2a2fb8dc..7e4688efd 100644 --- a/backend/library/libraries/aircyber-v1.5.2.yaml +++ b/backend/library/libraries/aircyber-v1.5.2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:aircyber-v1.5.2 -builtin: true locale: en ref_id: AirCyber-v1.5.2 name: Public AirCyber Maturity Level Matrix diff --git a/backend/library/libraries/anssi-genai-security-recommendations-1.0.yaml b/backend/library/libraries/anssi-genai-security-recommendations-1.0.yaml index bc9f128d4..ea13edc70 100644 --- a/backend/library/libraries/anssi-genai-security-recommendations-1.0.yaml +++ b/backend/library/libraries/anssi-genai-security-recommendations-1.0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:anssi-genai-security-recommendations-1.0 -builtin: true locale: fr ref_id: anssi-genai-security-recommendations-1.0 name: "ANSSI: RECOMMANDATIONS DE S\xC9CURIT\xC9 POUR UN SYST\xC8ME D'IA G\xC9N\xC9\ diff --git a/backend/library/libraries/anssi-guide-hygiene.yaml b/backend/library/libraries/anssi-guide-hygiene.yaml index 3ac2b7905..4d5ff44f5 100644 --- a/backend/library/libraries/anssi-guide-hygiene.yaml +++ b/backend/library/libraries/anssi-guide-hygiene.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:anssi-guide-hygiene -builtin: true locale: fr ref_id: ANSSI-GUIDE-HYGIENE name: "ANSSI - Guide d'hygi\xE8ne informatique" diff --git a/backend/library/libraries/anssi-nis-rules.yaml b/backend/library/libraries/anssi-nis-rules.yaml index dcfe7b564..4fbd0e57c 100644 --- a/backend/library/libraries/anssi-nis-rules.yaml +++ b/backend/library/libraries/anssi-nis-rules.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:anssi-nis-rules -builtin: true locale: en ref_id: ANSSI-NIS name: ANSSI NIS rules diff --git a/backend/library/libraries/anssi-recommandations-configuration-systeme-gnu-linux.yaml b/backend/library/libraries/anssi-recommandations-configuration-systeme-gnu-linux.yaml index 74f4e95e5..e381f13e5 100644 --- a/backend/library/libraries/anssi-recommandations-configuration-systeme-gnu-linux.yaml +++ b/backend/library/libraries/anssi-recommandations-configuration-systeme-gnu-linux.yaml @@ -1,5 +1,4 @@ urn: urn:protocolpaladin:risk:library:anssi-recommandations-configuration-systeme-gnu-linux -builtin: true locale: fr ref_id: 'ANSSI-RECOMMANDATIONS-CONFIGURATION-SYSTEME-GNU-LINUX ' name: "ANSSI - Recommandations de s\xE9curit\xE9 relatives \xE0 un syst\xE8me GNU-Linux" diff --git a/backend/library/libraries/ccb-cff-2023-03-01.yaml b/backend/library/libraries/ccb-cff-2023-03-01.yaml index 824a28fba..5aba346d8 100644 --- a/backend/library/libraries/ccb-cff-2023-03-01.yaml +++ b/backend/library/libraries/ccb-cff-2023-03-01.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:ccb-cff-2023-03-01 -builtin: true locale: en ref_id: CCB-CFF-2023-03-01 name: CCB CyberFundamentals Framework diff --git a/backend/library/libraries/cmmc-2.0.yaml b/backend/library/libraries/cmmc-2.0.yaml index 5a729678e..d3a7a6cd8 100644 --- a/backend/library/libraries/cmmc-2.0.yaml +++ b/backend/library/libraries/cmmc-2.0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:cmmc-2.0 -builtin: true locale: en ref_id: CMMC-2.0 name: CMMC version 2.0 diff --git a/backend/library/libraries/cra-proposal-annexes.yaml b/backend/library/libraries/cra-proposal-annexes.yaml index ebe52ac57..7e1458afa 100644 --- a/backend/library/libraries/cra-proposal-annexes.yaml +++ b/backend/library/libraries/cra-proposal-annexes.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:cra-proposal-annexes -builtin: true locale: en ref_id: CRA-proposal-annexes name: Cyber Resilience Act diff --git a/backend/library/libraries/critical_risk_matrix_3x3.yaml b/backend/library/libraries/critical_risk_matrix_3x3.yaml index a6393d09b..ba72f3327 100644 --- a/backend/library/libraries/critical_risk_matrix_3x3.yaml +++ b/backend/library/libraries/critical_risk_matrix_3x3.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:critical_risk_matrix_3x3 -builtin: true locale: en ref_id: critical_3x3 name: Critical risk matrix 3x3 diff --git a/backend/library/libraries/critical_risk_matrix_5x5.yaml b/backend/library/libraries/critical_risk_matrix_5x5.yaml index 6653cb6c9..89c3c7f10 100644 --- a/backend/library/libraries/critical_risk_matrix_5x5.yaml +++ b/backend/library/libraries/critical_risk_matrix_5x5.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:critical_risk_matrix_5x5 -builtin: true locale: en ref_id: critical_5x5 name: Critical risk matrix 5x5 diff --git a/backend/library/libraries/dfs-500-2023-11.yaml b/backend/library/libraries/dfs-500-2023-11.yaml index 2e111217f..15af8ca94 100644 --- a/backend/library/libraries/dfs-500-2023-11.yaml +++ b/backend/library/libraries/dfs-500-2023-11.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:dfs-500-2023-11 -builtin: true locale: en ref_id: DFS-500-2023-11 name: NY DFS 500 with 2023-11 amendments diff --git a/backend/library/libraries/doc-pol.yaml b/backend/library/libraries/doc-pol.yaml index af8b3358a..07883530d 100644 --- a/backend/library/libraries/doc-pol.yaml +++ b/backend/library/libraries/doc-pol.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:doc-pol -builtin: true locale: en ref_id: doc-pol name: Documents and policies diff --git a/backend/library/libraries/dora.yaml b/backend/library/libraries/dora.yaml index 5a28c03f5..207c72d52 100644 --- a/backend/library/libraries/dora.yaml +++ b/backend/library/libraries/dora.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:dora -builtin: true locale: en ref_id: DORA name: Digital Operational Resilience Act diff --git a/backend/library/libraries/ecc-1.yaml b/backend/library/libraries/ecc-1.yaml index 475a7e9f3..b6c01ceeb 100644 --- a/backend/library/libraries/ecc-1.yaml +++ b/backend/library/libraries/ecc-1.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:ecc-1 -builtin: true locale: en ref_id: essential-cybersecurity-controls name: Essential Cybersecurity Controls diff --git a/backend/library/libraries/essential-eight.yaml b/backend/library/libraries/essential-eight.yaml index c70922aba..d672e55ef 100644 --- a/backend/library/libraries/essential-eight.yaml +++ b/backend/library/libraries/essential-eight.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:essential-eight -builtin: true locale: en ref_id: Essential Eight name: Essential Eight Maturity Model diff --git a/backend/library/libraries/fadp.yaml b/backend/library/libraries/fadp.yaml index e528cbb1a..f2bddad53 100644 --- a/backend/library/libraries/fadp.yaml +++ b/backend/library/libraries/fadp.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:fadp -builtin: true locale: en ref_id: FADP name: 'Federal Act on Data Protection ' diff --git a/backend/library/libraries/fedramp-rev5.yaml b/backend/library/libraries/fedramp-rev5.yaml index b3e8752da..da4e131e4 100644 --- a/backend/library/libraries/fedramp-rev5.yaml +++ b/backend/library/libraries/fedramp-rev5.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:fedramp-rev5 -builtin: true locale: en ref_id: GSA-FEDRAMP-rev5 name: GSA FedRAMP Rev5 diff --git a/backend/library/libraries/gdpr-checklist.yaml b/backend/library/libraries/gdpr-checklist.yaml index 80cf4101e..5774efe26 100644 --- a/backend/library/libraries/gdpr-checklist.yaml +++ b/backend/library/libraries/gdpr-checklist.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:gdpr-checklist -builtin: true locale: en ref_id: GDPR-checklist name: GDPR checklist for data controllers diff --git a/backend/library/libraries/hds-v2023-a.yaml b/backend/library/libraries/hds-v2023-a.yaml index b24a4644d..81ee2a49e 100644 --- a/backend/library/libraries/hds-v2023-a.yaml +++ b/backend/library/libraries/hds-v2023-a.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:hds-v2023-a -builtin: true locale: fr ref_id: HDS-v2023-A name: HDS v2023-A diff --git a/backend/library/libraries/iso27001-2022-fr.yaml b/backend/library/libraries/iso27001-2022-fr.yaml index 697a8ac06..513e3c714 100644 --- a/backend/library/libraries/iso27001-2022-fr.yaml +++ b/backend/library/libraries/iso27001-2022-fr.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:iso27001-2022-fr -builtin: true locale: fr ref_id: ISO/IEC 27001:2022 name: Norme internationale ISO/IEC 27001:2022 diff --git a/backend/library/libraries/iso27001-2022.yaml b/backend/library/libraries/iso27001-2022.yaml index 3644cd71f..d1d0d37a1 100644 --- a/backend/library/libraries/iso27001-2022.yaml +++ b/backend/library/libraries/iso27001-2022.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:iso27001-2022 -builtin: true locale: en ref_id: ISO/IEC 27001:2022 name: International standard ISO/IEC 27001:2022 diff --git a/backend/library/libraries/lpm-oiv-2019.yaml b/backend/library/libraries/lpm-oiv-2019.yaml index d9569afff..cdadeaffe 100644 --- a/backend/library/libraries/lpm-oiv-2019.yaml +++ b/backend/library/libraries/lpm-oiv-2019.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:lpm-oiv-2019 -builtin: true locale: fr ref_id: LPM-OIV-2019 name: "R\xE8gles OIV" diff --git a/backend/library/libraries/matrice-des-risques-critiques-3x3.yaml b/backend/library/libraries/matrice-des-risques-critiques-3x3.yaml index cb1622e3f..64d7b0323 100644 --- a/backend/library/libraries/matrice-des-risques-critiques-3x3.yaml +++ b/backend/library/libraries/matrice-des-risques-critiques-3x3.yaml @@ -1,5 +1,4 @@ urn: urn:protocolpaladin:risk:library:matrice-des-risques-critiques-3x3 -builtin: true locale: fr ref_id: matrice-des-risques-critiques-3x3 name: Matrice des risques critiques 3x3 diff --git a/backend/library/libraries/matrice-des-risques-critiques-5x5.yaml b/backend/library/libraries/matrice-des-risques-critiques-5x5.yaml index 9320d5f07..61aa35815 100644 --- a/backend/library/libraries/matrice-des-risques-critiques-5x5.yaml +++ b/backend/library/libraries/matrice-des-risques-critiques-5x5.yaml @@ -1,5 +1,4 @@ urn: urn:protocolpaladin:risk:library:matrice-des-risques-critiques-5x5 -builtin: true locale: fr ref_id: matrice-des-risques-critiques-5x5 name: Matrice des risques critiques 5x5 diff --git a/backend/library/libraries/mitre-attack-v14.yaml b/backend/library/libraries/mitre-attack-v14.yaml index 9247baa0d..0e75d9219 100644 --- a/backend/library/libraries/mitre-attack-v14.yaml +++ b/backend/library/libraries/mitre-attack-v14.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:mitre-attack-v14 -builtin: true locale: en ref_id: mitre-attack name: Mitre ATT&CK v14 - Threats and mitigations diff --git a/backend/library/libraries/nis2-directive.yaml b/backend/library/libraries/nis2-directive.yaml index fe0c81018..4555f70fd 100644 --- a/backend/library/libraries/nis2-directive.yaml +++ b/backend/library/libraries/nis2-directive.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nis2-directive -builtin: true locale: en ref_id: NIS2-directive name: NIS 2 directive requirements diff --git a/backend/library/libraries/nist-800-171-rev2.yaml b/backend/library/libraries/nist-800-171-rev2.yaml index 93a652a39..077b48dee 100644 --- a/backend/library/libraries/nist-800-171-rev2.yaml +++ b/backend/library/libraries/nist-800-171-rev2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-800-171-rev2 -builtin: true locale: en ref_id: nist-800-171-rev2 name: NIST SP 800-171 Rev. 2 diff --git a/backend/library/libraries/nist-ai-rmf-1.0.yaml b/backend/library/libraries/nist-ai-rmf-1.0.yaml index fe319e940..82088c3f8 100644 --- a/backend/library/libraries/nist-ai-rmf-1.0.yaml +++ b/backend/library/libraries/nist-ai-rmf-1.0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-ai-rmf-1.0 -builtin: true locale: en ref_id: NIST-AI-RMF-1.0 name: NIST AI RMF 1.0 diff --git a/backend/library/libraries/nist-csf-1.1.yaml b/backend/library/libraries/nist-csf-1.1.yaml index a6783f6f5..de2cefa6c 100644 --- a/backend/library/libraries/nist-csf-1.1.yaml +++ b/backend/library/libraries/nist-csf-1.1.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-csf-1.1 -builtin: true locale: en ref_id: NIST-CSF-1.1 name: NIST CSF version 1.1 diff --git a/backend/library/libraries/nist-csf-2.0.yaml b/backend/library/libraries/nist-csf-2.0.yaml index 7161b16c3..9769575ec 100644 --- a/backend/library/libraries/nist-csf-2.0.yaml +++ b/backend/library/libraries/nist-csf-2.0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-csf-2.0 -builtin: true locale: en ref_id: NIST-CSF-2.0 name: NIST CSF version 2.0 diff --git a/backend/library/libraries/nist-privacy-1.0.yaml b/backend/library/libraries/nist-privacy-1.0.yaml index 479388e49..8d79e24b9 100644 --- a/backend/library/libraries/nist-privacy-1.0.yaml +++ b/backend/library/libraries/nist-privacy-1.0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-privacy-1.0 -builtin: true locale: en ref_id: NIST-PRIVACY-1.0 name: NIST PRIVACY FRAMEWORK 1.0 diff --git a/backend/library/libraries/nist-sp-800-53-rev5.yaml b/backend/library/libraries/nist-sp-800-53-rev5.yaml index ac1718c6b..4b9e4e79e 100644 --- a/backend/library/libraries/nist-sp-800-53-rev5.yaml +++ b/backend/library/libraries/nist-sp-800-53-rev5.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-sp-800-53-rev5 -builtin: true locale: en ref_id: NIST-SP-800-53-rev5 name: NIST SP 800-53 revision 5 diff --git a/backend/library/libraries/nist-sp-800-66-rev2.yaml b/backend/library/libraries/nist-sp-800-66-rev2.yaml index 813d47052..642cf412b 100644 --- a/backend/library/libraries/nist-sp-800-66-rev2.yaml +++ b/backend/library/libraries/nist-sp-800-66-rev2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-sp-800-66-rev2 -builtin: true locale: en ref_id: NIST-SP-800-66-rev2 name: NIST SP-800-66 rev2 (HIPAA) diff --git a/backend/library/libraries/nist-ssdf-1.1.yaml b/backend/library/libraries/nist-ssdf-1.1.yaml index e886a23fa..1ab939427 100644 --- a/backend/library/libraries/nist-ssdf-1.1.yaml +++ b/backend/library/libraries/nist-ssdf-1.1.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:nist-ssdf-1.1 -builtin: true locale: en ref_id: nist-ssdf-1.1 name: Secure Software Development Framework (SSDF) diff --git a/backend/library/libraries/owasp-asvs-4.0.3.yaml b/backend/library/libraries/owasp-asvs-4.0.3.yaml index fe8c0608b..b1da8b1cd 100644 --- a/backend/library/libraries/owasp-asvs-4.0.3.yaml +++ b/backend/library/libraries/owasp-asvs-4.0.3.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:owasp-asvs-4.0.3 -builtin: true locale: en ref_id: OWASP-ASVS-4.0.3 name: OWASP ASVS 4.0.3 diff --git a/backend/library/libraries/owasp-top-10-web.yaml b/backend/library/libraries/owasp-top-10-web.yaml index 56ce04262..ce1366760 100644 --- a/backend/library/libraries/owasp-top-10-web.yaml +++ b/backend/library/libraries/owasp-top-10-web.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:owasp-top-10-web -builtin: true locale: en ref_id: OWASP top 10 Web name: OWASP top 10 Web diff --git a/backend/library/libraries/pcidss-4_0.yaml b/backend/library/libraries/pcidss-4_0.yaml index dc5fda82d..f762bc48e 100644 --- a/backend/library/libraries/pcidss-4_0.yaml +++ b/backend/library/libraries/pcidss-4_0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:pcidss-4_0 -builtin: true locale: en ref_id: PCI DSS 4.0 name: Payment Card Industry Data Security Standard diff --git a/backend/library/libraries/pgssi-s-1.0.yaml b/backend/library/libraries/pgssi-s-1.0.yaml index 9d8141edb..1dea53c7d 100644 --- a/backend/library/libraries/pgssi-s-1.0.yaml +++ b/backend/library/libraries/pgssi-s-1.0.yaml @@ -1,5 +1,4 @@ urn: urn:ackwa:risk:library:pgssi-s-1.0 -builtin: true locale: fr ref_id: pgssi-s-1.0 name: PGSSI-S v1.0 diff --git a/backend/library/libraries/pspf.yaml b/backend/library/libraries/pspf.yaml index c7188148c..ce447e4f4 100644 --- a/backend/library/libraries/pspf.yaml +++ b/backend/library/libraries/pspf.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:pspf -builtin: true locale: en ref_id: PSPF name: Protective Security Policy Framework diff --git a/backend/library/libraries/rgs-v2.0.yaml b/backend/library/libraries/rgs-v2.0.yaml index b7c57baae..f6e32ae79 100644 --- a/backend/library/libraries/rgs-v2.0.yaml +++ b/backend/library/libraries/rgs-v2.0.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:rgs-v2.0 -builtin: true locale: fr ref_id: RGS-v2.0 name: "R\xE9f\xE9rentiel G\xE9n\xE9ral de S\xE9curit\xE9 version 2.0" diff --git a/backend/library/libraries/risk-matrix-3x3-mult.yaml b/backend/library/libraries/risk-matrix-3x3-mult.yaml index d78dc061a..d0d7fb7c6 100644 --- a/backend/library/libraries/risk-matrix-3x3-mult.yaml +++ b/backend/library/libraries/risk-matrix-3x3-mult.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:risk-matrix-3x3-mult -builtin: true locale: fr ref_id: risk-matrix-3x3-mult name: Matrice 3x3 multiplicative diff --git a/backend/library/libraries/risk-matrix-4x4-pgssi-s-1.0.yaml b/backend/library/libraries/risk-matrix-4x4-pgssi-s-1.0.yaml index 7e94277d5..3ab7926d0 100644 --- a/backend/library/libraries/risk-matrix-4x4-pgssi-s-1.0.yaml +++ b/backend/library/libraries/risk-matrix-4x4-pgssi-s-1.0.yaml @@ -1,5 +1,4 @@ urn: urn:ackwa:risk:library:risk-matrix-4x4-pgssi-s-1.0 -builtin: true locale: fr ref_id: risk-matrix-4x4-pgssi-s-1.0 name: Matrice de risques 4x4 PGSSI-S v1.0 diff --git a/backend/library/libraries/risk-matrix-5x5-sensitive.yaml b/backend/library/libraries/risk-matrix-5x5-sensitive.yaml index bca59ed03..e4ec0b12e 100644 --- a/backend/library/libraries/risk-matrix-5x5-sensitive.yaml +++ b/backend/library/libraries/risk-matrix-5x5-sensitive.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:risk-matrix-5x5-sensitive -builtin: true locale: en ref_id: risk-matrix-5x5-sensitive name: 5x5 sensitive diff --git a/backend/library/libraries/secnumcloud-3.2-annexe-2.yaml b/backend/library/libraries/secnumcloud-3.2-annexe-2.yaml index 72f2aac6a..dbed2bf22 100644 --- a/backend/library/libraries/secnumcloud-3.2-annexe-2.yaml +++ b/backend/library/libraries/secnumcloud-3.2-annexe-2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:secnumcloud-3.2-annexe-2 -builtin: true locale: fr ref_id: SecNumCloud v3.2-A2 name: 'SecNumCloud v3.2 Annexe 2 : recommandations aux commanditaires' diff --git a/backend/library/libraries/secnumcloud-3.2.yaml b/backend/library/libraries/secnumcloud-3.2.yaml index 85e144d86..8c62a8642 100644 --- a/backend/library/libraries/secnumcloud-3.2.yaml +++ b/backend/library/libraries/secnumcloud-3.2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:secnumcloud-3.2 -builtin: true locale: fr ref_id: SecNumCloud v3.2 name: "Prestataires de services d\u2019informatique en nuage (SecNumCloud) - r\xE9\ diff --git a/backend/library/libraries/soc2-2017.yaml b/backend/library/libraries/soc2-2017.yaml index b0ef49e95..9be234ff5 100644 --- a/backend/library/libraries/soc2-2017.yaml +++ b/backend/library/libraries/soc2-2017.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:soc2-2017 -builtin: true locale: en ref_id: SOC2-2017 name: 'SOC2-2017 Trust Services Criteria ' diff --git a/backend/library/libraries/tiber-eu-2018.yaml b/backend/library/libraries/tiber-eu-2018.yaml index 7e87280c7..902ad6985 100644 --- a/backend/library/libraries/tiber-eu-2018.yaml +++ b/backend/library/libraries/tiber-eu-2018.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:tiber-eu-2018 -builtin: true locale: en ref_id: TIBER-EU-2018 name: TIBER-EU FRAMEWORK diff --git a/backend/library/libraries/tisax-v6.0.2.yaml b/backend/library/libraries/tisax-v6.0.2.yaml index f93cb5953..7c2c6ccce 100644 --- a/backend/library/libraries/tisax-v6.0.2.yaml +++ b/backend/library/libraries/tisax-v6.0.2.yaml @@ -1,5 +1,4 @@ urn: urn:intuitem:risk:library:tisax-v6.0.2 -builtin: true locale: en ref_id: TISAX v6.0.2 name: 'Trusted Information Security Assessment Exchange ' diff --git a/tools/add_builtin.py b/tools/add_builtin.py deleted file mode 100644 index cfb10e58e..000000000 --- a/tools/add_builtin.py +++ /dev/null @@ -1,20 +0,0 @@ -import re, os - -# This is a temporary tool that adds "builtin: true" to all libraries stored as yaml files under ./backend/library/libraries. - -current_path = "/".join(__file__.split("/")[:-1]) -library_path = os.path.join(current_path, "..", "backend", "library", "libraries") -fnames = [fname for fname in os.listdir(library_path) if fname.endswith(".yaml")] - -for fname in fnames: - lib = os.path.join(library_path, fname) - - with open(lib, "r", encoding="utf-8") as f: - content = f.read() - lines = content.split("\n") - - if not any(re.match(r"^builtin.*:.*true", line) is not None for line in lines): - lines.insert(1, "builtin: true") - with open(lib, "w", encoding="utf-8") as f: - f.write("\n".join(lines)) - print(f"Library '{fname}' updated.") From 7a311f8d731f24339aa98f7d254dd1916c09b90c Mon Sep 17 00:00:00 2001 From: monsieurswag Date: Wed, 15 May 2024 18:36:34 +0200 Subject: [PATCH 81/86] Formatter --- backend/core/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index ea87c84d0..d783710b1 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -125,7 +125,9 @@ def __init_class__(cls): ) @classmethod - def store_library_content(cls, library_content: bytes,builtin: bool=False) -> "StoredLibrary | None": + def store_library_content( + cls, library_content: bytes, builtin: bool = False + ) -> "StoredLibrary | None": hash_checksum = sha256(library_content) if hash_checksum in StoredLibrary.HASH_CHECKSUM_SET: return None # We do not store the library if its hash checksum is in the database. @@ -182,10 +184,12 @@ def store_library_content(cls, library_content: bytes,builtin: bool=False) -> "S ) @classmethod - def store_library_file(cls, fname: Path,builtin: bool=False) -> "StoredLibrary | None": + def store_library_file( + cls, fname: Path, builtin: bool = False + ) -> "StoredLibrary | None": with open(fname, "rb") as f: library_content = f.read() - return StoredLibrary.store_library_content(library_content,builtin) + return StoredLibrary.store_library_content(library_content, builtin) def load(self) -> Union[str, None]: from library.utils import LibraryImporter From 091ffee0f4397dbda8de3c6dd5963ee37057ccef Mon Sep 17 00:00:00 2001 From: eric-intuitem <71850047+eric-intuitem@users.noreply.github.com> Date: Wed, 15 May 2024 22:43:11 +0200 Subject: [PATCH 82/86] fix builtin --- backend/library/management/commands/storelibraries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/library/management/commands/storelibraries.py b/backend/library/management/commands/storelibraries.py index d4f7395d5..36e2d3288 100644 --- a/backend/library/management/commands/storelibraries.py +++ b/backend/library/management/commands/storelibraries.py @@ -25,7 +25,7 @@ def handle(self, *args, **options): library_files = [path] for fname in library_files: # logger.info("Begin library file storage", filename=fname) - library = StoredLibrary.store_library_file(fname) + library = StoredLibrary.store_library_file(fname, True) if library: logger.info( "Successfully stored library", From 2def62a43d30e658f9a0b383f5a984db03b6c28e Mon Sep 17 00:00:00 2001 From: eric-intuitem <71850047+eric-intuitem@users.noreply.github.com> Date: Wed, 15 May 2024 22:43:57 +0200 Subject: [PATCH 83/86] fix number shown for framework object The len(value) does not really make sense for a framework --- backend/core/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/models.py b/backend/core/models.py index d783710b1..d8495dfee 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -154,7 +154,7 @@ def store_library_content( ).exists() objects_meta = { - key: len(value) for key, value in library_data["objects"].items() + key: (1 if key == "framework" else len(value)) for key, value in library_data["objects"].items() } dependencies = library_data.get( From ea4d3eef391b22837d6577a9e491ff6b1ae027eb Mon Sep 17 00:00:00 2001 From: Mohamed-Hacene Date: Thu, 16 May 2024 12:38:14 +0200 Subject: [PATCH 84/86] feat: show import button for library stored detailed view --- backend/core/models.py | 3 ++- frontend/src/routes/(app)/libraries/[id=urn]/+page.svelte | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index d8495dfee..8369da767 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -154,7 +154,8 @@ def store_library_content( ).exists() objects_meta = { - key: (1 if key == "framework" else len(value)) for key, value in library_data["objects"].items() + key: (1 if key == "framework" else len(value)) + for key, value in library_data["objects"].items() } dependencies = library_data.get( diff --git a/frontend/src/routes/(app)/libraries/[id=urn]/+page.svelte b/frontend/src/routes/(app)/libraries/[id=urn]/+page.svelte index 063117abc..1e3811706 100644 --- a/frontend/src/routes/(app)/libraries/[id=urn]/+page.svelte +++ b/frontend/src/routes/(app)/libraries/[id=urn]/+page.svelte @@ -68,7 +68,7 @@ return riskMatricesDumps; } - $: displayImportButton = data.library.id === undefined; + $: displayImportButton = !(data.library.is_loaded ?? true);
From 7566effb01ad1d69e10cf4f244e0629d26b1ce88 Mon Sep 17 00:00:00 2001 From: Mohamed-Hacene Date: Thu, 16 May 2024 13:51:18 +0200 Subject: [PATCH 85/86] ci/cd: fix functional library import --- frontend/tests/functional/detailed/libraries.test.ts | 2 +- frontend/tests/utils/page-content.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/tests/functional/detailed/libraries.test.ts b/frontend/tests/functional/detailed/libraries.test.ts index dad419bce..54041b13b 100644 --- a/frontend/tests/functional/detailed/libraries.test.ts +++ b/frontend/tests/functional/detailed/libraries.test.ts @@ -23,7 +23,7 @@ test('every libraries can be loaded', async ({ logedPage, librariesPage, page }) expect( previousRemainingLibrary, 'An error occured while importing library: ' + previousRemainingLibrary - ).not.toEqual(nextRemainingLibrary); + ).toEqual(nextRemainingLibrary); } }); diff --git a/frontend/tests/utils/page-content.ts b/frontend/tests/utils/page-content.ts index 284b6f653..8f648da63 100644 --- a/frontend/tests/utils/page-content.ts +++ b/frontend/tests/utils/page-content.ts @@ -78,7 +78,7 @@ export class PageContent extends BasePage { } } // If the library is not visible, it might have already been loaded - if (await this.getRow(ref).isHidden()) { + if (await this.importItemButton(ref, language === 'any' ? undefined : language).isHidden()) { await this.tab('Loaded libraries').click(); expect(this.tab('Loaded libraries').getAttribute('aria-selected')).toBeTruthy(); expect(this.getRow(ref)).toBeVisible(); From a41b0a324c06920fe421a40168d7b5437e63e03d Mon Sep 17 00:00:00 2001 From: Nassim Tabchiche Date: Thu, 16 May 2024 19:21:12 +0100 Subject: [PATCH 86/86] Fix functional test for library import --- .../functional/detailed/libraries.test.ts | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/frontend/tests/functional/detailed/libraries.test.ts b/frontend/tests/functional/detailed/libraries.test.ts index 54041b13b..4675615d2 100644 --- a/frontend/tests/functional/detailed/libraries.test.ts +++ b/frontend/tests/functional/detailed/libraries.test.ts @@ -1,33 +1,34 @@ -import { test, expect } from '../../utils/test-utils.js'; +import { test, expect, type Locator } from '../../utils/test-utils.js'; test.describe.configure({ mode: 'serial' }); -test('every libraries can be loaded', async ({ logedPage, librariesPage, page }) => { +test('every library can be loaded', async ({ logedPage, librariesPage, page }) => { test.slow(); await librariesPage.goto(); await librariesPage.hasUrl(); + const libraries: Locator[] = await page.locator('tbody tr td:nth-child(1)').all(); + const libraryNames: string[] = await Promise.all( + libraries.map(async (library) => await library.innerText()) + ); + let previousRemainingLibrary = ''; - let nextRemainingLibrary = await page.locator('tbody tr td:nth-child(1)').first()?.innerText(); - while (nextRemainingLibrary) { + let nextRemainingLibrary = libraryNames[0]; + for (let i = 1; i < libraryNames.length; i++) { await librariesPage.importLibrary(nextRemainingLibrary, undefined, 'any'); await librariesPage.tab('Libraries store').click(); expect(librariesPage.tab('Libraries store').getAttribute('aria-selected')).toBeTruthy(); previousRemainingLibrary = nextRemainingLibrary; - if ((await page.locator('tbody tr td:nth-child(1)').count()) !== 0) { - nextRemainingLibrary = await page.locator('tbody tr td:nth-child(1)').first()?.innerText(); - } else { - break; - } + nextRemainingLibrary = libraryNames[i]; expect( previousRemainingLibrary, 'An error occured while importing library: ' + previousRemainingLibrary - ).toEqual(nextRemainingLibrary); + ).not.toEqual(nextRemainingLibrary); } }); -test('every libraries can be deleted', async ({ logedPage, librariesPage, page }) => { +test('every library can be deleted', async ({ logedPage, librariesPage, page }) => { test.slow(); test.skip( true,