Ca 294 refactor library management

.github/workflows/functional-tests.yml

@@ -27,15 +27,13 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       mailhog:
         image: mailhog/mailhog
-        ports: [
-          "1025:1025",
-          "8025:8025"
-        ]
+        ports: ["1025:1025", "8025:8025"]
 
     strategy:
       max-parallel: 4
       matrix:
         python-version: ["3.11"]
+        playwright-browser: ["chromium", "firefox"]
 
     steps:
       - uses: actions/checkout@v4
@@ -55,9 +53,9 @@ jobs:
       - name: Install dependencies
         working-directory: ${{ env.working-directory }}
         run: npm ci
-      - name: Install Playwright Browsers
+      - name: Install Playwright browser ${{ matrix.playwright-browser }}
         working-directory: ${{ env.working-directory }}
-        run: npx playwright install --with-deps
+        run: npx playwright install --with-deps ${{ matrix.playwright-browser }}
       - name: Create frontend environment variables file
         working-directory: ${{ env.working-directory }}
         run: |
@@ -93,14 +91,14 @@ jobs:
           export $(grep -v '^#' .env | xargs)
           python manage.py createsuperuser --noinput
           nohup python manage.py runserver &
-      - name: Run tests
+      - name: Run tests with browser ${{ matrix.playwright-browser }}
         working-directory: ${{ env.working-directory }}
-        run: npx playwright test
+        run: npx playwright test --project=${{ matrix.playwright-browser }}
       - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: functional-tests-report
           path: |
-            ${{ env.working-directory }}/tests/results/
-            ${{ env.working-directory }}/tests/reports/
+            ${{ env.working-directory }}/tests/${{ matrix.playwright-browser }}results/
+            ${{ env.working-directory }}/tests/${{ matrix.playwright-browser }}reports/
           retention-days: 5

.pre-commit-config.yaml

@@ -9,11 +9,12 @@ repos:
     rev: v0.4.1
     hooks:
       # Run the linter.
-      - id: ruff
-        args: [--fix backend]
+      # - id: ruff
+      #   args: [--fix, backend]
       # Run the formatter.
       - id: ruff-format
         args: [backend]
+        exclude: (migrations)
   - repo: local
     hooks:
       - id: format-frontend

backend/app_tests/api/test_api_libraries.py

@@ -1,6 +1,8 @@
+import json
 import pytest
 from rest_framework.test import APIClient
-from core.models import Framework
+from app_tests.test_vars import TEST_FRAMEWORK_URN, TEST_RISK_MATRIX_URN
+from core.models import Framework, StoredLibrary
 from core.models import RiskMatrix
 from iam.models import Folder
 from rest_framework import status
@@ -17,7 +19,7 @@ class TestLibrariesUnauthenticated:
     def test_get_libraries(self):
         """test to get libraries from the API without authentication"""
 
-        EndpointTestsQueries.get_object(self.client, "Libraries")
+        EndpointTestsQueries.get_object(self.client, "Stored libraries")
 
     def test_import_frameworks(self):
         """test to import libraries with the API without authentication"""
@@ -50,21 +52,27 @@ def test_get_libraries(self, test):
         """test to get libraries from the API with authentication"""
 
         EndpointTestsQueries.Auth.get_object(
-            test.client, "Libraries", base_count=-1, user_group=test.user_group
+            test.client, "Stored libraries", base_count=-1, user_group=test.user_group
         )
 
     def test_import_frameworks(self, test):
         """test to import frameworks with the API with authentication"""
 
         # Uses the API endpoint to get library details with the admin client
         lib_detail_response = test.admin_client.get(
-            EndpointTestsUtils.get_object_urn("Framework")
-        ).json()["objects"]["framework"]
+            EndpointTestsUtils.get_stored_library_content(
+                test.client, TEST_FRAMEWORK_URN
+            )
+        )
+        lib_detail_response = lib_detail_response.content
+        lib_detail_response = json.loads(lib_detail_response)
+        lib_detail_response = json.loads(lib_detail_response)
+        lib_detail_response = lib_detail_response["framework"]
 
-        # Asserts that the library is not already imported
+        # Asserts that the library is not already loaded
         assert (
             Framework.objects.all().count() == 0
-        ), "libraries are already imported in the database"
+        ), "libraries are already loaded in the database"
         EndpointTestsQueries.Auth.get_object(
             test.client,
             "Frameworks",
@@ -78,12 +86,12 @@ def test_import_frameworks(self, test):
         assert Framework.objects.all().count() == (
             1
             if not EndpointTestsUtils.expected_request_response(
-                "add", "library", str(test.folder), test.user_group
+                "add", "loadedlibrary", str(test.folder), test.user_group
             )[0]
             else 0
         ), "Frameworks are not correctly imported in the database"
 
-        # Uses the API endpoint to assert that the library was properly imported
+        # Uses the API endpoint to assert that the library was properly loaded
         EndpointTestsQueries.Auth.get_object(
             test.client,
             "Frameworks",
@@ -96,7 +104,7 @@ def test_import_frameworks(self, test):
             base_count=1,
             user_group=test.user_group,
             fails=EndpointTestsUtils.expected_request_response(
-                "add", "library", str(test.folder), test.user_group
+                "add", "loadedlibrary", str(test.folder), test.user_group
             )[0],
         )
 
@@ -121,13 +129,19 @@ def test_import_risk_matrix(self, test):
 
         # Uses the API endpoint to get library details with the admin client
         lib_detail_response = test.admin_client.get(
-            EndpointTestsUtils.get_object_urn("Risk matrix")
-        ).json()["objects"]["risk_matrix"][0]
+            EndpointTestsUtils.get_stored_library_content(
+                test.client, TEST_RISK_MATRIX_URN
+            )
+        )
+        lib_detail_response = lib_detail_response.content
+        lib_detail_response = json.loads(lib_detail_response)
+        lib_detail_response = json.loads(lib_detail_response)
+        lib_detail_response = lib_detail_response["risk_matrix"][0]
 
-        # Asserts that the library is not already imported
+        # Asserts that the library is not already loaded
         assert (
             RiskMatrix.objects.all().count() == 0
-        ), "libraries are already imported in the database"
+        ), "libraries are already loaded in the database"
         EndpointTestsQueries.Auth.get_object(
             test.client, "Risk matrices", user_group=test.user_group
         )
@@ -139,12 +153,12 @@ def test_import_risk_matrix(self, test):
         assert RiskMatrix.objects.all().count() == (
             1
             if not EndpointTestsUtils.expected_request_response(
-                "add", "library", str(test.folder), test.user_group
+                "add", "loadedlibrary", str(test.folder), test.user_group
             )[0]
             else 0
         ), "Risk matrices are not correctly imported in the database"
 
-        # Uses the API endpoint to assert that the library was properly imported
+        # Uses the API endpoint to assert that the library was properly loaded
         EndpointTestsQueries.Auth.get_object(
             test.client,
             "Risk matrices",
@@ -158,7 +172,7 @@ def test_import_risk_matrix(self, test):
             base_count=1,
             user_group=test.user_group,
             fails=EndpointTestsUtils.expected_request_response(
-                "add", "library", str(test.folder), test.user_group
+                "add", "loadedlibrary", str(test.folder), test.user_group
             )[0],
         )
 

backend/app_tests/api/test_api_requirement_nodes.py

@@ -1,6 +1,7 @@
 import pytest
 from rest_framework.test import APIClient
-from core.models import RequirementNode, Framework
+from app_tests.test_vars import TEST_FRAMEWORK_URN
+from core.models import RequirementNode, Framework, StoredLibrary
 from iam.models import Folder
 
 from test_utils import EndpointTestsQueries, EndpointTestsUtils
@@ -76,7 +77,9 @@ def test_import_requirement_nodes(self, test):
             test.client,
             "Requirement nodes",
             EndpointTestsUtils.get_endpoint_url("Requirement nodes"),
-            EndpointTestsUtils.get_object_urn("Framework"),
+            EndpointTestsUtils.get_stored_library_content(
+                test.client, TEST_FRAMEWORK_URN
+            ),
             [
                 "name",
                 "description",

backend/app_tests/api/test_utils.py

@@ -1,3 +1,4 @@
+from django.db import models
 from knox.auth import AuthToken
 import pytest
 import json
@@ -6,6 +7,7 @@
 from django.urls import reverse
 from rest_framework import status
 from rest_framework.test import APIClient
+from core.models import StoredLibrary
 
 from test_vars import *
 
@@ -25,7 +27,17 @@ def get_object_urn(object_name: str, resolved: bool = True):
 
         urn_varname = format_urn(object_name)
         urn = get_var(urn_varname)
-        return f"{reverse(LIBRARIES_ENDPOINT)}{urn}/" if resolved else eval(urn)
+        return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/" if resolved else eval(urn)
+
+    def get_referential_object_url_from_urn(
+        authenticated_client, urn: str, model: models.Model = StoredLibrary
+    ):
+        """Get the object URL from the URN"""
+        return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/"
+
+    def get_stored_library_content(authenticated_client, urn: str) -> str:
+        """Return an URL to fetch the content of a stored library"""
+        return f"{reverse(STORED_LIBRARIES_ENDPOINT)}{urn}/content/"
 
     @pytest.mark.django_db
     def get_test_client_and_folder(
@@ -958,7 +970,7 @@ def import_object(
                     user_perm_expected_status,
                     user_perm_reason,
                 ) = EndpointTestsUtils.expected_request_response(
-                    "add", "library", scope, user_group, expected_status
+                    "add", "loadedlibrary", scope, user_group, expected_status
                 )
 
             url = urn or EndpointTestsUtils.get_object_urn(verbose_name)
@@ -1014,9 +1026,12 @@ def compare_results(
                 reference.status_code == status.HTTP_200_OK
             ), "reference endpoint is not accessible"
 
-            for object in reference.json()["objects"]["framework"][
-                object_name.lower().replace(" ", "_")
-            ][:count]:
+            content = json.loads(reference.content)
+            content = json.loads(content)
+
+            for object in content["framework"][object_name.lower().replace(" ", "_")][
+                :count
+            ]:
                 comparelist = authenticated_client.get(compare_url)
                 compare = dict()
                 assert (

backend/app_tests/conftest.py

@@ -15,7 +15,7 @@ def __init__(self, *args, **kwargs):
 
 @pytest.fixture
 def app_config():
-    startup()
+    startup(sender=None, **{})
 
 
 @pytest.fixture

backend/app_tests/test_vars.py

@@ -13,7 +13,7 @@
 EVIDENCES_ENDPOINT = "evidences-list"
 FOLDERS_ENDPOINT = "folders-list"
 FRAMEWORKS_ENDPOINT = "frameworks-list"
-LIBRARIES_ENDPOINT = "libraries-list"
+STORED_LIBRARIES_ENDPOINT = "stored-libraries-list"
 RISK_MATRICES_ENDPOINT = "risk-matrices-list"
 PROJECTS_ENDPOINT = "projects-list"
 REQUIREMENT_ASSESSMENTS_ENDPOINT = "requirement-assessments-list"
@@ -95,7 +95,7 @@ def get_var(varname: str) -> Any:
 
 def get_singular_name(plural_name: str) -> str:
     exceptions = {
-        "Libraries": "Library",
+        "Stored libraries": "Stored library",
         "Risk matrices": "Risk matrix",
         "Policies": "Policy",
     }

backend/ciso_assistant/settings.py

@@ -280,7 +280,7 @@ def set_ciso_assistant_url(_, __, event_dict):
 
 # SQLIte file can be changed, useful for tests
 SQLITE_FILE = os.environ.get("SQLITE_FILE", BASE_DIR / "db/ciso-assistant.sqlite3")
-
+LIBRARIES_PATH = library_path = BASE_DIR / "library/libraries"
 
 if "POSTGRES_NAME" in os.environ:
     DATABASES = {