Update filtering logic and improve access tests for Bytes

Signed-off-by: Donny Peeters <[email protected]>

bytes/bytes/database/sql_meta_repository.py

@@ -96,14 +96,14 @@ def get_normalizer_meta_by_id(self, normalizer_meta_id: uuid.UUID) -> Normalizer
     def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> List[NormalizerMeta]:
         logger.debug("Querying normalizer meta: %s", query_filter.json())
 
+        query = self.session.query(NormalizerMetaInDB)
+
         if query_filter.raw_id is not None:
-            query = self.session.query(NormalizerMetaInDB).filter(
-                NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id)
-            )
-        else:
+            query = query.filter(NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id))
+
+        if query_filter.organization is not None:
             query = (
-                self.session.query(NormalizerMetaInDB)
-                .join(RawFileInDB)
+                query.join(RawFileInDB)
                 .join(BoefjeMetaInDB)
                 .filter(RawFileInDB.boefje_meta_id == BoefjeMetaInDB.id)
                 .filter(BoefjeMetaInDB.organization == query_filter.organization)
@@ -143,17 +143,13 @@ def save_raw(self, raw: RawData) -> uuid.UUID:
 
     def get_raw(self, query_filter: RawDataFilter) -> List[RawDataMeta]:
         logger.debug("Querying raw data: %s", query_filter.json())
+        query = self.session.query(RawFileInDB)
 
         if query_filter.boefje_meta_id:
-            query = self.session.query(RawFileInDB).filter(
-                RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id)
-            )
-        else:
-            query = (
-                self.session.query(RawFileInDB)
-                .join(BoefjeMetaInDB)
-                .filter(BoefjeMetaInDB.organization == query_filter.organization)
-            )
+            query = query.filter(RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id))
+
+        if query_filter.organization:
+            query = query.join(BoefjeMetaInDB).filter(BoefjeMetaInDB.organization == query_filter.organization)
 
         if query_filter.normalized:
             query = query.join(NormalizerMetaInDB, isouter=False)

bytes/bytes/repositories/meta_repository.py

@@ -1,7 +1,7 @@
-from typing import Any, Dict, List, Optional, Type
+from typing import Dict, List, Optional, Type
 from uuid import UUID
 
-from pydantic import BaseModel, Field, root_validator
+from pydantic import BaseModel, Field
 
 from bytes.models import BoefjeMeta, MimeType, NormalizerMeta, RawData, RawDataMeta
 
@@ -33,15 +33,6 @@ class RawDataFilter(BaseModel):
     limit: int = 1
     offset: int = 0
 
-    @root_validator(pre=False)
-    def either_organization_or_boefje_meta_id(  # pylint: disable=no-self-argument
-        cls, values: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        if values.get("organization") or values.get("boefje_meta_id"):
-            return values
-
-        raise ValueError("boefje_meta_id and organization cannot both be None.")
-
 
 class MetaDataRepository:
     def __enter__(self) -> None:

bytes/tests/conftest.py

@@ -84,11 +84,14 @@ def bytes_api_client(settings) -> Iterator[BytesAPIClient]:
     alembicArgs = ["--config", "/app/bytes/bytes/alembic.ini", "--raiseerr", "upgrade", "head"]
     alembic.config.main(argv=alembicArgs)
 
-    yield BytesAPIClient(
+    client = BytesAPIClient(
         "http://ci_bytes:8000",
         settings.username,
         settings.password,
     )
+    client.login()
+
+    yield client
 
     sessionmaker(bind=get_engine(settings.db_uri), autocommit=True)().execute(
         ";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables])

bytes/tests/integration/test_meta_repository.py

@@ -170,7 +170,35 @@ def test_filter_raw_on_organization(meta_repository: SQLMetaDataRepository) -> N
     assert len(meta_repository.get_raw(query_filter)) == 1
 
     query_filter.organization = "test2"
-    assert len(meta_repository.get_raw(query_filter)) == 1
+    assert len(meta_repository.get_raw(query_filter)) == 0
+
+
+def test_filter_raw_not_on_organization(meta_repository: SQLMetaDataRepository) -> None:
+    raw = get_raw_data()
+    raw2 = get_raw_data()
+    raw2.boefje_meta.id = "c5d7d1da-7d94-4ac4-b0f6-ac065eeb0c97"
+
+    with meta_repository:
+        meta_repository.save_boefje_meta(raw.boefje_meta)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_raw(raw)
+        meta_repository.save_boefje_meta(raw2.boefje_meta)
+        meta_repository.save_raw(raw2)
+        meta_repository.save_raw(raw2)
+
+    # Test offset-limit
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=6))) == 5
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=1))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=4))) == 1
+
+    # Test without boefje_meta id
+    assert len(meta_repository.get_raw(RawDataFilter(limit=2, offset=4))) == 2
+    assert len(meta_repository.get_raw(RawDataFilter(limit=100))) == 7
+    assert len(meta_repository.get_raw(RawDataFilter(limit=100, normalized=False))) == 7
 
 
 def test_filter_normalizer_meta(meta_repository: SQLMetaDataRepository) -> None:

bytes/tests/unit/test_raw_repository.py

@@ -1,12 +1,10 @@
 import uuid
 
 import pytest
-from pydantic import ValidationError
 
 from bytes.config import get_settings
 from bytes.raw.file_raw_repository import FileRawRepository
 from bytes.raw.middleware import NaclBoxMiddleware
-from bytes.repositories.meta_repository import RawDataFilter
 from tests.loading import get_raw_data
 
 
@@ -34,11 +32,3 @@ def test_nacl_middleware(nacl_middleware: NaclBoxMiddleware) -> None:
 
     assert encrypted != msg
     assert decrypted == msg
-
-
-def test_filter_validator() -> None:
-    with pytest.raises(ValidationError):
-        RawDataFilter()
-
-    RawDataFilter(organization="Not None")
-    RawDataFilter(boefje_meta_id="3a9f1b33-f703-4c32-b0ae-44d0473c8aa5")