diff --git a/bytes/bytes/database/sql_meta_repository.py b/bytes/bytes/database/sql_meta_repository.py index 8806fff6f05..25a36acf174 100644 --- a/bytes/bytes/database/sql_meta_repository.py +++ b/bytes/bytes/database/sql_meta_repository.py @@ -96,14 +96,14 @@ def get_normalizer_meta_by_id(self, normalizer_meta_id: uuid.UUID) -> Normalizer def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> List[NormalizerMeta]: logger.debug("Querying normalizer meta: %s", query_filter.json()) + query = self.session.query(NormalizerMetaInDB) + if query_filter.raw_id is not None: - query = self.session.query(NormalizerMetaInDB).filter( - NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id) - ) - else: + query = query.filter(NormalizerMetaInDB.raw_file_id == str(query_filter.raw_id)) + + if query_filter.organization is not None: query = ( - self.session.query(NormalizerMetaInDB) - .join(RawFileInDB) + query.join(RawFileInDB) .join(BoefjeMetaInDB) .filter(RawFileInDB.boefje_meta_id == BoefjeMetaInDB.id) .filter(BoefjeMetaInDB.organization == query_filter.organization) @@ -143,17 +143,13 @@ def save_raw(self, raw: RawData) -> uuid.UUID: def get_raw(self, query_filter: RawDataFilter) -> List[RawDataMeta]: logger.debug("Querying raw data: %s", query_filter.json()) + query = self.session.query(RawFileInDB) if query_filter.boefje_meta_id: - query = self.session.query(RawFileInDB).filter( - RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id) - ) - else: - query = ( - self.session.query(RawFileInDB) - .join(BoefjeMetaInDB) - .filter(BoefjeMetaInDB.organization == query_filter.organization) - ) + query = query.filter(RawFileInDB.boefje_meta_id == str(query_filter.boefje_meta_id)) + + if query_filter.organization: + query = query.join(BoefjeMetaInDB).filter(BoefjeMetaInDB.organization == query_filter.organization) if query_filter.normalized: query = query.join(NormalizerMetaInDB, isouter=False) diff --git a/bytes/bytes/repositories/meta_repository.py b/bytes/bytes/repositories/meta_repository.py index 4f5df354ae3..5b4a9332b70 100644 --- a/bytes/bytes/repositories/meta_repository.py +++ b/bytes/bytes/repositories/meta_repository.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, List, Optional, Type +from typing import Dict, List, Optional, Type from uuid import UUID -from pydantic import BaseModel, Field, root_validator +from pydantic import BaseModel, Field from bytes.models import BoefjeMeta, MimeType, NormalizerMeta, RawData, RawDataMeta @@ -33,15 +33,6 @@ class RawDataFilter(BaseModel): limit: int = 1 offset: int = 0 - @root_validator(pre=False) - def either_organization_or_boefje_meta_id( # pylint: disable=no-self-argument - cls, values: Dict[str, Any] - ) -> Dict[str, Any]: - if values.get("organization") or values.get("boefje_meta_id"): - return values - - raise ValueError("boefje_meta_id and organization cannot both be None.") - class MetaDataRepository: def __enter__(self) -> None: diff --git a/bytes/tests/conftest.py b/bytes/tests/conftest.py index d1eb020904a..6f704da77cb 100644 --- a/bytes/tests/conftest.py +++ b/bytes/tests/conftest.py @@ -84,11 +84,14 @@ def bytes_api_client(settings) -> Iterator[BytesAPIClient]: alembicArgs = ["--config", "/app/bytes/bytes/alembic.ini", "--raiseerr", "upgrade", "head"] alembic.config.main(argv=alembicArgs) - yield BytesAPIClient( + client = BytesAPIClient( "http://ci_bytes:8000", settings.username, settings.password, ) + client.login() + + yield client sessionmaker(bind=get_engine(settings.db_uri), autocommit=True)().execute( ";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables]) diff --git a/bytes/tests/integration/test_meta_repository.py b/bytes/tests/integration/test_meta_repository.py index 494b6c41581..bb5b85efcb1 100644 --- a/bytes/tests/integration/test_meta_repository.py +++ b/bytes/tests/integration/test_meta_repository.py @@ -170,7 +170,35 @@ def test_filter_raw_on_organization(meta_repository: SQLMetaDataRepository) -> N assert len(meta_repository.get_raw(query_filter)) == 1 query_filter.organization = "test2" - assert len(meta_repository.get_raw(query_filter)) == 1 + assert len(meta_repository.get_raw(query_filter)) == 0 + + +def test_filter_raw_not_on_organization(meta_repository: SQLMetaDataRepository) -> None: + raw = get_raw_data() + raw2 = get_raw_data() + raw2.boefje_meta.id = "c5d7d1da-7d94-4ac4-b0f6-ac065eeb0c97" + + with meta_repository: + meta_repository.save_boefje_meta(raw.boefje_meta) + meta_repository.save_raw(raw) + meta_repository.save_raw(raw) + meta_repository.save_raw(raw) + meta_repository.save_raw(raw) + meta_repository.save_raw(raw) + meta_repository.save_boefje_meta(raw2.boefje_meta) + meta_repository.save_raw(raw2) + meta_repository.save_raw(raw2) + + # Test offset-limit + assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2))) == 2 + assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=6))) == 5 + assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=1))) == 2 + assert len(meta_repository.get_raw(RawDataFilter(boefje_meta_id=raw.boefje_meta.id, limit=2, offset=4))) == 1 + + # Test without boefje_meta id + assert len(meta_repository.get_raw(RawDataFilter(limit=2, offset=4))) == 2 + assert len(meta_repository.get_raw(RawDataFilter(limit=100))) == 7 + assert len(meta_repository.get_raw(RawDataFilter(limit=100, normalized=False))) == 7 def test_filter_normalizer_meta(meta_repository: SQLMetaDataRepository) -> None: diff --git a/bytes/tests/unit/test_raw_repository.py b/bytes/tests/unit/test_raw_repository.py index 8ab047bb97e..cf49357ee22 100644 --- a/bytes/tests/unit/test_raw_repository.py +++ b/bytes/tests/unit/test_raw_repository.py @@ -1,12 +1,10 @@ import uuid import pytest -from pydantic import ValidationError from bytes.config import get_settings from bytes.raw.file_raw_repository import FileRawRepository from bytes.raw.middleware import NaclBoxMiddleware -from bytes.repositories.meta_repository import RawDataFilter from tests.loading import get_raw_data @@ -34,11 +32,3 @@ def test_nacl_middleware(nacl_middleware: NaclBoxMiddleware) -> None: assert encrypted != msg assert decrypted == msg - - -def test_filter_validator() -> None: - with pytest.raises(ValidationError): - RawDataFilter() - - RawDataFilter(organization="Not None") - RawDataFilter(boefje_meta_id="3a9f1b33-f703-4c32-b0ae-44d0473c8aa5")