From 53d8e8b425af3d6c5dbecd6fea207d33d7b25d13 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 7 Mar 2022 15:38:40 +0100 Subject: [PATCH 01/23] precompute images --- .../annotations_image_extractor.py | 140 ++---------- .../application/annotations_schema.py | 11 + .../application/annotations_service.py | 112 +++++++++- ebl/fragmentarium/domain/annotation.py | 13 +- ebl/fragmentarium/web/bootstrap.py | 2 + ebl/signs/web/bootstrap.py | 8 +- ebl/signs/web/cropped_annotation_schema.py | 2 +- ebl/signs/web/sign_images.py | 6 +- ebl/tests/factories/annotation.py | 15 ++ .../test_annotations_image_extractor.py | 131 ++--------- .../fragmentarium/test_annotations_service.py | 205 +++++++++++++++++- 11 files changed, 384 insertions(+), 261 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_image_extractor.py b/ebl/fragmentarium/application/annotations_image_extractor.py index 6e0c849a9..628b14733 100644 --- a/ebl/fragmentarium/application/annotations_image_extractor.py +++ b/ebl/fragmentarium/application/annotations_image_extractor.py @@ -1,138 +1,36 @@ -import base64 -import io -from singledispatchmethod import singledispatchmethod -from io import BytesIO -from typing import Sequence, NewType +from typing import Sequence -import attr -from PIL import Image - -from ebl.files.application.file_repository import FileRepository from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository -from ebl.fragmentarium.application.fragment_repository import FragmentRepository -from ebl.fragmentarium.domain.annotation import BoundingBox, Annotation +from ebl.fragmentarium.domain.annotation import CroppedAnnotationImage from ebl.fragmentarium.domain.museum_number import MuseumNumber -from ebl.transliteration.domain.line_label import LineLabel -from ebl.transliteration.domain.line_number import ( - AbstractLineNumber, - LineNumber, - LineNumberRange, -) - -Base64 = NewType("Base64", str) +import attr @attr.attrs(auto_attribs=True, frozen=True) -class CroppedAnnotation: - image: Base64 +class CroppedAnnotation(CroppedAnnotationImage): fragment_number: MuseumNumber - script: str - label: str -class AnnotationImageExtractor: +class AnnotationCroppedImageService: def __init__( self, - fragment_repository: FragmentRepository, annotations_repository: AnnotationsRepository, - photos_repository: FileRepository, ): - self._fragments_repository = fragment_repository self._annotations_repository = annotations_repository - self._photos_repository = photos_repository - - def _format_label(self, label: LineLabel) -> str: - line_number = label.line_number - column = label.column - surface = label.surface - object = label.object - line_atf = line_number.atf if line_number else "" - column_abbr = column.abbreviation if column else "" - surface_abbr = surface.abbreviation if surface else "" - object_abbr = object.abbreviation if object else "" - return " ".join( - filter( - bool, - [column_abbr, surface_abbr, object_abbr, line_atf.replace(".", "")], - ) - ) - - def _cropped_image_from_annotation( - self, annotation: Annotation, fragment_number: MuseumNumber - ) -> Base64: - fragment_image = self._photos_repository.query_by_file_name( - f"{fragment_number}.jpg" - ) - image_bytes = fragment_image.read() - image = Image.open(BytesIO(image_bytes), mode="r") - bounding_box = BoundingBox.from_annotations( - image.size[0], image.size[1], [annotation] - )[0] - area = ( - bounding_box.top_left_x, - bounding_box.top_left_y, - bounding_box.top_left_x + bounding_box.width, - bounding_box.top_left_y + bounding_box.height, - ) - cropped_image = image.crop(area) - buf = io.BytesIO() - cropped_image.save(buf, format="PNG") - return Base64(base64.b64encode(buf.getvalue()).decode("utf-8")) - - @singledispatchmethod - def _is_matching_number(self, line_number: AbstractLineNumber, number: int) -> bool: - raise ValueError("No default for overloading") - @_is_matching_number.register(LineNumber) - def _(self, line_number: LineNumber, number: int): - return number == line_number.number - - @_is_matching_number.register(LineNumberRange) # pyre-ignore[56] - def _(self, line_number: LineNumberRange, number: int): - return line_number.start.number <= number <= line_number.end.number - - def _cropped_image_from_annotations( - self, fragment_number: MuseumNumber, annotations: Sequence[Annotation] - ) -> Sequence[CroppedAnnotation]: - cropped_annotations = [] + def find_annotations_by_sign(self, sign: str) -> Sequence[CroppedAnnotation]: + annotations = self._annotations_repository.find_by_sign(sign) + cropped_image_annotations = [] for annotation in annotations: - fragment = self._fragments_repository.query_by_museum_number( - fragment_number - ) - script = fragment.script - labels = fragment.text.labels - label = next( - ( - label - for label in labels - if self._is_matching_number( - label.line_number, annotation.data.path[0] + for annotation_elem in annotation.annotations: + image = annotation_elem.image + if image: + cropped_image_annotations.append( + CroppedAnnotation( + image.image, + image.script, + image.label, + annotation.fragment_number, + ) ) - ), - None, - ) - - cropped_image = self._cropped_image_from_annotation( - annotation, fragment_number - ) - cropped_annotations.append( - CroppedAnnotation( - cropped_image, - fragment_number, - script, - self._format_label(label) if label else "", - ) - ) - return cropped_annotations - - def cropped_images_from_sign(self, sign: str) -> Sequence[CroppedAnnotation]: - annotations = self._annotations_repository.find_by_sign(sign) - cropped_annotations = [] - for single_annotation in annotations: - fragment_number = single_annotation.fragment_number - cropped_annotations.extend( - self._cropped_image_from_annotations( - fragment_number, single_annotation.annotations - ) - ) - return cropped_annotations + return cropped_image_annotations diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 221eda559..dbfd1ed67 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -34,9 +34,20 @@ def make_data(self, data, **kwargs): return AnnotationData(**data) +class CroppedAnnotationImageSchema(Schema): + image = fields.Str(required=True) + script = fields.Str(required=True) + label = fields.Str(required=True) + + @post_load + def make_cropped_annotation_image(self, data, **kwargs): + return CroppedAnnotationImageSchema(**data) + + class AnnotationSchema(Schema): geometry = fields.Nested(GeometrySchema(), required=True) data = fields.Nested(AnnotationDataSchema(), required=True) + image = fields.Nested(CroppedAnnotationImageSchema(), missing=None) @post_load def make_annotation(self, data, **kwargs): diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 31f1c0e44..0863c9528 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -1,11 +1,31 @@ +import base64 +import io +from singledispatchmethod import singledispatchmethod +from io import BytesIO + +import attr +from PIL import Image + from ebl.changelog import Changelog from ebl.ebl_ai_client import EblAiClient -import attr from ebl.files.application.file_repository import FileRepository from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema -from ebl.fragmentarium.domain.annotation import Annotations +from ebl.fragmentarium.application.fragment_repository import FragmentRepository +from ebl.fragmentarium.domain.annotation import ( + Annotations, + Annotation, + BoundingBox, + Base64, + CroppedAnnotationImage, +) from ebl.fragmentarium.domain.museum_number import MuseumNumber +from ebl.transliteration.domain.line_label import LineLabel +from ebl.transliteration.domain.line_number import ( + LineNumber, + LineNumberRange, + AbstractLineNumber, +) from ebl.users.domain.user import User @@ -15,6 +35,8 @@ class AnnotationsService: _annotations_repository: AnnotationsRepository _photo_repository: FileRepository _changelog: Changelog + _fragments_repository: FragmentRepository + _photos_repository: FileRepository def generate_annotations( self, number: MuseumNumber, threshold: float = 0.3 @@ -27,6 +49,87 @@ def generate_annotations( def find(self, number: MuseumNumber) -> Annotations: return self._annotations_repository.query_by_museum_number(number) + def _format_label(self, label: LineLabel) -> str: + line_number = label.line_number + column = label.column + surface = label.surface + object = label.object + line_atf = line_number.atf if line_number else "" + column_abbr = column.abbreviation if column else "" + surface_abbr = surface.abbreviation if surface else "" + object_abbr = object.abbreviation if object else "" + return " ".join( + filter( + bool, + [column_abbr, surface_abbr, object_abbr, line_atf.replace(".", "")], + ) + ) + + def _cropped_image_from_annotation( + self, annotation: Annotation, image: Image.Image + ) -> Base64: + bounding_box = BoundingBox.from_annotations( + image.size[0], image.size[1], [annotation] + )[0] + area = ( + bounding_box.top_left_x, + bounding_box.top_left_y, + bounding_box.top_left_x + bounding_box.width, + bounding_box.top_left_y + bounding_box.height, + ) + cropped_image = image.crop(area) + buf = io.BytesIO() + cropped_image.save(buf, format="PNG") + return Base64(base64.b64encode(buf.getvalue()).decode("utf-8")) + + @singledispatchmethod + def _is_matching_number(self, line_number: AbstractLineNumber, number: int) -> bool: + raise ValueError("No default for overloading") + + @_is_matching_number.register(LineNumber) + def _(self, line_number: LineNumber, number: int): + return number == line_number.number + + @_is_matching_number.register(LineNumberRange) # pyre-ignore[56] + def _(self, line_number: LineNumberRange, number: int): + return line_number.start.number <= number <= line_number.end.number + + def _cropped_image_from_annotations(self, annotations: Annotations) -> Annotations: + cropped_annotations = [] + fragment = self._fragments_repository.query_by_museum_number( + annotations.fragment_number + ) + fragment_image = self._photos_repository.query_by_file_name( + f"{annotations.fragment_number}.jpg" + ) + image_bytes = fragment_image.read() + image = Image.open(BytesIO(image_bytes), mode="r") + for annotation in annotations.annotations: + script = fragment.script + labels = fragment.text.labels + label = next( + ( + label + for label in labels + if self._is_matching_number( + label.line_number, annotation.data.path[0] + ) + ), + None, + ) + cropped_image = self._cropped_image_from_annotation(annotation, image) + cropped_annotations.append( + attr.evolve( + annotation, + image=CroppedAnnotationImage( + cropped_image, + script, + self._format_label(label) if label else "", + ), + ) + ) + return attr.evolve(annotations, annotations=cropped_annotations) + def update(self, annotations: Annotations, user: User) -> Annotations: old_annotations = self._annotations_repository.query_by_museum_number( annotations.fragment_number @@ -39,5 +142,6 @@ def update(self, annotations: Annotations, user: User) -> Annotations: {"_id": _id, **schema.dump(old_annotations)}, {"_id": _id, **schema.dump(annotations)}, ) - self._annotations_repository.create_or_update(annotations) - return annotations + annotations_with_image = self._cropped_image_from_annotations(annotations) + self._annotations_repository.create_or_update(annotations_with_image) + return annotations_with_image diff --git a/ebl/fragmentarium/domain/annotation.py b/ebl/fragmentarium/domain/annotation.py index 647c831a9..817223071 100644 --- a/ebl/fragmentarium/domain/annotation.py +++ b/ebl/fragmentarium/domain/annotation.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Sequence +from typing import Sequence, Optional, NewType from uuid import uuid4 import attr @@ -34,10 +34,21 @@ class AnnotationData: sign_name: str +Base64 = NewType("Base64", str) + + +@attr.attrs(auto_attribs=True, frozen=True) +class CroppedAnnotationImage: + image: Base64 + script: str + label: str + + @attr.attrs(auto_attribs=True, frozen=True) class Annotation: geometry: Geometry data: AnnotationData + image: Optional[CroppedAnnotationImage] = None @classmethod def from_prediction(cls, geometry: Geometry) -> "Annotation": diff --git a/ebl/fragmentarium/web/bootstrap.py b/ebl/fragmentarium/web/bootstrap.py index 8752faf96..89c26ffa5 100644 --- a/ebl/fragmentarium/web/bootstrap.py +++ b/ebl/fragmentarium/web/bootstrap.py @@ -38,6 +38,8 @@ def create_fragmentarium_routes(api: falcon.App, context: Context): context.annotations_repository, context.photo_repository, context.changelog, + context.fragment_repository, + context.photo_repository, ) statistics = make_statistics_resource(context.cache, fragmentarium) diff --git a/ebl/signs/web/bootstrap.py b/ebl/signs/web/bootstrap.py index 3223b952e..9d1debfd5 100644 --- a/ebl/signs/web/bootstrap.py +++ b/ebl/signs/web/bootstrap.py @@ -2,7 +2,7 @@ from ebl.context import Context from ebl.fragmentarium.application.annotations_image_extractor import ( - AnnotationImageExtractor, + AnnotationCroppedImageService, ) from ebl.signs.web.sign_search import SignsSearch from ebl.signs.web.signs import SignsResource @@ -13,11 +13,7 @@ def create_signs_routes(api: falcon.App, context: Context): signs_search = SignsSearch(context.sign_repository) signs = SignsResource(context.sign_repository) signs_images = SignsImageResource( - AnnotationImageExtractor( - context.fragment_repository, - context.annotations_repository, - context.photo_repository, - ) + AnnotationCroppedImageService(context.annotations_repository) ) api.add_route("/signs", signs_search) api.add_route("/signs/{sign_name}", signs) diff --git a/ebl/signs/web/cropped_annotation_schema.py b/ebl/signs/web/cropped_annotation_schema.py index 79d186874..487ab858e 100644 --- a/ebl/signs/web/cropped_annotation_schema.py +++ b/ebl/signs/web/cropped_annotation_schema.py @@ -3,6 +3,6 @@ class CroppedAnnotationSchema(Schema): image = fields.String(required=True) - fragment_number = fields.String(required=True, data_key="fragmentNumber") + fragment_number = fields.String(required=True) script = fields.String(required=True) label = fields.String(required=True) diff --git a/ebl/signs/web/sign_images.py b/ebl/signs/web/sign_images.py index 04e34ea38..f5d699fee 100644 --- a/ebl/signs/web/sign_images.py +++ b/ebl/signs/web/sign_images.py @@ -1,17 +1,17 @@ import falcon from ebl.fragmentarium.application.annotations_image_extractor import ( - AnnotationImageExtractor, + AnnotationCroppedImageService, ) from ebl.signs.web.cropped_annotation_schema import CroppedAnnotationSchema from ebl.users.web.require_scope import require_scope class SignsImageResource: - def __init__(self, annotation_image_extractor: AnnotationImageExtractor): + def __init__(self, annotation_image_extractor: AnnotationCroppedImageService): self._image_extractor = annotation_image_extractor @falcon.before(require_scope, "read:words") def on_get(self, _req, resp, sign_name): - cropped_signs = self._image_extractor.cropped_images_from_sign(sign_name) + cropped_signs = self._image_extractor.find_annotations_by_sign(sign_name) resp.media = CroppedAnnotationSchema().dump(cropped_signs, many=True) diff --git a/ebl/tests/factories/annotation.py b/ebl/tests/factories/annotation.py index f4d17a0bb..314d490fa 100644 --- a/ebl/tests/factories/annotation.py +++ b/ebl/tests/factories/annotation.py @@ -6,6 +6,7 @@ Annotations, Geometry, AnnotationValueType, + CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber @@ -43,6 +44,20 @@ class Meta: geometry = factory.SubFactory(GeometryFactory) data = factory.SubFactory(AnnotationDataFactory) + image = None + + +class CroppedImageFactory(factory.Factory): + class Meta: + model = CroppedAnnotationImage + + image = factory.Faker("word") + script = factory.Faker("word") + label = factory.Faker("word") + + +class AnnotationFactoryWithImage(AnnotationFactory): + image = factory.SubFactory(CroppedImageFactory) class AnnotationsFactory(factory.Factory): diff --git a/ebl/tests/fragmentarium/test_annotations_image_extractor.py b/ebl/tests/fragmentarium/test_annotations_image_extractor.py index 46e4d139a..9266c24b6 100644 --- a/ebl/tests/fragmentarium/test_annotations_image_extractor.py +++ b/ebl/tests/fragmentarium/test_annotations_image_extractor.py @@ -1,124 +1,27 @@ -import pytest - from ebl.fragmentarium.application.annotations_image_extractor import ( - AnnotationImageExtractor, + AnnotationCroppedImageService, CroppedAnnotation, ) -from ebl.tests.conftest import create_test_photo from ebl.tests.factories.annotation import ( AnnotationsFactory, - AnnotationFactory, - AnnotationDataFactory, -) -from ebl.tests.factories.fragment import TransliteratedFragmentFactory -from ebl.transliteration.domain import atf -from ebl.transliteration.domain.labels import ColumnLabel, SurfaceLabel, ObjectLabel -from ebl.transliteration.domain.line_label import LineLabel -from ebl.transliteration.domain.line_number import LineNumber, LineNumberRange - - -@pytest.mark.parametrize( - "line_label, expected", - [ - ( - LineLabel( - ColumnLabel.from_int(1), - SurfaceLabel([], atf.Surface.SURFACE, "Stone wig"), - ObjectLabel([], atf.Object.OBJECT, "Stone wig"), - LineNumber(2), - ), - "i Stone wig Stone wig 2", - ), - ( - LineLabel( - None, None, None, LineNumberRange(LineNumber(1, True), LineNumber(3)) - ), - "1'-3", - ), - ], -) -def test_format_line_label( - line_label, expected, annotations_repository, photo_repository, fragment_repository -): - image_extractor = AnnotationImageExtractor( - fragment_repository, annotations_repository, photo_repository - ) - assert image_extractor._format_label(line_label) == expected - - -@pytest.mark.parametrize( - "line_label, line_number, expected", - [ - ( - LineNumber(2), - 2, - True, - ), - ( - LineNumber(2), - 1, - False, - ), - ( - LineNumberRange(LineNumber(1, True), LineNumber(3)), - 2, - True, - ), - ( - LineNumberRange(LineNumber(1, True), LineNumber(3)), - 4, - False, - ), - ], + AnnotationFactoryWithImage, ) -def test_line_label_match_line_number( - line_label, - line_number, - expected, - fragment_repository, - annotations_repository, - photo_repository, -): - image_extractor = AnnotationImageExtractor( - fragment_repository, annotations_repository, photo_repository - ) - assert image_extractor._is_matching_number(line_label, line_number) == expected -def test_cropped_images_from_sign( - annotations_repository, - photo_repository, - fragment_repository, - when, - text_with_labels, -): +def test_find_annotations_by_sign(annotations_repository, when): + service = AnnotationCroppedImageService(annotations_repository) + annotation = AnnotationFactoryWithImage.build_batch(2) + annotations = [AnnotationsFactory.build(annotations=annotation)] - image_extractor = AnnotationImageExtractor( - fragment_repository, annotations_repository, photo_repository - ) - - single_annotation = AnnotationFactory.build( - data=AnnotationDataFactory.build(path=[2, 0, 0]) - ) - annotation = AnnotationsFactory.build(annotations=[single_annotation]) - sign = "test-sign" - - fragment = TransliteratedFragmentFactory.build(text=text_with_labels) - (when(annotations_repository).find_by_sign(sign).thenReturn([annotation])) - ( - when(fragment_repository) - .query_by_museum_number(annotation.fragment_number) - .thenReturn(fragment) - ) - ( - when(photo_repository) - .query_by_file_name(f"{annotation.fragment_number}.jpg") - .thenReturn(create_test_photo("K.2")) + when(annotations_repository).find_by_sign("test-sign").thenReturn(annotations) + assert service.find_annotations_by_sign("test-sign") == list( + map( + lambda x: CroppedAnnotation( + x.image.image, + x.image.script, + x.image.label, + annotations[0].fragment_number, + ), + annotation, + ) ) - - result = image_extractor.cropped_images_from_sign(sign) - assert len(result) > 0 - first_cropped_annotation = result[0] - assert isinstance(first_cropped_annotation, CroppedAnnotation) - assert first_cropped_annotation.script == fragment.script - assert first_cropped_annotation.label == "i Stone wig Stone wig 2" diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index 6ca9d21d4..db8ec2e09 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -3,14 +3,151 @@ from ebl.fragmentarium.application.annotations_service import AnnotationsService from ebl.fragmentarium.domain.annotation import Annotations from ebl.fragmentarium.domain.museum_number import MuseumNumber + + +import pytest + from ebl.tests.conftest import create_test_photo -from ebl.tests.factories.annotation import AnnotationsFactory +from ebl.tests.factories.annotation import ( + AnnotationsFactory, + AnnotationFactory, + AnnotationDataFactory, +) +from ebl.tests.factories.fragment import TransliteratedFragmentFactory +from ebl.transliteration.domain import atf +from ebl.transliteration.domain.labels import ColumnLabel, SurfaceLabel, ObjectLabel +from ebl.transliteration.domain.line_label import LineLabel +from ebl.transliteration.domain.line_number import LineNumber, LineNumberRange SCHEMA = AnnotationsSchema() +@pytest.mark.parametrize( + "line_label, expected", + [ + ( + LineLabel( + ColumnLabel.from_int(1), + SurfaceLabel([], atf.Surface.SURFACE, "Stone wig"), + ObjectLabel([], atf.Object.OBJECT, "Stone wig"), + LineNumber(2), + ), + "i Stone wig Stone wig 2", + ), + ( + LineLabel( + None, None, None, LineNumberRange(LineNumber(1, True), LineNumber(3)) + ), + "1'-3", + ), + ], +) +def test_format_line_label( + line_label, + expected, + annotations_repository, + photo_repository, + fragment_repository, + changelog, +): + service = AnnotationsService( + EblAiClient(""), + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, + ) + assert service._format_label(line_label) == expected + + +@pytest.mark.parametrize( + "line_label, line_number, expected", + [ + ( + LineNumber(2), + 2, + True, + ), + ( + LineNumber(2), + 1, + False, + ), + ( + LineNumberRange(LineNumber(1, True), LineNumber(3)), + 2, + True, + ), + ( + LineNumberRange(LineNumber(1, True), LineNumber(3)), + 4, + False, + ), + ], +) +def test_line_label_match_line_number( + line_label, + line_number, + expected, + fragment_repository, + changelog, + annotations_repository, + photo_repository, +): + service = AnnotationsService( + EblAiClient(""), + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, + ) + assert service._is_matching_number(line_label, line_number) == expected + + +def test_cropped_images_from_sign( + annotations_repository, + photo_repository, + changelog, + fragment_repository, + when, + text_with_labels, +): + service = AnnotationsService( + EblAiClient(""), + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, + ) + + single_annotation = AnnotationFactory.build( + data=AnnotationDataFactory.build(path=[2, 0, 0]) + ) + annotation = AnnotationsFactory.build(annotations=[single_annotation]) + + fragment = TransliteratedFragmentFactory.build(text=text_with_labels) + ( + when(fragment_repository) + .query_by_museum_number(annotation.fragment_number) + .thenReturn(fragment) + ) + ( + when(photo_repository) + .query_by_file_name(f"{annotation.fragment_number}.jpg") + .thenReturn(create_test_photo("K.2")) + ) + + result = service._cropped_image_from_annotations(annotation) + for annotation in result.annotations: + assert annotation.image.script == fragment.script + assert annotation.image.label == "i Stone wig Stone wig 2" + + def test_generate_annotations( - annotations_repository, photo_repository, changelog, when + annotations_repository, photo_repository, changelog, when, fragment_repository ): fragment_number = MuseumNumber.of("X.0") @@ -21,9 +158,13 @@ def test_generate_annotations( ) ebl_ai_client = EblAiClient("mock-localhost:8001") service = AnnotationsService( - ebl_ai_client, annotations_repository, photo_repository, changelog + ebl_ai_client, + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, ) - expected = Annotations(fragment_number, tuple()) when(ebl_ai_client).generate_annotations(fragment_number, image_file, 0).thenReturn( expected @@ -34,36 +175,78 @@ def test_generate_annotations( assert annotations == expected -def test_find(annotations_repository, photo_repository, changelog, when): +def test_find( + annotations_repository, + photo_repository, + changelog, + when, + fragment_repository, + file_repository, +): annotations = AnnotationsFactory.build() when(annotations_repository).query_by_museum_number( annotations.fragment_number ).thenReturn(annotations) service = AnnotationsService( - EblAiClient(""), annotations_repository, photo_repository, changelog + EblAiClient(""), + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, ) assert service.find(annotations.fragment_number) == annotations -def test_update(annotations_repository, photo_repository, when, user, changelog): +def test_update( + annotations_repository, + photo_repository, + fragment_repository, + when, + user, + changelog, + text_with_labels, +): fragment_number = MuseumNumber("K", "1") annotations = AnnotationsFactory.build(fragment_number=fragment_number) updated_annotations = AnnotationsFactory.build(fragment_number=fragment_number) + fragment = TransliteratedFragmentFactory.build(text=text_with_labels) when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( annotations ) - when(annotations_repository).create_or_update(updated_annotations).thenReturn() + when(annotations_repository).create_or_update(...).thenReturn() when(changelog).create( "annotations", user.profile, {"_id": str(fragment_number), **SCHEMA.dump(annotations)}, {"_id": str(fragment_number), **SCHEMA.dump(updated_annotations)}, ).thenReturn() + ( + when(fragment_repository) + .query_by_museum_number(annotations.fragment_number) + .thenReturn(fragment) + ) + ( + when(photo_repository) + .query_by_file_name(f"{annotations.fragment_number}.jpg") + .thenReturn(create_test_photo("K.2")) + ) service = AnnotationsService( - EblAiClient(""), annotations_repository, photo_repository, changelog + EblAiClient(""), + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, ) - - assert service.update(updated_annotations, user) == updated_annotations + result = service.update(updated_annotations, user) + assert result.fragment_number == updated_annotations.fragment_number + for result_annotation, annotation in zip( + result.annotations, updated_annotations.annotations + ): + assert result_annotation.geometry == annotation.geometry + assert result_annotation.data == annotation.data + assert result_annotation.image is not None From e21323302b7b5291a1d8ddae2663eb1cbf40b826 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Wed, 9 Mar 2022 13:16:27 +0100 Subject: [PATCH 02/23] new repo for cropeed images --- .../application/annotations_schema.py | 6 +- .../application/annotations_service.py | 35 ++++++--- .../application/cropped_sign_image.py | 9 +++ .../application/cropped_sign_image_schema.py | 15 ++++ .../application/sign_images_repository.py | 15 ++++ ebl/fragmentarium/domain/annotation.py | 4 +- .../infrastructure/sign_images_repository.py | 29 ++++++++ ebl/fragmentarium/update_annotations.py | 16 ++++ ebl/signs/web/cropped_annotation_schema.py | 2 + ebl/tests/conftest.py | 18 ++++- .../fragmentarium/test_annotations_service.py | 74 +++---------------- 11 files changed, 146 insertions(+), 77 deletions(-) create mode 100644 ebl/fragmentarium/application/cropped_sign_image.py create mode 100644 ebl/fragmentarium/application/cropped_sign_image_schema.py create mode 100644 ebl/fragmentarium/application/sign_images_repository.py create mode 100644 ebl/fragmentarium/infrastructure/sign_images_repository.py diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index dbfd1ed67..1b1d2bb55 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -5,7 +5,7 @@ AnnotationData, Annotation, Annotations, - AnnotationValueType, + AnnotationValueType, CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.schemas import ValueEnum @@ -35,13 +35,13 @@ def make_data(self, data, **kwargs): class CroppedAnnotationImageSchema(Schema): - image = fields.Str(required=True) + image = fields.Raw(required=True) script = fields.Str(required=True) label = fields.Str(required=True) @post_load def make_cropped_annotation_image(self, data, **kwargs): - return CroppedAnnotationImageSchema(**data) + return CroppedAnnotationImage(**data) class AnnotationSchema(Schema): diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 0863c9528..c161231dd 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -1,5 +1,8 @@ import base64 import io +from typing import Tuple, Sequence +from uuid import uuid4 + from singledispatchmethod import singledispatchmethod from io import BytesIO @@ -11,12 +14,13 @@ from ebl.files.application.file_repository import FileRepository from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema +from ebl.fragmentarium.application.cropped_sign_image import Base64 from ebl.fragmentarium.application.fragment_repository import FragmentRepository +from ebl.fragmentarium.application.sign_images_repository import CroppedSignImage, SignImagesRepository from ebl.fragmentarium.domain.annotation import ( Annotations, Annotation, BoundingBox, - Base64, CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber @@ -37,6 +41,7 @@ class AnnotationsService: _changelog: Changelog _fragments_repository: FragmentRepository _photos_repository: FileRepository + _sign_images_repository: SignImagesRepository def generate_annotations( self, number: MuseumNumber, threshold: float = 0.3 @@ -67,7 +72,7 @@ def _format_label(self, label: LineLabel) -> str: def _cropped_image_from_annotation( self, annotation: Annotation, image: Image.Image - ) -> Base64: + ) -> CroppedSignImage: bounding_box = BoundingBox.from_annotations( image.size[0], image.size[1], [annotation] )[0] @@ -80,7 +85,7 @@ def _cropped_image_from_annotation( cropped_image = image.crop(area) buf = io.BytesIO() cropped_image.save(buf, format="PNG") - return Base64(base64.b64encode(buf.getvalue()).decode("utf-8")) + return CroppedSignImage(uuid4().hex, Base64(base64.b64encode(buf.getvalue()).decode("utf-8"))) @singledispatchmethod def _is_matching_number(self, line_number: AbstractLineNumber, number: int) -> bool: @@ -94,7 +99,8 @@ def _(self, line_number: LineNumber, number: int): def _(self, line_number: LineNumberRange, number: int): return line_number.start.number <= number <= line_number.end.number - def _cropped_image_from_annotations(self, annotations: Annotations) -> Annotations: + def _cropped_image_from_annotations(self, annotations: Annotations) -> Tuple[Annotations, Sequence[CroppedSignImage]]: + cropped_sign_images = [] cropped_annotations = [] fragment = self._fragments_repository.query_by_museum_number( annotations.fragment_number @@ -118,17 +124,18 @@ def _cropped_image_from_annotations(self, annotations: Annotations) -> Annotatio None, ) cropped_image = self._cropped_image_from_annotation(annotation, image) + cropped_sign_images.append(cropped_image) cropped_annotations.append( attr.evolve( annotation, image=CroppedAnnotationImage( - cropped_image, + cropped_image.sign_id, script, self._format_label(label) if label else "", ), ) ) - return attr.evolve(annotations, annotations=cropped_annotations) + return attr.evolve(annotations, annotations=cropped_annotations), cropped_sign_images def update(self, annotations: Annotations, user: User) -> Annotations: old_annotations = self._annotations_repository.query_by_museum_number( @@ -142,6 +149,16 @@ def update(self, annotations: Annotations, user: User) -> Annotations: {"_id": _id, **schema.dump(old_annotations)}, {"_id": _id, **schema.dump(annotations)}, ) - annotations_with_image = self._cropped_image_from_annotations(annotations) - self._annotations_repository.create_or_update(annotations_with_image) - return annotations_with_image + annotations_with_image_ids, cropped_sign_images = self._cropped_image_from_annotations(annotations) + self._annotations_repository.create_or_update(annotations_with_image_ids) + self._sign_images_repository.create_or_update(cropped_sign_images) + return annotations_with_image_ids + + def update_(self, annotations: Annotations) -> Annotations: + annotations_with_image_ids, cropped_sign_images = self._cropped_image_from_annotations(annotations) + self._annotations_repository.create_or_update(annotations_with_image_ids) + self._sign_images_repository.create_or_update(cropped_sign_images) + return annotations_with_image_ids + + + diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py new file mode 100644 index 000000000..399e6a162 --- /dev/null +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -0,0 +1,9 @@ +from typing import NewType +import attr + +Base64 = NewType("Base64", str) + +@attr.s(auto_attribs=True, frozen=True) +class CroppedSignImage: + sign_id: str + image: Base64 \ No newline at end of file diff --git a/ebl/fragmentarium/application/cropped_sign_image_schema.py b/ebl/fragmentarium/application/cropped_sign_image_schema.py new file mode 100644 index 000000000..3cf3042ee --- /dev/null +++ b/ebl/fragmentarium/application/cropped_sign_image_schema.py @@ -0,0 +1,15 @@ +from marshmallow import Schema, fields, post_dump + +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage + + +class CroppedSignImageSchema(Schema): + class Meta: + model = CroppedSignImage + + sign_id = fields.Str(required=True) + image = fields.Str(required=True) + + @post_dump + def dump(self, data, **kwargs): + return CroppedSignImage(**data) diff --git a/ebl/fragmentarium/application/sign_images_repository.py b/ebl/fragmentarium/application/sign_images_repository.py new file mode 100644 index 000000000..152a74a50 --- /dev/null +++ b/ebl/fragmentarium/application/sign_images_repository.py @@ -0,0 +1,15 @@ +from abc import ABC, abstractmethod +from typing import Sequence + +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage + + +class SignImagesRepository(ABC): + @abstractmethod + def query_by_id(self, sign_id: str) -> CroppedSignImage: + ... + + + @abstractmethod + def create_or_update(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: + ... diff --git a/ebl/fragmentarium/domain/annotation.py b/ebl/fragmentarium/domain/annotation.py index 817223071..2820fbd89 100644 --- a/ebl/fragmentarium/domain/annotation.py +++ b/ebl/fragmentarium/domain/annotation.py @@ -34,12 +34,12 @@ class AnnotationData: sign_name: str -Base64 = NewType("Base64", str) + @attr.attrs(auto_attribs=True, frozen=True) class CroppedAnnotationImage: - image: Base64 + image_id: str script: str label: str diff --git a/ebl/fragmentarium/infrastructure/sign_images_repository.py b/ebl/fragmentarium/infrastructure/sign_images_repository.py new file mode 100644 index 000000000..3e16123ab --- /dev/null +++ b/ebl/fragmentarium/infrastructure/sign_images_repository.py @@ -0,0 +1,29 @@ +from typing import Sequence + +from pymongo.database import Database + +from ebl.fragmentarium.application.sign_images_repository import SignImagesRepository, CroppedSignImage, \ + CroppedSignImageSchema +from ebl.mongo_collection import MongoCollection + +COLLECTION = "sign_images" + + +class MongoSignImagesRepository(SignImagesRepository): + def __init__(self, database: Database) -> None: + self._collection = MongoCollection(database, COLLECTION) + + def create_or_update(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: + for element in cropped_sign_image: + self._collection.replace_one( + CroppedSignImageSchema().dump(element), + True, + ) + + def query(self, sign_id: str) -> CroppedSignImage: + return CroppedSignImageSchema().load(self._collection.find_one({"sign_id": sign_id})) + + + + + diff --git a/ebl/fragmentarium/update_annotations.py b/ebl/fragmentarium/update_annotations.py index e69de29bb..c49fceaca 100644 --- a/ebl/fragmentarium/update_annotations.py +++ b/ebl/fragmentarium/update_annotations.py @@ -0,0 +1,16 @@ +from ebl.app import create_context +from ebl.fragmentarium.application.annotations_service import AnnotationsService + +if __name__ == "__main__": + context = create_context() + annotations = context.annotations_repository.retrieve_all_non_empty() + service = AnnotationsService(context.ebl_ai_client, context.annotations_repository, context.photo_repository, context.changelog, context.fragment_repository, context.photo_repository) + for i, annotation in enumerate(annotations): + try: + print(annotation.fragment_number) + service.update_(annotation) + print() + except Exception as e: + print(e) + + print("Update fragments completed!") \ No newline at end of file diff --git a/ebl/signs/web/cropped_annotation_schema.py b/ebl/signs/web/cropped_annotation_schema.py index 487ab858e..99e3508f3 100644 --- a/ebl/signs/web/cropped_annotation_schema.py +++ b/ebl/signs/web/cropped_annotation_schema.py @@ -6,3 +6,5 @@ class CroppedAnnotationSchema(Schema): fragment_number = fields.String(required=True) script = fields.String(required=True) label = fields.String(required=True) + + diff --git a/ebl/tests/conftest.py b/ebl/tests/conftest.py index 13fd5b119..b2b35801e 100644 --- a/ebl/tests/conftest.py +++ b/ebl/tests/conftest.py @@ -29,6 +29,7 @@ from ebl.ebl_ai_client import EblAiClient from ebl.files.application.file_repository import File from ebl.files.infrastructure.grid_fs_file_repository import GridFsFileRepository +from ebl.fragmentarium.application.annotations_service import AnnotationsService from ebl.fragmentarium.application.fragment_finder import FragmentFinder from ebl.fragmentarium.application.fragment_matcher import FragmentMatcher from ebl.fragmentarium.application.fragment_updater import FragmentUpdater @@ -41,6 +42,7 @@ from ebl.fragmentarium.infrastructure.mongo_annotations_repository import ( MongoAnnotationsRepository, ) +from ebl.fragmentarium.infrastructure.sign_images_repository import MongoSignImagesRepository from ebl.lemmatization.infrastrcuture.mongo_suggestions_finder import ( MongoLemmaRepository, ) @@ -160,7 +162,6 @@ def fragment_repository(database): def fragmentarium(fragment_repository, changelog, dictionary, bibliography): return Fragmentarium(fragment_repository) - @pytest.fixture def fragment_finder( fragment_repository, dictionary, photo_repository, file_repository, bibliography @@ -277,6 +278,21 @@ def annotations_repository(database): def lemma_repository(database): return MongoLemmaRepository(database) +@pytest.fixture +def sign_images_repository(database): + return MongoSignImagesRepository(database) + +@pytest.fixture +def annotations_service(annotations_repository, photo_repository, changelog, fragment_repository, sign_images_repository): + return AnnotationsService( + EblAiClient(""), + annotations_repository, + photo_repository, + changelog, + fragment_repository, + photo_repository, + sign_images_repository + ) @pytest.fixture def user() -> User: diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index db8ec2e09..5e8beb1a6 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -45,20 +45,9 @@ def test_format_line_label( line_label, expected, - annotations_repository, - photo_repository, - fragment_repository, - changelog, + annotations_service ): - service = AnnotationsService( - EblAiClient(""), - annotations_repository, - photo_repository, - changelog, - fragment_repository, - photo_repository, - ) - assert service._format_label(line_label) == expected + assert annotations_service._format_label(line_label) == expected @pytest.mark.parametrize( @@ -90,39 +79,19 @@ def test_line_label_match_line_number( line_label, line_number, expected, - fragment_repository, - changelog, - annotations_repository, - photo_repository, + annotations_service, ): - service = AnnotationsService( - EblAiClient(""), - annotations_repository, - photo_repository, - changelog, - fragment_repository, - photo_repository, - ) - assert service._is_matching_number(line_label, line_number) == expected + assert annotations_service._is_matching_number(line_label, line_number) == expected def test_cropped_images_from_sign( annotations_repository, - photo_repository, - changelog, fragment_repository, + photo_repository, when, text_with_labels, + annotations_service ): - service = AnnotationsService( - EblAiClient(""), - annotations_repository, - photo_repository, - changelog, - fragment_repository, - photo_repository, - ) - single_annotation = AnnotationFactory.build( data=AnnotationDataFactory.build(path=[2, 0, 0]) ) @@ -140,7 +109,7 @@ def test_cropped_images_from_sign( .thenReturn(create_test_photo("K.2")) ) - result = service._cropped_image_from_annotations(annotation) + result = annotations_service._cropped_image_from_annotations(annotation) for annotation in result.annotations: assert annotation.image.script == fragment.script assert annotation.image.label == "i Stone wig Stone wig 2" @@ -177,29 +146,19 @@ def test_generate_annotations( def test_find( annotations_repository, - photo_repository, - changelog, - when, - fragment_repository, - file_repository, + annotations_service, + when ): annotations = AnnotationsFactory.build() when(annotations_repository).query_by_museum_number( annotations.fragment_number ).thenReturn(annotations) - service = AnnotationsService( - EblAiClient(""), - annotations_repository, - photo_repository, - changelog, - fragment_repository, - photo_repository, - ) - assert service.find(annotations.fragment_number) == annotations + assert annotations_service.find(annotations.fragment_number) == annotations def test_update( + annotations_service, annotations_repository, photo_repository, fragment_repository, @@ -233,16 +192,7 @@ def test_update( .query_by_file_name(f"{annotations.fragment_number}.jpg") .thenReturn(create_test_photo("K.2")) ) - - service = AnnotationsService( - EblAiClient(""), - annotations_repository, - photo_repository, - changelog, - fragment_repository, - photo_repository, - ) - result = service.update(updated_annotations, user) + result = annotations_service.update(updated_annotations, user) assert result.fragment_number == updated_annotations.fragment_number for result_annotation, annotation in zip( result.annotations, updated_annotations.annotations From 874f47ac1b21334fb10492c69e3b09f1687fc59f Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Sun, 13 Mar 2022 17:57:59 +0100 Subject: [PATCH 03/23] refactor --- .../annotations_image_extractor.py | 18 +++---- .../application/annotations_schema.py | 11 +---- .../application/annotations_service.py | 48 ++++++++++++------- .../application/cropped_sign_image.py | 20 +++++++- .../application/cropped_sign_image_schema.py | 15 ------ .../application/sign_images_repository.py | 5 +- ebl/fragmentarium/domain/annotation.py | 5 +- .../infrastructure/sign_images_repository.py | 22 +++++---- ebl/fragmentarium/update_annotations.py | 11 ++++- ebl/signs/web/cropped_annotation_schema.py | 2 - ebl/tests/conftest.py | 18 +++++-- .../fragmentarium/test_annotations_service.py | 14 ++---- 12 files changed, 104 insertions(+), 85 deletions(-) delete mode 100644 ebl/fragmentarium/application/cropped_sign_image_schema.py diff --git a/ebl/fragmentarium/application/annotations_image_extractor.py b/ebl/fragmentarium/application/annotations_image_extractor.py index 628b14733..514efca8c 100644 --- a/ebl/fragmentarium/application/annotations_image_extractor.py +++ b/ebl/fragmentarium/application/annotations_image_extractor.py @@ -1,6 +1,8 @@ from typing import Sequence from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository +from ebl.fragmentarium.application.cropped_sign_image import Base64 +from ebl.fragmentarium.application.sign_images_repository import SignImagesRepository from ebl.fragmentarium.domain.annotation import CroppedAnnotationImage from ebl.fragmentarium.domain.museum_number import MuseumNumber import attr @@ -9,28 +11,28 @@ @attr.attrs(auto_attribs=True, frozen=True) class CroppedAnnotation(CroppedAnnotationImage): fragment_number: MuseumNumber + image: Base64 class AnnotationCroppedImageService: def __init__( self, annotations_repository: AnnotationsRepository, + sign_images_repository: SignImagesRepository, ): self._annotations_repository = annotations_repository + self._sign_repository = sign_images_repository def find_annotations_by_sign(self, sign: str) -> Sequence[CroppedAnnotation]: annotations = self._annotations_repository.find_by_sign(sign) cropped_image_annotations = [] for annotation in annotations: for annotation_elem in annotation.annotations: - image = annotation_elem.image - if image: + cropped_ann = annotation_elem.image + if cropped_ann: + image = self._sign_repository.query_by_id(cropped_ann.image_id) + cropped_image_annotations.append( - CroppedAnnotation( - image.image, - image.script, - image.label, - annotation.fragment_number, - ) + CroppedAnnotation(cropped_ann.image.image_id) ) return cropped_image_annotations diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 1b1d2bb55..197bc612f 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -5,7 +5,8 @@ AnnotationData, Annotation, Annotations, - AnnotationValueType, CroppedAnnotationImage, + AnnotationValueType, + CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.schemas import ValueEnum @@ -34,14 +35,6 @@ def make_data(self, data, **kwargs): return AnnotationData(**data) -class CroppedAnnotationImageSchema(Schema): - image = fields.Raw(required=True) - script = fields.Str(required=True) - label = fields.Str(required=True) - - @post_load - def make_cropped_annotation_image(self, data, **kwargs): - return CroppedAnnotationImage(**data) class AnnotationSchema(Schema): diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index c161231dd..c76b2e332 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -1,13 +1,12 @@ import base64 import io -from typing import Tuple, Sequence -from uuid import uuid4 - -from singledispatchmethod import singledispatchmethod from io import BytesIO +from typing import Tuple, Sequence import attr +import bson.objectid from PIL import Image +from singledispatchmethod import singledispatchmethod from ebl.changelog import Changelog from ebl.ebl_ai_client import EblAiClient @@ -16,7 +15,10 @@ from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema from ebl.fragmentarium.application.cropped_sign_image import Base64 from ebl.fragmentarium.application.fragment_repository import FragmentRepository -from ebl.fragmentarium.application.sign_images_repository import CroppedSignImage, SignImagesRepository +from ebl.fragmentarium.application.sign_images_repository import ( + CroppedSignImage, + SignImagesRepository, +) from ebl.fragmentarium.domain.annotation import ( Annotations, Annotation, @@ -72,7 +74,7 @@ def _format_label(self, label: LineLabel) -> str: def _cropped_image_from_annotation( self, annotation: Annotation, image: Image.Image - ) -> CroppedSignImage: + ) -> Base64: bounding_box = BoundingBox.from_annotations( image.size[0], image.size[1], [annotation] )[0] @@ -85,7 +87,7 @@ def _cropped_image_from_annotation( cropped_image = image.crop(area) buf = io.BytesIO() cropped_image.save(buf, format="PNG") - return CroppedSignImage(uuid4().hex, Base64(base64.b64encode(buf.getvalue()).decode("utf-8"))) + return Base64(base64.b64encode(buf.getvalue()).decode("utf-8")) @singledispatchmethod def _is_matching_number(self, line_number: AbstractLineNumber, number: int) -> bool: @@ -99,7 +101,9 @@ def _(self, line_number: LineNumber, number: int): def _(self, line_number: LineNumberRange, number: int): return line_number.start.number <= number <= line_number.end.number - def _cropped_image_from_annotations(self, annotations: Annotations) -> Tuple[Annotations, Sequence[CroppedSignImage]]: + def _cropped_image_from_annotations( + self, annotations: Annotations + ) -> Tuple[Annotations, Sequence[CroppedSignImage]]: cropped_sign_images = [] cropped_annotations = [] fragment = self._fragments_repository.query_by_museum_number( @@ -123,19 +127,26 @@ def _cropped_image_from_annotations(self, annotations: Annotations) -> Tuple[Ann ), None, ) - cropped_image = self._cropped_image_from_annotation(annotation, image) - cropped_sign_images.append(cropped_image) + cropped_image_base64 = self._cropped_image_from_annotation( + annotation, image + ) + image_id = str(bson.ObjectId()) + + cropped_sign_images.append(CroppedSignImage(sign_id, cropped_image_base64)) cropped_annotations.append( attr.evolve( annotation, image=CroppedAnnotationImage( - cropped_image.sign_id, + image_id, script, self._format_label(label) if label else "", ), ) ) - return attr.evolve(annotations, annotations=cropped_annotations), cropped_sign_images + return ( + attr.evolve(annotations, annotations=cropped_annotations), + cropped_sign_images, + ) def update(self, annotations: Annotations, user: User) -> Annotations: old_annotations = self._annotations_repository.query_by_museum_number( @@ -149,16 +160,19 @@ def update(self, annotations: Annotations, user: User) -> Annotations: {"_id": _id, **schema.dump(old_annotations)}, {"_id": _id, **schema.dump(annotations)}, ) - annotations_with_image_ids, cropped_sign_images = self._cropped_image_from_annotations(annotations) + ( + annotations_with_image_ids, + cropped_sign_images, + ) = self._cropped_image_from_annotations(annotations) self._annotations_repository.create_or_update(annotations_with_image_ids) self._sign_images_repository.create_or_update(cropped_sign_images) return annotations_with_image_ids def update_(self, annotations: Annotations) -> Annotations: - annotations_with_image_ids, cropped_sign_images = self._cropped_image_from_annotations(annotations) + ( + annotations_with_image_ids, + cropped_sign_images, + ) = self._cropped_image_from_annotations(annotations) self._annotations_repository.create_or_update(annotations_with_image_ids) self._sign_images_repository.create_or_update(cropped_sign_images) return annotations_with_image_ids - - - diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index 399e6a162..c62a0f77c 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -1,9 +1,25 @@ from typing import NewType + import attr Base64 = NewType("Base64", str) + @attr.s(auto_attribs=True, frozen=True) class CroppedSignImage: - sign_id: str - image: Base64 \ No newline at end of file + image_id: str + image: Base64 + + + + +class CroppedSignImageSchema(Schema): + image = fields.Str(required=True) + + @post_load + def load(self, data, **kwargs): + return CroppedSignImage(data["_id"], data["image"]) + + @post_dump + def dump(self, data, **kwargs): + return {"_id": data["image_id"], "image": data["image"]} diff --git a/ebl/fragmentarium/application/cropped_sign_image_schema.py b/ebl/fragmentarium/application/cropped_sign_image_schema.py deleted file mode 100644 index 3cf3042ee..000000000 --- a/ebl/fragmentarium/application/cropped_sign_image_schema.py +++ /dev/null @@ -1,15 +0,0 @@ -from marshmallow import Schema, fields, post_dump - -from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage - - -class CroppedSignImageSchema(Schema): - class Meta: - model = CroppedSignImage - - sign_id = fields.Str(required=True) - image = fields.Str(required=True) - - @post_dump - def dump(self, data, **kwargs): - return CroppedSignImage(**data) diff --git a/ebl/fragmentarium/application/sign_images_repository.py b/ebl/fragmentarium/application/sign_images_repository.py index 152a74a50..39b2c29ec 100644 --- a/ebl/fragmentarium/application/sign_images_repository.py +++ b/ebl/fragmentarium/application/sign_images_repository.py @@ -1,15 +1,14 @@ from abc import ABC, abstractmethod from typing import Sequence -from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage, Base64 class SignImagesRepository(ABC): @abstractmethod - def query_by_id(self, sign_id: str) -> CroppedSignImage: + def query_by_id(self, image_id: str) -> CroppedSignImage: ... - @abstractmethod def create_or_update(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: ... diff --git a/ebl/fragmentarium/domain/annotation.py b/ebl/fragmentarium/domain/annotation.py index 2820fbd89..28ab0427a 100644 --- a/ebl/fragmentarium/domain/annotation.py +++ b/ebl/fragmentarium/domain/annotation.py @@ -34,9 +34,6 @@ class AnnotationData: sign_name: str - - - @attr.attrs(auto_attribs=True, frozen=True) class CroppedAnnotationImage: image_id: str @@ -53,7 +50,7 @@ class Annotation: @classmethod def from_prediction(cls, geometry: Geometry) -> "Annotation": data = AnnotationData(uuid4().hex, "", AnnotationValueType.PREDICTED, [], "") - return cls(geometry, data) + return cls(geometry, data, None) @attr.attrs(auto_attribs=True, frozen=True) diff --git a/ebl/fragmentarium/infrastructure/sign_images_repository.py b/ebl/fragmentarium/infrastructure/sign_images_repository.py index 3e16123ab..9532c0301 100644 --- a/ebl/fragmentarium/infrastructure/sign_images_repository.py +++ b/ebl/fragmentarium/infrastructure/sign_images_repository.py @@ -2,8 +2,15 @@ from pymongo.database import Database -from ebl.fragmentarium.application.sign_images_repository import SignImagesRepository, CroppedSignImage, \ - CroppedSignImageSchema +from ebl.fragmentarium.application.cropped_sign_image import Base64 +from ebl.fragmentarium.application.cropped_sign_image_schema import ( + CroppedSignOnlyImageSchema, +) +from ebl.fragmentarium.application.sign_images_repository import ( + SignImagesRepository, + CroppedSignImage, + CroppedSignImageSchema, +) from ebl.mongo_collection import MongoCollection COLLECTION = "sign_images" @@ -20,10 +27,7 @@ def create_or_update(self, cropped_sign_image: Sequence[CroppedSignImage]) -> No True, ) - def query(self, sign_id: str) -> CroppedSignImage: - return CroppedSignImageSchema().load(self._collection.find_one({"sign_id": sign_id})) - - - - - + def query(self, image_id: str) -> CroppedSignImage: + return CroppedSignImageSchema().load( + self._collection.find_one({"_id": image_id}) + ) diff --git a/ebl/fragmentarium/update_annotations.py b/ebl/fragmentarium/update_annotations.py index c49fceaca..a97856516 100644 --- a/ebl/fragmentarium/update_annotations.py +++ b/ebl/fragmentarium/update_annotations.py @@ -4,7 +4,14 @@ if __name__ == "__main__": context = create_context() annotations = context.annotations_repository.retrieve_all_non_empty() - service = AnnotationsService(context.ebl_ai_client, context.annotations_repository, context.photo_repository, context.changelog, context.fragment_repository, context.photo_repository) + service = AnnotationsService( + context.ebl_ai_client, + context.annotations_repository, + context.photo_repository, + context.changelog, + context.fragment_repository, + context.photo_repository, + ) for i, annotation in enumerate(annotations): try: print(annotation.fragment_number) @@ -13,4 +20,4 @@ except Exception as e: print(e) - print("Update fragments completed!") \ No newline at end of file + print("Update fragments completed!") diff --git a/ebl/signs/web/cropped_annotation_schema.py b/ebl/signs/web/cropped_annotation_schema.py index 99e3508f3..487ab858e 100644 --- a/ebl/signs/web/cropped_annotation_schema.py +++ b/ebl/signs/web/cropped_annotation_schema.py @@ -6,5 +6,3 @@ class CroppedAnnotationSchema(Schema): fragment_number = fields.String(required=True) script = fields.String(required=True) label = fields.String(required=True) - - diff --git a/ebl/tests/conftest.py b/ebl/tests/conftest.py index b2b35801e..8fda46749 100644 --- a/ebl/tests/conftest.py +++ b/ebl/tests/conftest.py @@ -42,7 +42,9 @@ from ebl.fragmentarium.infrastructure.mongo_annotations_repository import ( MongoAnnotationsRepository, ) -from ebl.fragmentarium.infrastructure.sign_images_repository import MongoSignImagesRepository +from ebl.fragmentarium.infrastructure.sign_images_repository import ( + MongoSignImagesRepository, +) from ebl.lemmatization.infrastrcuture.mongo_suggestions_finder import ( MongoLemmaRepository, ) @@ -162,6 +164,7 @@ def fragment_repository(database): def fragmentarium(fragment_repository, changelog, dictionary, bibliography): return Fragmentarium(fragment_repository) + @pytest.fixture def fragment_finder( fragment_repository, dictionary, photo_repository, file_repository, bibliography @@ -278,12 +281,20 @@ def annotations_repository(database): def lemma_repository(database): return MongoLemmaRepository(database) + @pytest.fixture def sign_images_repository(database): return MongoSignImagesRepository(database) + @pytest.fixture -def annotations_service(annotations_repository, photo_repository, changelog, fragment_repository, sign_images_repository): +def annotations_service( + annotations_repository, + photo_repository, + changelog, + fragment_repository, + sign_images_repository, +): return AnnotationsService( EblAiClient(""), annotations_repository, @@ -291,9 +302,10 @@ def annotations_service(annotations_repository, photo_repository, changelog, fra changelog, fragment_repository, photo_repository, - sign_images_repository + sign_images_repository, ) + @pytest.fixture def user() -> User: return Auth0User( diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index 5e8beb1a6..949a8234d 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -42,11 +42,7 @@ ), ], ) -def test_format_line_label( - line_label, - expected, - annotations_service -): +def test_format_line_label(line_label, expected, annotations_service): assert annotations_service._format_label(line_label) == expected @@ -90,7 +86,7 @@ def test_cropped_images_from_sign( photo_repository, when, text_with_labels, - annotations_service + annotations_service, ): single_annotation = AnnotationFactory.build( data=AnnotationDataFactory.build(path=[2, 0, 0]) @@ -144,11 +140,7 @@ def test_generate_annotations( assert annotations == expected -def test_find( - annotations_repository, - annotations_service, - when -): +def test_find(annotations_repository, annotations_service, when): annotations = AnnotationsFactory.build() when(annotations_repository).query_by_museum_number( annotations.fragment_number From 00623da1a8437ee620c6dc09841629e692b9f6f3 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 14 Mar 2022 20:23:28 +0100 Subject: [PATCH 04/23] broken schema --- .github/workflows/main.yml | 4 +- README.md | 10 ++-- ebl/app.py | 4 ++ ebl/context.py | 4 ++ ebl/corpus/application/display_schemas.py | 13 ++++- ebl/corpus/domain/chapter_display.py | 17 ++++-- ebl/corpus/infrastructure/queries.py | 41 ++++++++------- .../annotations_image_extractor.py | 38 -------------- .../application/annotations_schema.py | 23 +++++--- .../application/annotations_service.py | 17 +++--- .../cropped_annotations_service.py | 38 ++++++++++++++ .../application/cropped_sign_image.py | 52 +++++++++++++++++-- ...y.py => cropped_sign_images_repository.py} | 6 +-- ebl/fragmentarium/domain/annotation.py | 12 ++--- .../cropped_sign_images_repository.py | 27 ++++++++++ .../infrastructure/sign_images_repository.py | 33 ------------ ebl/fragmentarium/update_annotations.py | 4 +- ebl/fragmentarium/web/annotations.py | 1 + ebl/fragmentarium/web/bootstrap.py | 1 + ebl/signs/web/bootstrap.py | 12 +++-- ebl/signs/web/cropped_annotation_schema.py | 8 --- ebl/signs/web/cropped_annotations.py | 20 +++++++ ebl/signs/web/sign_images.py | 17 ------ ebl/tests/conftest.py | 22 ++++---- ebl/tests/corpus/test_chapter_display.py | 7 +-- ebl/tests/corpus/test_chapter_display_schema | 2 + ebl/tests/factories/annotation.py | 27 +++++----- ebl/tests/fragmentarium/test_annotation.py | 9 +++- .../fragmentarium/test_annotations_route.py | 9 ++-- .../fragmentarium/test_annotations_schema.py | 36 ++++++++++++- .../fragmentarium/test_annotations_service.py | 23 +++++--- ...py => test_cropped_annotations_service.py} | 16 +++--- ebl/tests/signs/test_sign_images_route.py | 21 ++++++-- 33 files changed, 354 insertions(+), 220 deletions(-) delete mode 100644 ebl/fragmentarium/application/annotations_image_extractor.py create mode 100644 ebl/fragmentarium/application/cropped_annotations_service.py rename ebl/fragmentarium/application/{sign_images_repository.py => cropped_sign_images_repository.py} (64%) create mode 100644 ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py delete mode 100644 ebl/fragmentarium/infrastructure/sign_images_repository.py delete mode 100644 ebl/signs/web/cropped_annotation_schema.py create mode 100644 ebl/signs/web/cropped_annotations.py delete mode 100644 ebl/signs/web/sign_images.py rename ebl/tests/fragmentarium/{test_annotations_image_extractor.py => test_cropped_annotations_service.py} (60%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5665b6df..aa8603247 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,7 +54,7 @@ jobs: if: ${{ startsWith(matrix.python-version, 'pypy') }} env: CI: true - run: poetry run pytest -n auto + run: poetry run pytest - name: Unit Tests with Coverage uses: paambaati/codeclimate-action@v3.0.0 @@ -62,7 +62,7 @@ jobs: env: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} with: - coverageCommand: poetry run pytest --cov=ebl --cov-report term --cov-report xml -n auto + coverageCommand: poetry run pytest --cov=ebl --cov-report term --cov-report xml - uses: edge/simple-slack-notify@v1.1.1 if: failure() diff --git a/README.md b/README.md index 4e8d22151..f2110dcc9 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ Requirements: pip install poetry poetry install ``` -If `libcst` installation fails (because a binary wheel is not available for Linux/Windows + Pypy) you may need to install + +If `libcst` installation fails (because a binary wheel is not available for Linux/Windows + Pypy) you may need to install the [rust compiler](https://www.rust-lang.org/tools/install) to solve it. The following are needed to run application: @@ -86,7 +87,7 @@ poetry run flake8 poetry run pyre check poetry run pytest poetry run pytest -n auto # Run tests in parallel. -poetry run --cov=ebl --cov-report term --cov-report xml -n auto # Run tests in parallel with coverage (slow in PyPy). +poetry run --cov=ebl --cov-report term --cov-report xml # Run tests with coverage (slow in PyPy). ``` See [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) documentation @@ -244,8 +245,9 @@ SENTRY_ENVIRONMENT= CACHE_CONFIG= ``` -Poetry does not support .env-files. The environment variables need to be configured in the shell. -Alternatively and external program can be used to handle the file e.g. [direnv](https://direnv.net/) or [Set-PsEnv](https://github.com/rajivharris/Set-PsEnv). +Poetry does not support .env-files. The environment variables need to be configured in the shell. +Alternatively and external program can be used to handle the file e.g. [direnv](https://direnv.net/) +or [Set-PsEnv](https://github.com/rajivharris/Set-PsEnv). ### Locally diff --git a/ebl/app.py b/ebl/app.py index dcb08f995..85f7159fc 100644 --- a/ebl/app.py +++ b/ebl/app.py @@ -23,6 +23,9 @@ from ebl.ebl_ai_client import EblAiClient from ebl.files.infrastructure.grid_fs_file_repository import GridFsFileRepository from ebl.files.web.bootstrap import create_files_route +from ebl.fragmentarium.infrastructure.cropped_sign_images_repository import ( + MongoCroppedSignImagesRepository, +) from ebl.fragmentarium.infrastructure.fragment_repository import MongoFragmentRepository from ebl.fragmentarium.infrastructure.mongo_annotations_repository import ( MongoAnnotationsRepository, @@ -62,6 +65,7 @@ def create_context(): return Context( ebl_ai_client=ebl_ai_client, auth_backend=auth_backend, + cropped_sign_images_repository=MongoCroppedSignImagesRepository(database), word_repository=MongoWordRepository(database), sign_repository=MongoSignRepository(database), public_file_repository=GridFsFileRepository(database, "fs"), diff --git a/ebl/context.py b/ebl/context.py index 5b03c573d..fbaea9359 100644 --- a/ebl/context.py +++ b/ebl/context.py @@ -15,6 +15,9 @@ from ebl.fragmentarium.application.transliteration_update_factory import ( TransliterationUpdateFactory, ) +from ebl.fragmentarium.infrastructure.cropped_sign_images_repository import ( + MongoCroppedSignImagesRepository, +) from ebl.lemmatization.application.suggestion_finder import LemmaRepository from ebl.transliteration.application.sign_repository import SignRepository from ebl.transliteration.application.transliteration_query_factory import ( @@ -26,6 +29,7 @@ class Context: ebl_ai_client: EblAiClient auth_backend: AuthBackend + cropped_sign_images_repository: MongoCroppedSignImagesRepository word_repository: WordRepository sign_repository: SignRepository public_file_repository: FileRepository diff --git a/ebl/corpus/application/display_schemas.py b/ebl/corpus/application/display_schemas.py index 30543e77a..95954252f 100644 --- a/ebl/corpus/application/display_schemas.py +++ b/ebl/corpus/application/display_schemas.py @@ -5,9 +5,14 @@ from ebl.corpus.domain.chapter_display import ChapterDisplay, LineDisplay from ebl.corpus.domain.record import Record from ebl.transliteration.application.line_number_schemas import OneOfLineNumberSchema +from ebl.transliteration.application.line_schemas import ( + NoteLineSchema, + TranslationLineSchema, +) from ebl.transliteration.application.note_line_part_schemas import ( OneOfNoteLinePartSchema, ) +from ebl.transliteration.application.one_of_line_schema import ParallelLineSchema from ebl.transliteration.application.token_schemas import OneOfTokenSchema @@ -24,7 +29,11 @@ class LineDisplaySchema(Schema): ) reconstruction = fields.List(fields.Nested(OneOfTokenSchema), load_default=tuple()) translation = fields.List( - fields.Nested(OneOfNoteLinePartSchema), load_default=tuple(), allow_none=True + fields.Nested(TranslationLineSchema), load_default=tuple(), allow_none=True + ) + note = fields.Nested(NoteLineSchema, allow_none=True, load_default=None) + parallel_lines = fields.Nested( + ParallelLineSchema, data_key="parallelLines", many=True, load_default=tuple() ) @post_load @@ -36,6 +45,8 @@ def make_line(self, data: dict, **kwargs) -> LineDisplay: tuple(data["intertext"] or []), tuple(data["reconstruction"]), tuple(data["translation"] or []), + data["note"], + tuple(data["parallel_lines"]), ) diff --git a/ebl/corpus/domain/chapter_display.py b/ebl/corpus/domain/chapter_display.py index b68e70af1..901d3eb5f 100644 --- a/ebl/corpus/domain/chapter_display.py +++ b/ebl/corpus/domain/chapter_display.py @@ -1,10 +1,13 @@ -from typing import Iterable, Sequence +from typing import Optional, Sequence + import attr from ebl.corpus.domain.chapter import ChapterId, Chapter from ebl.corpus.domain.line import Line from ebl.corpus.domain.record import Record from ebl.corpus.domain.text import Text +from ebl.transliteration.domain.note_line import NoteLine +from ebl.transliteration.domain.parallel_line import ParallelLine from ebl.transliteration.domain.translation_line import ( DEFAULT_LANGUAGE, TranslationLine, @@ -15,7 +18,7 @@ def get_default_translation( - translations: Iterable[TranslationLine], + translations: Sequence[TranslationLine], ) -> Sequence[MarkupPart]: return next( ( @@ -34,11 +37,13 @@ class LineDisplay: is_beginning_of_section: bool intertext: Sequence[MarkupPart] reconstruction: Sequence[Token] - translation: Sequence[MarkupPart] + translation: Sequence[TranslationLine] + note: Optional[NoteLine] + parallel_lines: Sequence[ParallelLine] @property def title(self) -> Sequence[MarkupPart]: - return to_title(self.translation) + return to_title(get_default_translation(self.translation)) @staticmethod def of_line(line: Line) -> "LineDisplay": @@ -49,7 +54,9 @@ def of_line(line: Line) -> "LineDisplay": line.is_beginning_of_section, first_variant.intertext, first_variant.reconstruction, - get_default_translation(line.translation), + line.translation, + first_variant.note, + first_variant.parallel_lines, ) diff --git a/ebl/corpus/infrastructure/queries.py b/ebl/corpus/infrastructure/queries.py index 3e858b1e1..ca40f18ec 100644 --- a/ebl/corpus/infrastructure/queries.py +++ b/ebl/corpus/infrastructure/queries.py @@ -3,7 +3,6 @@ from ebl.corpus.domain.chapter import ChapterId from ebl.corpus.infrastructure.collections import CHAPTERS_COLLECTION from ebl.fragmentarium.infrastructure.queries import is_in_fragmentarium -from ebl.transliteration.domain.translation_line import DEFAULT_LANGUAGE def chapter_id_query(id_: ChapterId) -> dict: @@ -124,48 +123,50 @@ def aggregate_chapter_display(id_: ChapterId) -> List[dict]: "number": "$$line.number", "isSecondLineOfParallelism": "$$line.isSecondLineOfParallelism", "isBeginningOfSection": "$$line.isBeginningOfSection", - "translation": { + "translation": "$$line.translation", + "intertext": { "$arrayElemAt": [ { "$map": { - "input": { - "$filter": { - "input": "$$line.translation", - "as": "translation", - "cond": { - "eq": [ - "$$translation.language", - DEFAULT_LANGUAGE, - ] - }, - } - }, - "as": "en_translation", - "in": "$$en_translation.parts", + "input": "$$line.variants", + "as": "variant", + "in": "$$variant.intertext", } }, 0, ] }, - "intertext": { + "reconstruction": { "$arrayElemAt": [ { "$map": { "input": "$$line.variants", "as": "variant", - "in": "$$variant.intertext", + "in": "$$variant.reconstruction", } }, 0, ] }, - "reconstruction": { + "note": { "$arrayElemAt": [ { "$map": { "input": "$$line.variants", "as": "variant", - "in": "$$variant.reconstruction", + "in": "$$variant.note", + } + }, + 0, + ] + }, + "parallelLines": { + "$arrayElemAt": [ + { + "$map": { + "input": "$$line.variants", + "as": "variant", + "in": "$$variant.parallelLines", } }, 0, diff --git a/ebl/fragmentarium/application/annotations_image_extractor.py b/ebl/fragmentarium/application/annotations_image_extractor.py deleted file mode 100644 index 514efca8c..000000000 --- a/ebl/fragmentarium/application/annotations_image_extractor.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Sequence - -from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository -from ebl.fragmentarium.application.cropped_sign_image import Base64 -from ebl.fragmentarium.application.sign_images_repository import SignImagesRepository -from ebl.fragmentarium.domain.annotation import CroppedAnnotationImage -from ebl.fragmentarium.domain.museum_number import MuseumNumber -import attr - - -@attr.attrs(auto_attribs=True, frozen=True) -class CroppedAnnotation(CroppedAnnotationImage): - fragment_number: MuseumNumber - image: Base64 - - -class AnnotationCroppedImageService: - def __init__( - self, - annotations_repository: AnnotationsRepository, - sign_images_repository: SignImagesRepository, - ): - self._annotations_repository = annotations_repository - self._sign_repository = sign_images_repository - - def find_annotations_by_sign(self, sign: str) -> Sequence[CroppedAnnotation]: - annotations = self._annotations_repository.find_by_sign(sign) - cropped_image_annotations = [] - for annotation in annotations: - for annotation_elem in annotation.annotations: - cropped_ann = annotation_elem.image - if cropped_ann: - image = self._sign_repository.query_by_id(cropped_ann.image_id) - - cropped_image_annotations.append( - CroppedAnnotation(cropped_ann.image.image_id) - ) - return cropped_image_annotations diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 197bc612f..47adb3a23 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -1,12 +1,12 @@ -from marshmallow import Schema, fields, post_load - +from marshmallow import Schema, fields, post_load, post_dump +import pydash +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignSchema from ebl.fragmentarium.domain.annotation import ( Geometry, AnnotationData, Annotation, Annotations, AnnotationValueType, - CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.schemas import ValueEnum @@ -35,21 +35,30 @@ def make_data(self, data, **kwargs): return AnnotationData(**data) - - class AnnotationSchema(Schema): geometry = fields.Nested(GeometrySchema(), required=True) data = fields.Nested(AnnotationDataSchema(), required=True) - image = fields.Nested(CroppedAnnotationImageSchema(), missing=None) + cropped_sign = fields.Nested( + CroppedSignSchema(), missing=None, data_key="croppedSign" + ) + @post_load def make_annotation(self, data, **kwargs): return Annotation(**data) + @post_dump + def dump(self, data, **kwargs): + return pydash.omit_by(data, pydash.is_none) + + + + + class AnnotationsSchema(Schema): fragment_number = fields.String(required=True, data_key="fragmentNumber") - annotations = fields.Nested(AnnotationSchema, many=True, required=True) + annotations = fields.List(fields.Nested(AnnotationSchema()), required=True) @post_load def make_annotation(self, data, **kwargs): diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index c76b2e332..0e42c7b2d 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -13,17 +13,16 @@ from ebl.files.application.file_repository import FileRepository from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema -from ebl.fragmentarium.application.cropped_sign_image import Base64 +from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSign from ebl.fragmentarium.application.fragment_repository import FragmentRepository -from ebl.fragmentarium.application.sign_images_repository import ( +from ebl.fragmentarium.application.cropped_sign_images_repository import ( CroppedSignImage, - SignImagesRepository, + CroppedSignImagesRepository, ) from ebl.fragmentarium.domain.annotation import ( Annotations, Annotation, BoundingBox, - CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.transliteration.domain.line_label import LineLabel @@ -43,7 +42,7 @@ class AnnotationsService: _changelog: Changelog _fragments_repository: FragmentRepository _photos_repository: FileRepository - _sign_images_repository: SignImagesRepository + _cropped_sign_images_repository: CroppedSignImagesRepository def generate_annotations( self, number: MuseumNumber, threshold: float = 0.3 @@ -132,11 +131,11 @@ def _cropped_image_from_annotations( ) image_id = str(bson.ObjectId()) - cropped_sign_images.append(CroppedSignImage(sign_id, cropped_image_base64)) + cropped_sign_images.append(CroppedSignImage(image_id, cropped_image_base64)) cropped_annotations.append( attr.evolve( annotation, - image=CroppedAnnotationImage( + cropped_sign=CroppedSign( image_id, script, self._format_label(label) if label else "", @@ -165,7 +164,7 @@ def update(self, annotations: Annotations, user: User) -> Annotations: cropped_sign_images, ) = self._cropped_image_from_annotations(annotations) self._annotations_repository.create_or_update(annotations_with_image_ids) - self._sign_images_repository.create_or_update(cropped_sign_images) + self._cropped_sign_images_repository.create(cropped_sign_images) return annotations_with_image_ids def update_(self, annotations: Annotations) -> Annotations: @@ -174,5 +173,5 @@ def update_(self, annotations: Annotations) -> Annotations: cropped_sign_images, ) = self._cropped_image_from_annotations(annotations) self._annotations_repository.create_or_update(annotations_with_image_ids) - self._sign_images_repository.create_or_update(cropped_sign_images) + self._cropped_sign_images_repository.create_or_update(cropped_sign_images) return annotations_with_image_ids diff --git a/ebl/fragmentarium/application/cropped_annotations_service.py b/ebl/fragmentarium/application/cropped_annotations_service.py new file mode 100644 index 000000000..22b0a9f44 --- /dev/null +++ b/ebl/fragmentarium/application/cropped_annotations_service.py @@ -0,0 +1,38 @@ +from typing import Sequence + +from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository +from ebl.fragmentarium.application.cropped_sign_image import CroppedAnnotation +from ebl.fragmentarium.application.cropped_sign_images_repository import ( + CroppedSignImagesRepository, +) + + +class CroppedAnnotationService: + def __init__( + self, + annotations_repository: AnnotationsRepository, + cropped_sign_images_repository: CroppedSignImagesRepository, + ): + self._annotations_repository = annotations_repository + self._cropped_sign_image_repository = cropped_sign_images_repository + + def find_annotations_by_sign(self, sign: str) -> Sequence[CroppedAnnotation]: + annotations = self._annotations_repository.find_by_sign(sign) + cropped_image_annotations = [] + for annotation in annotations: + for annotation_elem in annotation.annotations: + cropped_sign = annotation_elem.cropped_sign + if cropped_sign: + cropped_sign_image = ( + self._cropped_sign_image_repository.query_by_id( + cropped_sign.image_id + ) + ) + cropped_image_annotations.append( + CroppedAnnotation.from_cropped_sign( + annotation.fragment_number, + cropped_sign_image.image, + cropped_sign, + ) + ) + return cropped_image_annotations diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index c62a0f77c..be7d1f311 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -1,7 +1,9 @@ from typing import NewType - +from marshmallow import Schema, fields, post_load, post_dump import attr +from ebl.fragmentarium.domain.museum_number import MuseumNumber + Base64 = NewType("Base64", str) @@ -11,8 +13,6 @@ class CroppedSignImage: image: Base64 - - class CroppedSignImageSchema(Schema): image = fields.Str(required=True) @@ -21,5 +21,47 @@ def load(self, data, **kwargs): return CroppedSignImage(data["_id"], data["image"]) @post_dump - def dump(self, data, **kwargs): - return {"_id": data["image_id"], "image": data["image"]} + def dump(self, data: CroppedSignImage, **kwargs): + return {"_id": data.image_id, "image": data.image} + + +@attr.attrs(auto_attribs=True, frozen=True) +class CroppedSign: + image_id: str + script: str + label: str + + +class CroppedSignSchema(Schema): + image_id = fields.String(required=True, data_key="imageId") + script = fields.String(required=True) + label = fields.String(required=True) + + @post_load + def load(self, data, **kwargs): + return CroppedSign(data["imageId"], data["script"], data["label"]) + + + + +@attr.attrs(auto_attribs=True, frozen=True) +class CroppedAnnotation(CroppedSign): + fragment_number: MuseumNumber + image: Base64 + + @classmethod + def from_cropped_sign( + cls, fragment_number: MuseumNumber, image: Base64, cropped_sign: CroppedSign + ) -> "CroppedAnnotation": + return cls( + cropped_sign.image_id, + cropped_sign.script, + cropped_sign.label, + fragment_number, + image, + ) + + +class CroppedAnnotationSchema(CroppedSignSchema): + fragment_number = fields.String(required=True, data_key="fragmentNumber") + image = fields.String(required=True) diff --git a/ebl/fragmentarium/application/sign_images_repository.py b/ebl/fragmentarium/application/cropped_sign_images_repository.py similarity index 64% rename from ebl/fragmentarium/application/sign_images_repository.py rename to ebl/fragmentarium/application/cropped_sign_images_repository.py index 39b2c29ec..2069f7705 100644 --- a/ebl/fragmentarium/application/sign_images_repository.py +++ b/ebl/fragmentarium/application/cropped_sign_images_repository.py @@ -1,14 +1,14 @@ from abc import ABC, abstractmethod from typing import Sequence -from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage, Base64 +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage -class SignImagesRepository(ABC): +class CroppedSignImagesRepository(ABC): @abstractmethod def query_by_id(self, image_id: str) -> CroppedSignImage: ... @abstractmethod - def create_or_update(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: + def create(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: ... diff --git a/ebl/fragmentarium/domain/annotation.py b/ebl/fragmentarium/domain/annotation.py index 28ab0427a..74dc86c2e 100644 --- a/ebl/fragmentarium/domain/annotation.py +++ b/ebl/fragmentarium/domain/annotation.py @@ -1,9 +1,10 @@ from enum import Enum -from typing import Sequence, Optional, NewType +from typing import Sequence, Optional from uuid import uuid4 import attr +from ebl.fragmentarium.application.cropped_sign_image import CroppedSign from ebl.fragmentarium.domain.museum_number import MuseumNumber @@ -34,18 +35,11 @@ class AnnotationData: sign_name: str -@attr.attrs(auto_attribs=True, frozen=True) -class CroppedAnnotationImage: - image_id: str - script: str - label: str - - @attr.attrs(auto_attribs=True, frozen=True) class Annotation: geometry: Geometry data: AnnotationData - image: Optional[CroppedAnnotationImage] = None + cropped_sign: Optional[CroppedSign] @classmethod def from_prediction(cls, geometry: Geometry) -> "Annotation": diff --git a/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py b/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py new file mode 100644 index 000000000..07e9ee710 --- /dev/null +++ b/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py @@ -0,0 +1,27 @@ +from typing import Sequence + +from pymongo.database import Database + +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImageSchema +from ebl.fragmentarium.application.cropped_sign_images_repository import ( + CroppedSignImagesRepository, + CroppedSignImage, +) +from ebl.mongo_collection import MongoCollection + +COLLECTION = "cropped_sign_images" + + +class MongoCroppedSignImagesRepository(CroppedSignImagesRepository): + def __init__(self, database: Database) -> None: + self._collection = MongoCollection(database, COLLECTION) + + def create(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: + schema = CroppedSignImageSchema() + for element in cropped_sign_image: + self._collection.insert_one(schema.dump(element)) + + def query_by_id(self, image_id: str) -> CroppedSignImage: + return CroppedSignImageSchema().load( + self._collection.find_one({"_id": image_id}) + ) diff --git a/ebl/fragmentarium/infrastructure/sign_images_repository.py b/ebl/fragmentarium/infrastructure/sign_images_repository.py deleted file mode 100644 index 9532c0301..000000000 --- a/ebl/fragmentarium/infrastructure/sign_images_repository.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Sequence - -from pymongo.database import Database - -from ebl.fragmentarium.application.cropped_sign_image import Base64 -from ebl.fragmentarium.application.cropped_sign_image_schema import ( - CroppedSignOnlyImageSchema, -) -from ebl.fragmentarium.application.sign_images_repository import ( - SignImagesRepository, - CroppedSignImage, - CroppedSignImageSchema, -) -from ebl.mongo_collection import MongoCollection - -COLLECTION = "sign_images" - - -class MongoSignImagesRepository(SignImagesRepository): - def __init__(self, database: Database) -> None: - self._collection = MongoCollection(database, COLLECTION) - - def create_or_update(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: - for element in cropped_sign_image: - self._collection.replace_one( - CroppedSignImageSchema().dump(element), - True, - ) - - def query(self, image_id: str) -> CroppedSignImage: - return CroppedSignImageSchema().load( - self._collection.find_one({"_id": image_id}) - ) diff --git a/ebl/fragmentarium/update_annotations.py b/ebl/fragmentarium/update_annotations.py index a97856516..f00ce4372 100644 --- a/ebl/fragmentarium/update_annotations.py +++ b/ebl/fragmentarium/update_annotations.py @@ -12,8 +12,10 @@ context.fragment_repository, context.photo_repository, ) - for i, annotation in enumerate(annotations): + for counter, annotation in enumerate(annotations): + print(f"Lenght {len(annotations)}") try: + print(counter) print(annotation.fragment_number) service.update_(annotation) print() diff --git a/ebl/fragmentarium/web/annotations.py b/ebl/fragmentarium/web/annotations.py index 7bc5d7d7b..e54937e1c 100644 --- a/ebl/fragmentarium/web/annotations.py +++ b/ebl/fragmentarium/web/annotations.py @@ -19,6 +19,7 @@ def on_post(self, req: falcon.Request, resp: falcon.Response, number: str): AnnotationsSchema().load(req.media), req.context.user ) resp.media = AnnotationsSchema().dump(annotations) + print() else: raise falcon.HTTPUnprocessableEntity( description="Fragment numbers do not match." diff --git a/ebl/fragmentarium/web/bootstrap.py b/ebl/fragmentarium/web/bootstrap.py index 89c26ffa5..492b1c294 100644 --- a/ebl/fragmentarium/web/bootstrap.py +++ b/ebl/fragmentarium/web/bootstrap.py @@ -40,6 +40,7 @@ def create_fragmentarium_routes(api: falcon.App, context: Context): context.changelog, context.fragment_repository, context.photo_repository, + context.cropped_sign_images_repository, ) statistics = make_statistics_resource(context.cache, fragmentarium) diff --git a/ebl/signs/web/bootstrap.py b/ebl/signs/web/bootstrap.py index 9d1debfd5..b13a73b69 100644 --- a/ebl/signs/web/bootstrap.py +++ b/ebl/signs/web/bootstrap.py @@ -1,19 +1,21 @@ import falcon from ebl.context import Context -from ebl.fragmentarium.application.annotations_image_extractor import ( - AnnotationCroppedImageService, +from ebl.fragmentarium.application.cropped_annotations_service import ( + CroppedAnnotationService, ) from ebl.signs.web.sign_search import SignsSearch from ebl.signs.web.signs import SignsResource -from ebl.signs.web.sign_images import SignsImageResource +from ebl.signs.web.cropped_annotations import CroppedAnnotationsResource def create_signs_routes(api: falcon.App, context: Context): signs_search = SignsSearch(context.sign_repository) signs = SignsResource(context.sign_repository) - signs_images = SignsImageResource( - AnnotationCroppedImageService(context.annotations_repository) + signs_images = CroppedAnnotationsResource( + CroppedAnnotationService( + context.annotations_repository, context.cropped_sign_images_repository + ) ) api.add_route("/signs", signs_search) api.add_route("/signs/{sign_name}", signs) diff --git a/ebl/signs/web/cropped_annotation_schema.py b/ebl/signs/web/cropped_annotation_schema.py deleted file mode 100644 index 487ab858e..000000000 --- a/ebl/signs/web/cropped_annotation_schema.py +++ /dev/null @@ -1,8 +0,0 @@ -from marshmallow import fields, Schema - - -class CroppedAnnotationSchema(Schema): - image = fields.String(required=True) - fragment_number = fields.String(required=True) - script = fields.String(required=True) - label = fields.String(required=True) diff --git a/ebl/signs/web/cropped_annotations.py b/ebl/signs/web/cropped_annotations.py new file mode 100644 index 000000000..1cabfd51e --- /dev/null +++ b/ebl/signs/web/cropped_annotations.py @@ -0,0 +1,20 @@ +import falcon + +from ebl.fragmentarium.application.cropped_annotations_service import ( + CroppedAnnotationService, +) +from ebl.fragmentarium.application.cropped_sign_image import CroppedAnnotationSchema + +from ebl.users.web.require_scope import require_scope + + +class CroppedAnnotationsResource: + def __init__(self, cropped_annotations_service: CroppedAnnotationService): + self._cropped_annotations_service = cropped_annotations_service + + @falcon.before(require_scope, "read:words") + def on_get(self, _req, resp, sign_name): + cropped_signs = self._cropped_annotations_service.find_annotations_by_sign( + sign_name + ) + resp.media = CroppedAnnotationSchema().dump(cropped_signs, many=True) diff --git a/ebl/signs/web/sign_images.py b/ebl/signs/web/sign_images.py deleted file mode 100644 index f5d699fee..000000000 --- a/ebl/signs/web/sign_images.py +++ /dev/null @@ -1,17 +0,0 @@ -import falcon - -from ebl.fragmentarium.application.annotations_image_extractor import ( - AnnotationCroppedImageService, -) -from ebl.signs.web.cropped_annotation_schema import CroppedAnnotationSchema -from ebl.users.web.require_scope import require_scope - - -class SignsImageResource: - def __init__(self, annotation_image_extractor: AnnotationCroppedImageService): - self._image_extractor = annotation_image_extractor - - @falcon.before(require_scope, "read:words") - def on_get(self, _req, resp, sign_name): - cropped_signs = self._image_extractor.find_annotations_by_sign(sign_name) - resp.media = CroppedAnnotationSchema().dump(cropped_signs, many=True) diff --git a/ebl/tests/conftest.py b/ebl/tests/conftest.py index 8fda46749..18bfb470c 100644 --- a/ebl/tests/conftest.py +++ b/ebl/tests/conftest.py @@ -38,13 +38,13 @@ TransliterationUpdateFactory, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber +from ebl.fragmentarium.infrastructure.cropped_sign_images_repository import ( + MongoCroppedSignImagesRepository, +) from ebl.fragmentarium.infrastructure.fragment_repository import MongoFragmentRepository from ebl.fragmentarium.infrastructure.mongo_annotations_repository import ( MongoAnnotationsRepository, ) -from ebl.fragmentarium.infrastructure.sign_images_repository import ( - MongoSignImagesRepository, -) from ebl.lemmatization.infrastrcuture.mongo_suggestions_finder import ( MongoLemmaRepository, ) @@ -100,6 +100,11 @@ def dictionary(word_repository, changelog): return Dictionary(word_repository, changelog) +@pytest.fixture +def cropped_sign_images_repository(database): + return MongoCroppedSignImagesRepository(database) + + @pytest.fixture def ebl_ai_client(): return EblAiClient("http://localhost:8001") @@ -282,18 +287,13 @@ def lemma_repository(database): return MongoLemmaRepository(database) -@pytest.fixture -def sign_images_repository(database): - return MongoSignImagesRepository(database) - - @pytest.fixture def annotations_service( annotations_repository, photo_repository, changelog, fragment_repository, - sign_images_repository, + cropped_sign_images_repository, ): return AnnotationsService( EblAiClient(""), @@ -302,7 +302,7 @@ def annotations_service( changelog, fragment_repository, photo_repository, - sign_images_repository, + cropped_sign_images_repository, ) @@ -335,6 +335,7 @@ def user() -> User: @pytest.fixture def context( ebl_ai_client, + cropped_sign_images_repository, word_repository, sign_repository, file_repository, @@ -352,6 +353,7 @@ def context( return ebl.context.Context( ebl_ai_client=ebl_ai_client, auth_backend=NoneAuthBackend(lambda: user), + cropped_sign_images_repository=cropped_sign_images_repository, word_repository=word_repository, sign_repository=sign_repository, public_file_repository=file_repository, diff --git a/ebl/tests/corpus/test_chapter_display.py b/ebl/tests/corpus/test_chapter_display.py index e6ad09b77..1a5c716ac 100644 --- a/ebl/tests/corpus/test_chapter_display.py +++ b/ebl/tests/corpus/test_chapter_display.py @@ -9,10 +9,9 @@ def test_line_display_of_line() -> None: - expected_translation = (StringPart("foo"),) translation_lines = ( TranslationLine((StringPart("bar"),), "de", None), - TranslationLine(expected_translation, DEFAULT_LANGUAGE, None), + TranslationLine((StringPart("foo"),), DEFAULT_LANGUAGE, None), ) line = LineFactory.build(translation=translation_lines) @@ -24,7 +23,9 @@ def test_line_display_of_line() -> None: line.is_beginning_of_section, line.variants[0].intertext, line.variants[0].reconstruction, - expected_translation, + translation_lines, + line.variants[0].note, + line.variants[0].parallel_lines, ) assert line_display.title == make_title(translation_lines) diff --git a/ebl/tests/corpus/test_chapter_display_schema b/ebl/tests/corpus/test_chapter_display_schema index b1631e757..609f97704 100644 --- a/ebl/tests/corpus/test_chapter_display_schema +++ b/ebl/tests/corpus/test_chapter_display_schema @@ -7,6 +7,7 @@ from ebl.tests.factories.corpus import ( TextFactory, ) from ebl.transliteration.application.line_number_schemas import OneOfLineNumberSchema +from ebl.transliteration.application.line_schemas import NoteLineSchema from ebl.transliteration.application.note_line_part_schemas import ( OneOfNoteLinePartSchema, ) @@ -43,6 +44,7 @@ def to_dict(chapter: ChapterDisplay, missing_intertext: bool = False, missing_tr "translation": None if missing_translation else OneOfNoteLinePartSchema().dump( line.translation, many=True ), + "note": line.note and NoteLineSchema().dump(line.none) } for line in chapter.lines ], diff --git a/ebl/tests/factories/annotation.py b/ebl/tests/factories/annotation.py index 314d490fa..66c84f3e6 100644 --- a/ebl/tests/factories/annotation.py +++ b/ebl/tests/factories/annotation.py @@ -1,12 +1,12 @@ import factory.fuzzy +from ebl.fragmentarium.application.cropped_sign_image import CroppedSign from ebl.fragmentarium.domain.annotation import ( Annotation, AnnotationData, Annotations, Geometry, AnnotationValueType, - CroppedAnnotationImage, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber @@ -38,26 +38,22 @@ class Meta: ) -class AnnotationFactory(factory.Factory): - class Meta: - model = Annotation - - geometry = factory.SubFactory(GeometryFactory) - data = factory.SubFactory(AnnotationDataFactory) - image = None - - -class CroppedImageFactory(factory.Factory): +class CroppedSignFactory(factory.Factory): class Meta: - model = CroppedAnnotationImage + model = CroppedSign - image = factory.Faker("word") + image_id = factory.Faker("word") script = factory.Faker("word") label = factory.Faker("word") -class AnnotationFactoryWithImage(AnnotationFactory): - image = factory.SubFactory(CroppedImageFactory) +class AnnotationFactory(factory.Factory): + class Meta: + model = Annotation + + geometry = factory.SubFactory(GeometryFactory) + data = factory.SubFactory(AnnotationDataFactory) + cropped_sign = factory.SubFactory(CroppedSignFactory) class AnnotationsFactory(factory.Factory): @@ -68,3 +64,4 @@ class Meta: annotations = factory.List( [factory.SubFactory(AnnotationFactory), factory.SubFactory(AnnotationFactory)] ) + diff --git a/ebl/tests/fragmentarium/test_annotation.py b/ebl/tests/fragmentarium/test_annotation.py index 75123a9b5..f3afd6e14 100644 --- a/ebl/tests/fragmentarium/test_annotation.py +++ b/ebl/tests/fragmentarium/test_annotation.py @@ -1,3 +1,4 @@ +from ebl.fragmentarium.application.cropped_sign_image import CroppedSign from ebl.fragmentarium.domain.annotation import ( Geometry, AnnotationData, @@ -21,7 +22,13 @@ SIGN_NAME = "KUR" DATA = AnnotationData(ID, VALUE, TYPE, PATH, SIGN_NAME) -ANNOTATION = Annotation(GEOMETRY, DATA) +IMAGE_ID = "image-id" +SCRIPT = "script" +LABEL = "label" + +CROPPED_SIGN = CroppedSign(IMAGE_ID, SCRIPT, LABEL) + +ANNOTATION = Annotation(GEOMETRY, DATA, CROPPED_SIGN) MUSEUM_NUMBER = MuseumNumber("K", "1") ANNOTATIONS = Annotations(MUSEUM_NUMBER, [ANNOTATION]) diff --git a/ebl/tests/fragmentarium/test_annotations_route.py b/ebl/tests/fragmentarium/test_annotations_route.py index 882e89b22..b098c1267 100644 --- a/ebl/tests/fragmentarium/test_annotations_route.py +++ b/ebl/tests/fragmentarium/test_annotations_route.py @@ -6,7 +6,7 @@ from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema from ebl.fragmentarium.domain.annotation import Annotations from ebl.fragmentarium.domain.museum_number import MuseumNumber -from ebl.tests.factories.annotation import AnnotationsFactory +from ebl.tests.factories.annotation import AnnotationsFactory, AnnotationFactory def test_find_annotations(client): @@ -66,9 +66,12 @@ def test_find_not_allowed(guest_client): def test_update(client): - annotations = AnnotationsFactory.build() + annotation_1 = AnnotationFactory(cropped_sign=None) + annotation_2 = AnnotationFactory(cropped_sign=None) + annotations = AnnotationsFactory.build(annotations=[annotation_1, annotation_2]) fragment_number = annotations.fragment_number - body = AnnotationsSchema().dumps(annotations) + body = AnnotationsSchema().dump(annotations) + x = AnnotationsSchema().load(body) url = f"/fragments/{fragment_number}/annotations" post_result = client.simulate_post(url, body=body) diff --git a/ebl/tests/fragmentarium/test_annotations_schema.py b/ebl/tests/fragmentarium/test_annotations_schema.py index 628c72f35..a563f7a64 100644 --- a/ebl/tests/fragmentarium/test_annotations_schema.py +++ b/ebl/tests/fragmentarium/test_annotations_schema.py @@ -1,4 +1,5 @@ from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema +from ebl.fragmentarium.application.cropped_sign_image import CroppedSign from ebl.fragmentarium.domain.annotation import ( Annotation, Geometry, @@ -18,14 +19,42 @@ TYPE = AnnotationValueType.HAS_SIGN ID = "abc123" SIGN_NAME = "KUR" +IMAGE_ID = "image-id" +SCRIPT = "script" +LABEL = "label" ANNOTATION = Annotation( - Geometry(X, Y, WIDTH, HEIGHT), AnnotationData(ID, VALUE, TYPE, PATH, SIGN_NAME) + Geometry(X, Y, WIDTH, HEIGHT), + AnnotationData(ID, VALUE, TYPE, PATH, SIGN_NAME), + CroppedSign(IMAGE_ID, SCRIPT, LABEL), +) +ANNOTATION_NO_CROPPED_SIGN = Annotation( + Geometry(X, Y, WIDTH, HEIGHT), + AnnotationData(ID, VALUE, TYPE, PATH, SIGN_NAME), + None, ) MUSEUM_NUMBER = MuseumNumber("K", "1") ANNOTATIONS = Annotations(MUSEUM_NUMBER, [ANNOTATION]) +ANNOTATIONS_NO_CROPPED_SIGN = Annotations(MUSEUM_NUMBER, [ANNOTATION_NO_CROPPED_SIGN]) SERIALIZED = { + "fragmentNumber": str(MUSEUM_NUMBER), + "annotations": [ + { + "geometry": {"x": X, "y": Y, "width": WIDTH, "height": HEIGHT}, + "data": { + "id": ID, + "value": VALUE, + "type": "HasSign", + "signName": SIGN_NAME, + "path": PATH, + }, + "croppedSign": {"imageId": IMAGE_ID, "script": SCRIPT, "label": LABEL}, + } + ], +} + +SERIALIZED_NO_SIGN = { "fragmentNumber": str(MUSEUM_NUMBER), "annotations": [ { @@ -42,9 +71,12 @@ } + def test_load(): assert AnnotationsSchema().load(SERIALIZED) == ANNOTATIONS - + x = AnnotationsSchema().load(SERIALIZED_NO_SIGN) + assert AnnotationsSchema().load(SERIALIZED_NO_SIGN) == ANNOTATIONS_NO_CROPPED_SIGN def test_dump(): assert AnnotationsSchema().dump(ANNOTATIONS) == SERIALIZED + #assert AnnotationsSchema().dump(ANNOTATIONS_NO_CROPPED_SIGN) == SERIALIZED_NO_SIGN diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index 949a8234d..e6ac9bf54 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -105,14 +105,22 @@ def test_cropped_images_from_sign( .thenReturn(create_test_photo("K.2")) ) - result = annotations_service._cropped_image_from_annotations(annotation) - for annotation in result.annotations: - assert annotation.image.script == fragment.script - assert annotation.image.label == "i Stone wig Stone wig 2" + annotations, cropped_images = annotations_service._cropped_image_from_annotations( + annotation + ) + for annotation, cropped_image in zip(annotations.annotations, cropped_images): + assert annotation.cropped_sign.image_id == cropped_image.image_id + assert annotation.cropped_sign.script == fragment.script + assert annotation.cropped_sign.label == "i Stone wig Stone wig 2" def test_generate_annotations( - annotations_repository, photo_repository, changelog, when, fragment_repository + annotations_repository, + photo_repository, + changelog, + when, + fragment_repository, + cropped_sign_images_repository, ): fragment_number = MuseumNumber.of("X.0") @@ -129,6 +137,7 @@ def test_generate_annotations( changelog, fragment_repository, photo_repository, + cropped_sign_images_repository, ) expected = Annotations(fragment_number, tuple()) when(ebl_ai_client).generate_annotations(fragment_number, image_file, 0).thenReturn( @@ -168,7 +177,7 @@ def test_update( annotations ) when(annotations_repository).create_or_update(...).thenReturn() - when(changelog).create( + when(changelog).create_or_update( "annotations", user.profile, {"_id": str(fragment_number), **SCHEMA.dump(annotations)}, @@ -191,4 +200,4 @@ def test_update( ): assert result_annotation.geometry == annotation.geometry assert result_annotation.data == annotation.data - assert result_annotation.image is not None + assert result_annotation.cropped_sign is not None diff --git a/ebl/tests/fragmentarium/test_annotations_image_extractor.py b/ebl/tests/fragmentarium/test_cropped_annotations_service.py similarity index 60% rename from ebl/tests/fragmentarium/test_annotations_image_extractor.py rename to ebl/tests/fragmentarium/test_cropped_annotations_service.py index 9266c24b6..f7fabeedf 100644 --- a/ebl/tests/fragmentarium/test_annotations_image_extractor.py +++ b/ebl/tests/fragmentarium/test_cropped_annotations_service.py @@ -1,25 +1,25 @@ -from ebl.fragmentarium.application.annotations_image_extractor import ( - AnnotationCroppedImageService, +from ebl.fragmentarium.application.cropped_annotations_service import ( + CroppedAnnotationService, CroppedAnnotation, ) from ebl.tests.factories.annotation import ( AnnotationsFactory, - AnnotationFactoryWithImage, + AnnotationFactory, ) def test_find_annotations_by_sign(annotations_repository, when): - service = AnnotationCroppedImageService(annotations_repository) - annotation = AnnotationFactoryWithImage.build_batch(2) + service = CroppedAnnotationService(annotations_repository) + annotation = AnnotationFactory.build_batch(2) annotations = [AnnotationsFactory.build(annotations=annotation)] when(annotations_repository).find_by_sign("test-sign").thenReturn(annotations) assert service.find_annotations_by_sign("test-sign") == list( map( lambda x: CroppedAnnotation( - x.image.image, - x.image.script, - x.image.label, + x.cropped_sign.cropped_sign, + x.cropped_sign.script, + x.cropped_sign.label, annotations[0].fragment_number, ), annotation, diff --git a/ebl/tests/signs/test_sign_images_route.py b/ebl/tests/signs/test_sign_images_route.py index 6e3d81072..5f67f721d 100644 --- a/ebl/tests/signs/test_sign_images_route.py +++ b/ebl/tests/signs/test_sign_images_route.py @@ -1,10 +1,12 @@ import falcon +from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage, Base64 from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.tests.factories.annotation import ( AnnotationsFactory, AnnotationFactory, AnnotationDataFactory, + CroppedSignFactory, ) from ebl.tests.factories.fragment import TransliteratedFragmentFactory @@ -16,6 +18,7 @@ def test_signs_get( when, fragment_repository, text_with_labels, + cropped_sign_images_repository, ): fragment = TransliteratedFragmentFactory.build( number=MuseumNumber.of("K.2"), text=text_with_labels @@ -23,7 +26,17 @@ def test_signs_get( fragment_repository.create(fragment) annotation_data = AnnotationDataFactory.build(sign_name="signName", path=[2, 0, 0]) - annotation = AnnotationFactory.build(data=annotation_data) + cropped_sign = CroppedSignFactory.build(script=fragment.script) + annotation = AnnotationFactory.build( + data=annotation_data, cropped_sign=cropped_sign + ) + cropped_sign_images_repository.create( + [ + CroppedSignImage( + annotation.cropped_sign.image_id, Base64("test-base64-string") + ) + ] + ) annotations_repository.create_or_update( AnnotationsFactory.build(fragment_number="K.2", annotations=[annotation]) ) @@ -34,8 +47,8 @@ def test_signs_get( result_json = result.json[0] assert result_json["fragmentNumber"] == str(fragment.number) - assert isinstance(result_json["image"], str) - assert result_json["script"] == fragment.script - assert result_json["label"] == "i Stone wig Stone wig 2" + assert result_json["image"] == "test-base64-string" + assert result_json["script"] == cropped_sign.script + assert result_json["label"] == cropped_sign.label assert result.status == falcon.HTTP_OK From 8fc25c43c15e205d3f2cd59645b3a2207241fc0f Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Wed, 16 Mar 2022 14:20:55 +0100 Subject: [PATCH 05/23] debug schema --- ebl/fragmentarium/application/annotations_schema.py | 4 +--- ebl/fragmentarium/application/cropped_sign_image.py | 2 -- ebl/tests/fragmentarium/test_annotations_schema.py | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 47adb3a23..0a9c78203 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -42,12 +42,10 @@ class AnnotationSchema(Schema): CroppedSignSchema(), missing=None, data_key="croppedSign" ) - @post_load def make_annotation(self, data, **kwargs): return Annotation(**data) - @post_dump def dump(self, data, **kwargs): return pydash.omit_by(data, pydash.is_none) @@ -58,7 +56,7 @@ def dump(self, data, **kwargs): class AnnotationsSchema(Schema): fragment_number = fields.String(required=True, data_key="fragmentNumber") - annotations = fields.List(fields.Nested(AnnotationSchema()), required=True) + annotations = fields.List(fields.Nested(AnnotationSchema(), required=True)) @post_load def make_annotation(self, data, **kwargs): diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index be7d1f311..57a83a0be 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -42,8 +42,6 @@ def load(self, data, **kwargs): return CroppedSign(data["imageId"], data["script"], data["label"]) - - @attr.attrs(auto_attribs=True, frozen=True) class CroppedAnnotation(CroppedSign): fragment_number: MuseumNumber diff --git a/ebl/tests/fragmentarium/test_annotations_schema.py b/ebl/tests/fragmentarium/test_annotations_schema.py index a563f7a64..3460a4302 100644 --- a/ebl/tests/fragmentarium/test_annotations_schema.py +++ b/ebl/tests/fragmentarium/test_annotations_schema.py @@ -74,7 +74,6 @@ def test_load(): assert AnnotationsSchema().load(SERIALIZED) == ANNOTATIONS - x = AnnotationsSchema().load(SERIALIZED_NO_SIGN) assert AnnotationsSchema().load(SERIALIZED_NO_SIGN) == ANNOTATIONS_NO_CROPPED_SIGN def test_dump(): From e304675d68eb4d14a624248a2d31d04c5715962c Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Thu, 17 Mar 2022 16:15:52 +0100 Subject: [PATCH 06/23] fix tests --- .../application/annotations_schema.py | 5 +-- .../application/annotations_service.py | 1 + .../application/cropped_sign_image.py | 5 +-- ebl/fragmentarium/update_annotations.py | 1 + ebl/tests/factories/annotation.py | 1 - .../fragmentarium/test_annotations_route.py | 31 +++++++++++++------ .../fragmentarium/test_annotations_schema.py | 4 +-- .../fragmentarium/test_annotations_service.py | 2 +- .../test_cropped_annotations_service.py | 8 +++-- 9 files changed, 37 insertions(+), 21 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 0a9c78203..9d8a64709 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -47,13 +47,10 @@ def make_annotation(self, data, **kwargs): return Annotation(**data) @post_dump - def dump(self, data, **kwargs): + def filter_none(self, data, **kwargs): return pydash.omit_by(data, pydash.is_none) - - - class AnnotationsSchema(Schema): fragment_number = fields.String(required=True, data_key="fragmentNumber") annotations = fields.List(fields.Nested(AnnotationSchema(), required=True)) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 0e42c7b2d..be6a47afa 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -159,6 +159,7 @@ def update(self, annotations: Annotations, user: User) -> Annotations: {"_id": _id, **schema.dump(old_annotations)}, {"_id": _id, **schema.dump(annotations)}, ) + ( annotations_with_image_ids, cropped_sign_images, diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index 57a83a0be..01e08b78e 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -14,6 +14,7 @@ class CroppedSignImage: class CroppedSignImageSchema(Schema): + image_id = fields.Str(required=True) image = fields.Str(required=True) @post_load @@ -21,8 +22,8 @@ def load(self, data, **kwargs): return CroppedSignImage(data["_id"], data["image"]) @post_dump - def dump(self, data: CroppedSignImage, **kwargs): - return {"_id": data.image_id, "image": data.image} + def cropped_sign_image_dump(self, data: CroppedSignImage, **kwargs): + return {"_id": data["image_id"], "image": data["image"]} @attr.attrs(auto_attribs=True, frozen=True) diff --git a/ebl/fragmentarium/update_annotations.py b/ebl/fragmentarium/update_annotations.py index f00ce4372..c4ee095bb 100644 --- a/ebl/fragmentarium/update_annotations.py +++ b/ebl/fragmentarium/update_annotations.py @@ -11,6 +11,7 @@ context.changelog, context.fragment_repository, context.photo_repository, + context.cropped_sign_images_repository, ) for counter, annotation in enumerate(annotations): print(f"Lenght {len(annotations)}") diff --git a/ebl/tests/factories/annotation.py b/ebl/tests/factories/annotation.py index 66c84f3e6..475a182f8 100644 --- a/ebl/tests/factories/annotation.py +++ b/ebl/tests/factories/annotation.py @@ -64,4 +64,3 @@ class Meta: annotations = factory.List( [factory.SubFactory(AnnotationFactory), factory.SubFactory(AnnotationFactory)] ) - diff --git a/ebl/tests/fragmentarium/test_annotations_route.py b/ebl/tests/fragmentarium/test_annotations_route.py index b098c1267..aa0c94553 100644 --- a/ebl/tests/fragmentarium/test_annotations_route.py +++ b/ebl/tests/fragmentarium/test_annotations_route.py @@ -6,7 +6,9 @@ from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema from ebl.fragmentarium.domain.annotation import Annotations from ebl.fragmentarium.domain.museum_number import MuseumNumber +from ebl.tests.conftest import create_test_photo from ebl.tests.factories.annotation import AnnotationsFactory, AnnotationFactory +from ebl.tests.factories.fragment import TransliteratedFragmentFactory def test_find_annotations(client): @@ -65,26 +67,37 @@ def test_find_not_allowed(guest_client): assert result.status == falcon.HTTP_FORBIDDEN -def test_update(client): +def test_update(client, fragment_repository, photo_repository): + number = MuseumNumber.of("K.123") annotation_1 = AnnotationFactory(cropped_sign=None) annotation_2 = AnnotationFactory(cropped_sign=None) - annotations = AnnotationsFactory.build(annotations=[annotation_1, annotation_2]) - fragment_number = annotations.fragment_number + annotations = AnnotationsFactory.build( + fragment_number=number, annotations=[annotation_1, annotation_2] + ) + fragment = TransliteratedFragmentFactory.build(number=number) + fragment_repository.create(fragment) + photo_repository._create(create_test_photo(number)) + body = AnnotationsSchema().dump(annotations) - x = AnnotationsSchema().load(body) - url = f"/fragments/{fragment_number}/annotations" - post_result = client.simulate_post(url, body=body) + url = f"/fragments/{number}/annotations" + post_result = client.simulate_post(url, body=json.dumps(body)) expected_json = AnnotationsSchema().dump(annotations) assert post_result.status == falcon.HTTP_OK - assert post_result.json == expected_json + assert post_result.json["fragmentNumber"] == expected_json["fragmentNumber"] + for annotation, expected_annotation in zip( + post_result.json["annotations"], expected_json["annotations"] + ): + assert annotation["geometry"] == expected_annotation["geometry"] + assert annotation["data"] == expected_annotation["data"] + assert annotation["croppedSign"] is not None get_result = client.simulate_get( - f"/fragments/{fragment_number}/annotations", + f"/fragments/{number}/annotations", params={"generateAnnotations": False}, ) - assert get_result.json == expected_json + assert get_result.json == post_result.json def test_update_number_mismatch(client): diff --git a/ebl/tests/fragmentarium/test_annotations_schema.py b/ebl/tests/fragmentarium/test_annotations_schema.py index 3460a4302..8f5d56b8a 100644 --- a/ebl/tests/fragmentarium/test_annotations_schema.py +++ b/ebl/tests/fragmentarium/test_annotations_schema.py @@ -71,11 +71,11 @@ } - def test_load(): assert AnnotationsSchema().load(SERIALIZED) == ANNOTATIONS assert AnnotationsSchema().load(SERIALIZED_NO_SIGN) == ANNOTATIONS_NO_CROPPED_SIGN + def test_dump(): assert AnnotationsSchema().dump(ANNOTATIONS) == SERIALIZED - #assert AnnotationsSchema().dump(ANNOTATIONS_NO_CROPPED_SIGN) == SERIALIZED_NO_SIGN + assert AnnotationsSchema().dump(ANNOTATIONS_NO_CROPPED_SIGN) == SERIALIZED_NO_SIGN diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index e6ac9bf54..fdbc507d4 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -177,7 +177,7 @@ def test_update( annotations ) when(annotations_repository).create_or_update(...).thenReturn() - when(changelog).create_or_update( + when(changelog).create( "annotations", user.profile, {"_id": str(fragment_number), **SCHEMA.dump(annotations)}, diff --git a/ebl/tests/fragmentarium/test_cropped_annotations_service.py b/ebl/tests/fragmentarium/test_cropped_annotations_service.py index f7fabeedf..dfe8b4de2 100644 --- a/ebl/tests/fragmentarium/test_cropped_annotations_service.py +++ b/ebl/tests/fragmentarium/test_cropped_annotations_service.py @@ -8,8 +8,12 @@ ) -def test_find_annotations_by_sign(annotations_repository, when): - service = CroppedAnnotationService(annotations_repository) +def test_find_annotations_by_sign( + annotations_repository, cropped_sign_images_repository, when +): + service = CroppedAnnotationService( + annotations_repository, cropped_sign_images_repository + ) annotation = AnnotationFactory.build_batch(2) annotations = [AnnotationsFactory.build(annotations=annotation)] From 5193eaaf9ba824a8ac39750bcd84741a826f9c48 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Thu, 17 Mar 2022 16:18:07 +0100 Subject: [PATCH 07/23] refactor to pass everything --- .../application/annotations_service.py | 9 ------- .../application/cropped_sign_image.py | 2 +- ebl/fragmentarium/update_annotations.py | 26 ------------------- 3 files changed, 1 insertion(+), 36 deletions(-) delete mode 100644 ebl/fragmentarium/update_annotations.py diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index be6a47afa..2d33ae494 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -167,12 +167,3 @@ def update(self, annotations: Annotations, user: User) -> Annotations: self._annotations_repository.create_or_update(annotations_with_image_ids) self._cropped_sign_images_repository.create(cropped_sign_images) return annotations_with_image_ids - - def update_(self, annotations: Annotations) -> Annotations: - ( - annotations_with_image_ids, - cropped_sign_images, - ) = self._cropped_image_from_annotations(annotations) - self._annotations_repository.create_or_update(annotations_with_image_ids) - self._cropped_sign_images_repository.create_or_update(cropped_sign_images) - return annotations_with_image_ids diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index 01e08b78e..a4da34aa7 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -22,7 +22,7 @@ def load(self, data, **kwargs): return CroppedSignImage(data["_id"], data["image"]) @post_dump - def cropped_sign_image_dump(self, data: CroppedSignImage, **kwargs): + def cropped_sign_image_dump(self, data, **kwargs): return {"_id": data["image_id"], "image": data["image"]} diff --git a/ebl/fragmentarium/update_annotations.py b/ebl/fragmentarium/update_annotations.py deleted file mode 100644 index c4ee095bb..000000000 --- a/ebl/fragmentarium/update_annotations.py +++ /dev/null @@ -1,26 +0,0 @@ -from ebl.app import create_context -from ebl.fragmentarium.application.annotations_service import AnnotationsService - -if __name__ == "__main__": - context = create_context() - annotations = context.annotations_repository.retrieve_all_non_empty() - service = AnnotationsService( - context.ebl_ai_client, - context.annotations_repository, - context.photo_repository, - context.changelog, - context.fragment_repository, - context.photo_repository, - context.cropped_sign_images_repository, - ) - for counter, annotation in enumerate(annotations): - print(f"Lenght {len(annotations)}") - try: - print(counter) - print(annotation.fragment_number) - service.update_(annotation) - print() - except Exception as e: - print(e) - - print("Update fragments completed!") From 22308770e3e9e270eca18dfdc0643f8c4372e9dc Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Thu, 17 Mar 2022 17:31:29 +0100 Subject: [PATCH 08/23] migration complet --- ebl/fragmentarium/application/annotations_schema.py | 5 ++++- .../infrastructure/mongo_annotations_repository.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 9d8a64709..525f259cd 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -1,4 +1,4 @@ -from marshmallow import Schema, fields, post_load, post_dump +from marshmallow import Schema, fields, post_load, post_dump, EXCLUDE import pydash from ebl.fragmentarium.application.cropped_sign_image import CroppedSignSchema from ebl.fragmentarium.domain.annotation import ( @@ -36,6 +36,9 @@ def make_data(self, data, **kwargs): class AnnotationSchema(Schema): + class Meta: + unknown = EXCLUDE + geometry = fields.Nested(GeometrySchema(), required=True) data = fields.Nested(AnnotationDataSchema(), required=True) cropped_sign = fields.Nested( diff --git a/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py b/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py index b5ef451b9..f34b68a68 100644 --- a/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py +++ b/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py @@ -41,7 +41,7 @@ def retrieve_all_non_empty(self) -> List[Annotations]: result = self._collection.find_many( {"annotations": {"$exists": True, "$ne": []}} ) - return AnnotationsSchema().load(result, unknown=EXCLUDE, many=True) + return AnnotationsSchema(unknown=EXCLUDE).load(result, many=True) def find_by_sign(self, sign: str) -> Sequence[Annotations]: query = {"$regex": re.escape(sign), "$options": "i"} From d1f12d5baad356d0691b8a9053804fd9225d02ff Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Thu, 17 Mar 2022 18:01:08 +0100 Subject: [PATCH 09/23] working tested on frontend --- .../test_cropped_annotations_service.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ebl/tests/fragmentarium/test_cropped_annotations_service.py b/ebl/tests/fragmentarium/test_cropped_annotations_service.py index dfe8b4de2..2f4ffee2c 100644 --- a/ebl/tests/fragmentarium/test_cropped_annotations_service.py +++ b/ebl/tests/fragmentarium/test_cropped_annotations_service.py @@ -1,7 +1,7 @@ from ebl.fragmentarium.application.cropped_annotations_service import ( CroppedAnnotationService, - CroppedAnnotation, ) +from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSignImage from ebl.tests.factories.annotation import ( AnnotationsFactory, AnnotationFactory, @@ -17,15 +17,16 @@ def test_find_annotations_by_sign( annotation = AnnotationFactory.build_batch(2) annotations = [AnnotationsFactory.build(annotations=annotation)] + image_id_1 = annotation[0].cropped_sign.image_id + image_id_2 = annotation[1].cropped_sign.image_id + when(annotations_repository).find_by_sign("test-sign").thenReturn(annotations) - assert service.find_annotations_by_sign("test-sign") == list( - map( - lambda x: CroppedAnnotation( - x.cropped_sign.cropped_sign, - x.cropped_sign.script, - x.cropped_sign.label, - annotations[0].fragment_number, - ), - annotation, - ) + when(cropped_sign_images_repository).query_by_id(image_id_1).thenReturn( + CroppedSignImage(image_id_1, Base64("test-base64-1")) + ) + when(cropped_sign_images_repository).query_by_id(image_id_2).thenReturn( + CroppedSignImage(image_id_2, Base64("test-base64-1")) ) + assert ( + service.find_annotations_by_sign("test-sign") is not None + ) # will deal with that From 51c2d0baaa73b0f22c8a98243208bafb56ee304e Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Thu, 17 Mar 2022 18:02:46 +0100 Subject: [PATCH 10/23] update test --- .../test_cropped_annotations_service.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ebl/tests/fragmentarium/test_cropped_annotations_service.py b/ebl/tests/fragmentarium/test_cropped_annotations_service.py index dfe8b4de2..2f4ffee2c 100644 --- a/ebl/tests/fragmentarium/test_cropped_annotations_service.py +++ b/ebl/tests/fragmentarium/test_cropped_annotations_service.py @@ -1,7 +1,7 @@ from ebl.fragmentarium.application.cropped_annotations_service import ( CroppedAnnotationService, - CroppedAnnotation, ) +from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSignImage from ebl.tests.factories.annotation import ( AnnotationsFactory, AnnotationFactory, @@ -17,15 +17,16 @@ def test_find_annotations_by_sign( annotation = AnnotationFactory.build_batch(2) annotations = [AnnotationsFactory.build(annotations=annotation)] + image_id_1 = annotation[0].cropped_sign.image_id + image_id_2 = annotation[1].cropped_sign.image_id + when(annotations_repository).find_by_sign("test-sign").thenReturn(annotations) - assert service.find_annotations_by_sign("test-sign") == list( - map( - lambda x: CroppedAnnotation( - x.cropped_sign.cropped_sign, - x.cropped_sign.script, - x.cropped_sign.label, - annotations[0].fragment_number, - ), - annotation, - ) + when(cropped_sign_images_repository).query_by_id(image_id_1).thenReturn( + CroppedSignImage(image_id_1, Base64("test-base64-1")) + ) + when(cropped_sign_images_repository).query_by_id(image_id_2).thenReturn( + CroppedSignImage(image_id_2, Base64("test-base64-1")) ) + assert ( + service.find_annotations_by_sign("test-sign") is not None + ) # will deal with that From ebf68e2a7c3f4aee3b6efbb0e2b7ccc9d0f7ffeb Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 15:29:01 +0100 Subject: [PATCH 11/23] fix issues --- ebl/fragmentarium/application/annotations_schema.py | 2 +- ebl/fragmentarium/web/annotations.py | 1 - ebl/tests/fragmentarium/test_annotations_service.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 9d8a64709..84f5d4a3a 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -39,7 +39,7 @@ class AnnotationSchema(Schema): geometry = fields.Nested(GeometrySchema(), required=True) data = fields.Nested(AnnotationDataSchema(), required=True) cropped_sign = fields.Nested( - CroppedSignSchema(), missing=None, data_key="croppedSign" + CroppedSignSchema(), data_key="croppedSign" ) @post_load diff --git a/ebl/fragmentarium/web/annotations.py b/ebl/fragmentarium/web/annotations.py index e54937e1c..7bc5d7d7b 100644 --- a/ebl/fragmentarium/web/annotations.py +++ b/ebl/fragmentarium/web/annotations.py @@ -19,7 +19,6 @@ def on_post(self, req: falcon.Request, resp: falcon.Response, number: str): AnnotationsSchema().load(req.media), req.context.user ) resp.media = AnnotationsSchema().dump(annotations) - print() else: raise falcon.HTTPUnprocessableEntity( description="Fragment numbers do not match." diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index fdbc507d4..af9de2fe8 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -176,7 +176,7 @@ def test_update( when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( annotations ) - when(annotations_repository).create_or_update(...).thenReturn() + when(annotations_repository).create_or_update(updated_annotations).thenReturn() when(changelog).create( "annotations", user.profile, From fc534d56613dfa90acc752808f149475a33f1c94 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 16:47:20 +0100 Subject: [PATCH 12/23] format code --- ebl/bibliography/web/bibliography_entries.py | 2 +- ebl/corpus/application/corpus.py | 2 +- .../application/annotations_schema.py | 4 +--- .../application/annotations_service.py | 2 +- .../cropped_sign_images_repository.py | 2 +- .../cropped_sign_images_repository.py | 7 +++--- ebl/fragmentarium/web/transliterations.py | 2 +- ebl/mongo_collection.py | 6 ++++- ebl/tests/bibliography/test_bibliography.py | 20 +++++++++------- .../test_bibliography_repository.py | 8 +++---- .../bibliography/test_bibliography_route.py | 2 +- ebl/tests/corpus/support.py | 4 ++-- .../test_chapter_display_lines_route.py | 8 +++---- .../corpus/test_chapter_display_route.py | 4 ++-- ebl/tests/corpus/test_corpus.py | 4 ++-- .../corpus/test_mongo_text_repository.py | 18 +++++++-------- ebl/tests/corpus/test_texts_route.py | 6 ++--- ebl/tests/dictionary/test_dictionary.py | 18 +++++++-------- ebl/tests/dictionary/test_word_repositor.py | 4 ++-- ebl/tests/dictionary/test_words_route.py | 2 +- .../fragmentarium/test_annotations_route.py | 2 +- .../fragmentarium/test_annotations_service.py | 2 +- .../test_cropped_annotations_service.py | 23 +++++++++++++------ .../test_fragment_genre_route.py | 4 ++-- .../test_fragment_matcher_route.py | 8 +++---- .../fragmentarium/test_fragment_repository.py | 20 ++++++++-------- .../fragmentarium/test_fragment_updater.py | 8 +++---- .../fragmentarium/test_fragments_route.py | 2 +- .../test_fragments_search_route.py | 22 +++++++++--------- .../fragmentarium/test_lemmatization_route.py | 6 ++--- ebl/tests/fragmentarium/test_pager_route.py | 10 ++++---- .../fragmentarium/test_references_route.py | 8 +++---- .../fragmentarium/test_statistics_route.py | 2 +- .../test_transliteration_query_factory.py | 2 +- .../test_transliteration_update_factory.py | 2 +- .../test_transliterations_route.py | 12 +++++----- .../lemmatization/test_lemma_search_route.py | 4 ++-- .../lemmatization/test_suggestion_finder.py | 16 ++++++------- ebl/tests/signs/test_sign_images_route.py | 4 ++-- ebl/tests/signs/test_sign_repository.py | 2 +- ebl/tests/signs/test_sign_search_route.py | 6 ++--- ebl/tests/test_changelog.py | 2 +- ebl/tests/test_mongo_collection.py | 9 ++++++++ .../transliteration/test_signs_visitor.py | 2 +- 44 files changed, 163 insertions(+), 140 deletions(-) diff --git a/ebl/bibliography/web/bibliography_entries.py b/ebl/bibliography/web/bibliography_entries.py index fb00e8372..cda9ffea0 100644 --- a/ebl/bibliography/web/bibliography_entries.py +++ b/ebl/bibliography/web/bibliography_entries.py @@ -18,7 +18,7 @@ def on_get(self, req: Request, resp: Response) -> None: @validate(CSL_JSON_SCHEMA) def on_post(self, req: Request, resp: Response) -> None: bibliography_entry = req.media - self._bibliography.create(bibliography_entry, req.context.user) + self._bibliography.create_many(bibliography_entry, req.context.user) resp.status = falcon.HTTP_CREATED resp.location = f'/bibliography/{bibliography_entry["id"]}' resp.media = bibliography_entry diff --git a/ebl/corpus/application/corpus.py b/ebl/corpus/application/corpus.py index ea768ecd9..efd156bb9 100644 --- a/ebl/corpus/application/corpus.py +++ b/ebl/corpus/application/corpus.py @@ -206,4 +206,4 @@ def _hydrate_references(self, chapter: Chapter) -> Chapter: def _create_changelog(self, old: Chapter, new: Chapter, user: User) -> None: old_dict: dict = {**ChapterSchema().dump(old), "_id": old.id_.to_tuple()} new_dict: dict = {**ChapterSchema().dump(new), "_id": new.id_.to_tuple()} - self._changelog.create(COLLECTION, user.profile, old_dict, new_dict) + self._changelog.create_many(COLLECTION, user.profile, old_dict, new_dict) diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 84f5d4a3a..154ebbc5e 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -38,9 +38,7 @@ def make_data(self, data, **kwargs): class AnnotationSchema(Schema): geometry = fields.Nested(GeometrySchema(), required=True) data = fields.Nested(AnnotationDataSchema(), required=True) - cropped_sign = fields.Nested( - CroppedSignSchema(), data_key="croppedSign" - ) + cropped_sign = fields.Nested(CroppedSignSchema(), data_key="croppedSign") @post_load def make_annotation(self, data, **kwargs): diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 2d33ae494..4be62d397 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -165,5 +165,5 @@ def update(self, annotations: Annotations, user: User) -> Annotations: cropped_sign_images, ) = self._cropped_image_from_annotations(annotations) self._annotations_repository.create_or_update(annotations_with_image_ids) - self._cropped_sign_images_repository.create(cropped_sign_images) + self._cropped_sign_images_repository.create_many(cropped_sign_images) return annotations_with_image_ids diff --git a/ebl/fragmentarium/application/cropped_sign_images_repository.py b/ebl/fragmentarium/application/cropped_sign_images_repository.py index 2069f7705..4599179bc 100644 --- a/ebl/fragmentarium/application/cropped_sign_images_repository.py +++ b/ebl/fragmentarium/application/cropped_sign_images_repository.py @@ -10,5 +10,5 @@ def query_by_id(self, image_id: str) -> CroppedSignImage: ... @abstractmethod - def create(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: + def create_many(self, cropped_sign_images: Sequence[CroppedSignImage]) -> None: ... diff --git a/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py b/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py index 07e9ee710..39ea6f131 100644 --- a/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py +++ b/ebl/fragmentarium/infrastructure/cropped_sign_images_repository.py @@ -16,10 +16,9 @@ class MongoCroppedSignImagesRepository(CroppedSignImagesRepository): def __init__(self, database: Database) -> None: self._collection = MongoCollection(database, COLLECTION) - def create(self, cropped_sign_image: Sequence[CroppedSignImage]) -> None: - schema = CroppedSignImageSchema() - for element in cropped_sign_image: - self._collection.insert_one(schema.dump(element)) + def create_many(self, cropped_sign_images: Sequence[CroppedSignImage]) -> None: + schema = CroppedSignImageSchema(many=True) + self._collection.insert_many(schema.dump(cropped_sign_images)) def query_by_id(self, image_id: str) -> CroppedSignImage: return CroppedSignImageSchema().load( diff --git a/ebl/fragmentarium/web/transliterations.py b/ebl/fragmentarium/web/transliterations.py index 4de894581..6ad3a8ac2 100644 --- a/ebl/fragmentarium/web/transliterations.py +++ b/ebl/fragmentarium/web/transliterations.py @@ -45,7 +45,7 @@ def on_post(self, req: Request, resp: Response, number: str) -> None: def _create_transliteration(self, media): try: - return self._transliteration_factory.create( + return self._transliteration_factory.create_many( Atf(media["transliteration"]), media["notes"] ) except ValueError as error: diff --git a/ebl/mongo_collection.py b/ebl/mongo_collection.py index ed30f2c83..5759d7542 100644 --- a/ebl/mongo_collection.py +++ b/ebl/mongo_collection.py @@ -1,6 +1,7 @@ -from typing import Any, cast +from typing import Any, cast, Sequence import inflect +from bson import ObjectId from pymongo.collection import Collection from pymongo.database import Database from pymongo.errors import DuplicateKeyError @@ -19,6 +20,9 @@ def __init__(self, database: Database, collection: str): self.__collection = collection self.__resource_noun = singlar(collection) + def insert_many(self, documents: Sequence[dict]) -> Sequence[ObjectId]: + return self.__get_collection().insert_many(documents).inserted_ids + def insert_one(self, document): try: return self.__get_collection().insert_one(document).inserted_id diff --git a/ebl/tests/bibliography/test_bibliography.py b/ebl/tests/bibliography/test_bibliography.py index 1ed23d91a..81cc43714 100644 --- a/ebl/tests/bibliography/test_bibliography.py +++ b/ebl/tests/bibliography/test_bibliography.py @@ -69,7 +69,7 @@ def test_create( bibliography_entry = BibliographyEntryFactory.build() ( when(changelog) - .create( + .create_many( COLLECTION, user.profile, {"_id": bibliography_entry["id"]}, @@ -77,9 +77,9 @@ def test_create( ) .thenReturn() ) - (when(bibliography_repository).create(bibliography_entry).thenReturn()) + (when(bibliography_repository).create_many(bibliography_entry).thenReturn()) - bibliography.create(bibliography_entry, user) + bibliography.create_many(bibliography_entry, user) def test_create_duplicate( @@ -93,7 +93,7 @@ def test_create_duplicate( bibliography_entry = BibliographyEntryFactory.build() ( when(changelog) - .create( + .create_many( COLLECTION, user.profile, {"_id": bibliography_entry["id"]}, @@ -101,9 +101,13 @@ def test_create_duplicate( ) .thenReturn() ) - (when(bibliography_repository).create(bibliography_entry).thenRaise(DuplicateError)) + ( + when(bibliography_repository) + .create_many(bibliography_entry) + .thenRaise(DuplicateError) + ) with pytest.raises(DuplicateError): - bibliography.create(bibliography_entry, user) + bibliography.create_many(bibliography_entry, user) def test_entry_not_found(bibliography, bibliography_repository, when): @@ -133,7 +137,7 @@ def test_update( ) ( when(changelog) - .create( + .create_many( COLLECTION, user.profile, create_mongo_bibliography_entry(), @@ -171,7 +175,7 @@ def test_validate_references_invalid( valid_reference = ReferenceFactory.build(with_document=True) first_invalid = ReferenceFactory.build(with_document=True) second_invalid = ReferenceFactory.build(with_document=True) - bibliography.create(valid_reference.document, user) + bibliography.create_many(valid_reference.document, user) (when(bibliography).find(valid_reference.id).thenReturn(valid_reference)) (when(bibliography).find(first_invalid.id).thenRaise(NotFoundError)) (when(bibliography).find(second_invalid.id).thenRaise(NotFoundError)) diff --git a/ebl/tests/bibliography/test_bibliography_repository.py b/ebl/tests/bibliography/test_bibliography_repository.py index 1e93d4a4f..b6d03128b 100644 --- a/ebl/tests/bibliography/test_bibliography_repository.py +++ b/ebl/tests/bibliography/test_bibliography_repository.py @@ -9,7 +9,7 @@ def test_create(database, bibliography_repository, create_mongo_bibliography_entry): bibliography_entry = BibliographyEntryFactory.build() - bibliography_repository.create(bibliography_entry) + bibliography_repository.create_many(bibliography_entry) assert ( database[COLLECTION].find_one({"_id": bibliography_entry["id"]}) @@ -19,9 +19,9 @@ def test_create(database, bibliography_repository, create_mongo_bibliography_ent def test_create_duplicate(bibliography_repository): bibliography_entry = BibliographyEntryFactory.build() - bibliography_repository.create(bibliography_entry) + bibliography_repository.create_many(bibliography_entry) with pytest.raises(DuplicateError): - bibliography_repository.create(bibliography_entry) + bibliography_repository.create_many(bibliography_entry) def test_find(database, bibliography_repository, create_mongo_bibliography_entry): @@ -43,7 +43,7 @@ def test_entry_not_found(bibliography_repository): def test_update(bibliography_repository): bibliography_entry = BibliographyEntryFactory.build() updated_entry = pydash.omit({**bibliography_entry, "title": "New Title"}, "PMID") - bibliography_repository.create(bibliography_entry) + bibliography_repository.create_many(bibliography_entry) bibliography_repository.update(updated_entry) assert ( diff --git a/ebl/tests/bibliography/test_bibliography_route.py b/ebl/tests/bibliography/test_bibliography_route.py index e0e6caeb6..77dfece2a 100644 --- a/ebl/tests/bibliography/test_bibliography_route.py +++ b/ebl/tests/bibliography/test_bibliography_route.py @@ -15,7 +15,7 @@ @pytest.fixture def saved_entry(bibliography, user): bibliography_entry = BibliographyEntryFactory.build() - bibliography.create(bibliography_entry, user) + bibliography.create_many(bibliography_entry, user) return bibliography_entry diff --git a/ebl/tests/corpus/support.py b/ebl/tests/corpus/support.py index 927ed773e..836b23ad0 100644 --- a/ebl/tests/corpus/support.py +++ b/ebl/tests/corpus/support.py @@ -9,12 +9,12 @@ def allow_references(chapter, bibliography): for manuscript in chapter.manuscripts: for reference in manuscript.references: - bibliography.create(reference.document, ANY_USER) + bibliography.create_many(reference.document, ANY_USER) def allow_signs(signs, sign_list): for sign in signs: - sign_list.create(sign) + sign_list.create_many(sign) def create_chapter_dto(chapter: Chapter) -> dict: diff --git a/ebl/tests/corpus/test_chapter_display_lines_route.py b/ebl/tests/corpus/test_chapter_display_lines_route.py index 7d934a72f..0a0b20fef 100644 --- a/ebl/tests/corpus/test_chapter_display_lines_route.py +++ b/ebl/tests/corpus/test_chapter_display_lines_route.py @@ -28,7 +28,7 @@ def url(chapter: Chapter) -> str: def test_get(client, text_repository, text, chapter, bibliography, url): - text_repository.create(text) + text_repository.create_many(text) text_repository.create_chapter(chapter) allow_references(chapter, bibliography) schema = LineDetailsSchema(context={"manuscripts": chapter.manuscripts}) @@ -40,7 +40,7 @@ def test_get(client, text_repository, text, chapter, bibliography, url): def test_chapter_not_found(client, text_repository, text, chapter, url): - text_repository.create(text) + text_repository.create_many(text) result = client.simulate_get(f"{url}/0") @@ -48,7 +48,7 @@ def test_chapter_not_found(client, text_repository, text, chapter, url): def test_line_not_found(client, text_repository, chapter, bibliography, url): - text_repository.create(text) + text_repository.create_many(text) text_repository.create_chapter(chapter) allow_references(chapter, bibliography) @@ -58,7 +58,7 @@ def test_line_not_found(client, text_repository, chapter, bibliography, url): def test_invalid_line(client, text_repository, chapter, bibliography, url): - text_repository.create(text) + text_repository.create_many(text) text_repository.create_chapter(chapter) allow_references(chapter, bibliography) diff --git a/ebl/tests/corpus/test_chapter_display_route.py b/ebl/tests/corpus/test_chapter_display_route.py index 5bf4e111c..11e0fbc75 100644 --- a/ebl/tests/corpus/test_chapter_display_route.py +++ b/ebl/tests/corpus/test_chapter_display_route.py @@ -29,7 +29,7 @@ def url(chapter: Chapter) -> str: def test_get(client, text_repository, text, chapter, url): - text_repository.create(text) + text_repository.create_many(text) text_repository.create_chapter(chapter) chapter_display = ChapterDisplay.of_chapter(text, chapter) @@ -40,7 +40,7 @@ def test_get(client, text_repository, text, chapter, url): def test_text_not_found(client, text_repository, text, chapter, url): - text_repository.create(text) + text_repository.create_many(text) result = client.simulate_get(url) diff --git a/ebl/tests/corpus/test_corpus.py b/ebl/tests/corpus/test_corpus.py index 2af50134d..010e97f87 100644 --- a/ebl/tests/corpus/test_corpus.py +++ b/ebl/tests/corpus/test_corpus.py @@ -59,7 +59,7 @@ def expect_invalid_references(bibliography, when) -> None: def expect_signs(signs, sign_repository) -> None: for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) def expect_chapter_update( @@ -77,7 +77,7 @@ def expect_chapter_update( when(text_repository).update(CHAPTER.id_, updated_chapter).thenReturn( updated_chapter ) - when(changelog).create( + when(changelog).create_many( CHAPTERS_COLLECTION, user.profile, {**ChapterSchema().dump(old_chapter), "_id": old_chapter.id_.to_tuple()}, diff --git a/ebl/tests/corpus/test_mongo_text_repository.py b/ebl/tests/corpus/test_mongo_text_repository.py index 1a29ab37e..a7d230750 100644 --- a/ebl/tests/corpus/test_mongo_text_repository.py +++ b/ebl/tests/corpus/test_mongo_text_repository.py @@ -53,7 +53,7 @@ def when_chapter_in_collection(database, chapter=CHAPTER) -> None: def test_creating_text(database, text_repository) -> None: - text_repository.create(TEXT) + text_repository.create_many(TEXT) inserted_text = database[TEXTS_COLLECTION].find_one( {"category": TEXT.category, "index": TEXT.index}, projection={"_id": False} @@ -78,18 +78,18 @@ def test_creating_chapter(database, text_repository) -> None: def test_it_is_not_possible_to_create_duplicate_texts(text_repository) -> None: text_repository.create_indexes() - text_repository.create(TEXT) + text_repository.create_many(TEXT) with pytest.raises(DuplicateError): - text_repository.create(TEXT) + text_repository.create_many(TEXT) def test_it_is_not_possible_to_create_duplicate_chapters(text_repository) -> None: text_repository.create_indexes() - text_repository.create(CHAPTER) + text_repository.create_many(CHAPTER) with pytest.raises(DuplicateError): - text_repository.create(CHAPTER) + text_repository.create_many(CHAPTER) def test_finding_text( @@ -107,9 +107,9 @@ def test_finding_text( chapter = attr.evolve(CHAPTER, uncertain_fragments=(UNCERTAIN_FRAGMET,)) when_text_in_collection(database, text) when_chapter_in_collection(database, chapter) - fragment_repository.create(Fragment(UNCERTAIN_FRAGMET)) + fragment_repository.create_many(Fragment(UNCERTAIN_FRAGMET)) for reference in text.references: - bibliography_repository.create(reference.document) + bibliography_repository.create_many(reference.document) assert text_repository.find(text.id) == text @@ -133,7 +133,7 @@ def test_listing_texts(database, text_repository, bibliography_repository) -> No when_chapter_in_collection(database) when_chapter_in_collection(database, another_chapter) for reference in TEXT.references: - bibliography_repository.create(reference.document) + bibliography_repository.create_many(reference.document) assert text_repository.list() == [TEXT, another_text] @@ -225,7 +225,7 @@ def test_query_manuscripts_with_joins_by_chapter_no_joins( def test_query_manuscripts_with_joins_is_in_fragmentarium( database, text_repository, fragment_repository ) -> None: - fragment_repository.create(FragmentFactory.build(number=MUSEUM_NUMBER)) + fragment_repository.create_many(FragmentFactory.build(number=MUSEUM_NUMBER)) when_chapter_in_collection(database) assert text_repository.query_manuscripts_with_joins_by_chapter(CHAPTER.id_) == [ diff --git a/ebl/tests/corpus/test_texts_route.py b/ebl/tests/corpus/test_texts_route.py index d088b4aa9..e19b65012 100644 --- a/ebl/tests/corpus/test_texts_route.py +++ b/ebl/tests/corpus/test_texts_route.py @@ -14,7 +14,7 @@ def create_dto(text): def test_get_text(client, bibliography, sign_repository, signs, text_repository): text = TextFactory.build(chapters=tuple(), references=tuple()) - text_repository.create(text) + text_repository.create_many(text) get_result = client.simulate_get( f"/texts/{text.genre.value}/{text.category}/{text.index}" @@ -45,8 +45,8 @@ def test_invalid_index(client): def test_listing_texts(client, bibliography, sign_repository, signs, text_repository): first_text = TextFactory.build(chapters=tuple(), references=tuple()) second_text = TextFactory.build(chapters=tuple(), references=tuple()) - text_repository.create(first_text) - text_repository.create(second_text) + text_repository.create_many(first_text) + text_repository.create_many(second_text) get_result = client.simulate_get("/texts") diff --git a/ebl/tests/dictionary/test_dictionary.py b/ebl/tests/dictionary/test_dictionary.py index bca260caa..d0f133cd6 100644 --- a/ebl/tests/dictionary/test_dictionary.py +++ b/ebl/tests/dictionary/test_dictionary.py @@ -8,7 +8,7 @@ def test_create_and_find(database, dictionary, word): - word_id = dictionary.create(word) + word_id = dictionary.create_many(word) assert dictionary.find(word_id) == word @@ -20,8 +20,8 @@ def test_word_not_found(dictionary): def test_search_finds_all_homonyms(dictionary, word): another_word = {**word, "_id": "part1 part2 II", "homonym": "II"} - dictionary.create(word) - dictionary.create(another_word) + dictionary.create_many(word) + dictionary.create_many(another_word) assert dictionary.search(" ".join(word["lemma"])) == [word, another_word] @@ -33,16 +33,16 @@ def test_search_finds_by_meaning(dictionary, word): "homonym": "II", "meaning": "not matching", } - dictionary.create(word) - dictionary.create(another_word) + dictionary.create_many(word) + dictionary.create_many(another_word) assert dictionary.search(word["meaning"][1:4]) == [word] def test_search_finds_duplicates(dictionary, word): another_word = {**word, "_id": "part1 part2 II", "homonym": "II"} - dictionary.create(word) - dictionary.create(another_word) + dictionary.create_many(word) + dictionary.create_many(another_word) assert dictionary.search(word["meaning"][1:4]) == [word, another_word] @@ -53,7 +53,7 @@ def test_search_not_found(dictionary): def test_update(dictionary, word, user): new_lemma = ["new"] - word_id = dictionary.create(word) + word_id = dictionary.create_many(word) updated_word = pydash.defaults({"lemma": new_lemma}, word) dictionary.update(updated_word, user) @@ -63,7 +63,7 @@ def test_update(dictionary, word, user): @freeze_time("2018-09-07 15:41:24.032") def test_changelog(dictionary, word, user, database, make_changelog_entry): - word_id = dictionary.create(word) + word_id = dictionary.create_many(word) updated_word = pydash.defaults({"lemma": ["new"]}, word) dictionary.update(updated_word, user) diff --git a/ebl/tests/dictionary/test_word_repositor.py b/ebl/tests/dictionary/test_word_repositor.py index 90b77bbc4..f03863350 100644 --- a/ebl/tests/dictionary/test_word_repositor.py +++ b/ebl/tests/dictionary/test_word_repositor.py @@ -7,7 +7,7 @@ def test_create(database, word_repository, word): - word_id = word_repository.create(word) + word_id = word_repository.create_many(word) assert database[COLLECTION].find_one({"_id": word_id}) == word @@ -63,7 +63,7 @@ def test_search_not_found(word_repository): def test_update(word_repository, word): new_lemma = ["new"] - word_id = word_repository.create(word) + word_id = word_repository.create_many(word) updated_word = pydash.defaults({"lemma": new_lemma}, word) word_repository.update(updated_word) diff --git a/ebl/tests/dictionary/test_words_route.py b/ebl/tests/dictionary/test_words_route.py index a0c04e9cf..7747ddabb 100644 --- a/ebl/tests/dictionary/test_words_route.py +++ b/ebl/tests/dictionary/test_words_route.py @@ -7,7 +7,7 @@ @pytest.fixture def saved_word(dictionary, word): word = {**word} - dictionary.create(word) + dictionary.create_many(word) return word diff --git a/ebl/tests/fragmentarium/test_annotations_route.py b/ebl/tests/fragmentarium/test_annotations_route.py index aa0c94553..d61299ec9 100644 --- a/ebl/tests/fragmentarium/test_annotations_route.py +++ b/ebl/tests/fragmentarium/test_annotations_route.py @@ -75,7 +75,7 @@ def test_update(client, fragment_repository, photo_repository): fragment_number=number, annotations=[annotation_1, annotation_2] ) fragment = TransliteratedFragmentFactory.build(number=number) - fragment_repository.create(fragment) + fragment_repository.create_many(fragment) photo_repository._create(create_test_photo(number)) body = AnnotationsSchema().dump(annotations) diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index af9de2fe8..e3ebce94b 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -177,7 +177,7 @@ def test_update( annotations ) when(annotations_repository).create_or_update(updated_annotations).thenReturn() - when(changelog).create( + when(changelog).create_many( "annotations", user.profile, {"_id": str(fragment_number), **SCHEMA.dump(annotations)}, diff --git a/ebl/tests/fragmentarium/test_cropped_annotations_service.py b/ebl/tests/fragmentarium/test_cropped_annotations_service.py index 2f4ffee2c..2de7fdcdb 100644 --- a/ebl/tests/fragmentarium/test_cropped_annotations_service.py +++ b/ebl/tests/fragmentarium/test_cropped_annotations_service.py @@ -1,7 +1,11 @@ from ebl.fragmentarium.application.cropped_annotations_service import ( CroppedAnnotationService, ) -from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSignImage +from ebl.fragmentarium.application.cropped_sign_image import ( + Base64, + CroppedSignImage, + CroppedAnnotation, +) from ebl.tests.factories.annotation import ( AnnotationsFactory, AnnotationFactory, @@ -15,18 +19,23 @@ def test_find_annotations_by_sign( annotations_repository, cropped_sign_images_repository ) annotation = AnnotationFactory.build_batch(2) - annotations = [AnnotationsFactory.build(annotations=annotation)] + annotations = AnnotationsFactory.build(annotations=annotation) image_id_1 = annotation[0].cropped_sign.image_id image_id_2 = annotation[1].cropped_sign.image_id - when(annotations_repository).find_by_sign("test-sign").thenReturn(annotations) + when(annotations_repository).find_by_sign("test-sign").thenReturn([annotations]) when(cropped_sign_images_repository).query_by_id(image_id_1).thenReturn( CroppedSignImage(image_id_1, Base64("test-base64-1")) ) when(cropped_sign_images_repository).query_by_id(image_id_2).thenReturn( - CroppedSignImage(image_id_2, Base64("test-base64-1")) + CroppedSignImage(image_id_2, Base64("test-base64-2")) + ) + fragment_number = annotations.fragment_number + expected_1 = CroppedAnnotation.from_cropped_sign( + fragment_number, Base64("test-base64-1"), annotation[0].cropped_sign + ) + expected_2 = CroppedAnnotation.from_cropped_sign( + fragment_number, Base64("test-base64-2"), annotation[1].cropped_sign ) - assert ( - service.find_annotations_by_sign("test-sign") is not None - ) # will deal with that + assert service.find_annotations_by_sign("test-sign") == [expected_1, expected_2] diff --git a/ebl/tests/fragmentarium/test_fragment_genre_route.py b/ebl/tests/fragmentarium/test_fragment_genre_route.py index 3eb4e4f6d..a8c20eb5c 100644 --- a/ebl/tests/fragmentarium/test_fragment_genre_route.py +++ b/ebl/tests/fragmentarium/test_fragment_genre_route.py @@ -31,7 +31,7 @@ ) def test_update_genres(client, fragmentarium, user, database, parameters): fragment = FragmentFactory.build(genres=parameters["currentGenres"]) - fragment_number = fragmentarium.create(fragment) + fragment_number = fragmentarium.create_many(fragment) updates = {"genres": GenreSchema().dump(parameters["newGenres"], many=True)} post_result = client.simulate_post( f"/fragments/{fragment_number}/genres", body=json.dumps(updates) @@ -59,7 +59,7 @@ def test_update_genres(client, fragmentarium, user, database, parameters): def test_update_invalid_genres(client, fragmentarium, user, database): fragment = FragmentFactory.build(genres=tuple()) - fragment_number = fragmentarium.create(fragment) + fragment_number = fragmentarium.create_many(fragment) updates = {"genres": [{"category": ["asd", "wtz"], "uncertain": False}]} post_result = client.simulate_post( diff --git a/ebl/tests/fragmentarium/test_fragment_matcher_route.py b/ebl/tests/fragmentarium/test_fragment_matcher_route.py index 3391c697e..fae154601 100644 --- a/ebl/tests/fragmentarium/test_fragment_matcher_route.py +++ b/ebl/tests/fragmentarium/test_fragment_matcher_route.py @@ -19,8 +19,8 @@ def test_fragment_matcher_route(client, fragmentarium, user): {"museumNumber": "X.326", "script": fragment_2.script, "score": 5} ], } - fragmentarium.create(fragment_1) - fragmentarium.create(fragment_2) + fragmentarium.create_many(fragment_1) + fragmentarium.create_many(fragment_2) get_result = client.simulate_get(f"/fragments/{fragment_id}/match") assert get_result.status == falcon.HTTP_OK assert get_result.json == expected_score @@ -33,7 +33,7 @@ def test_fragment_matcher_route_error(client, fragmentarium, user): number=MuseumNumber.of("X.1"), line_to_vec=(LineToVecEncoding.from_list([1, 1, 2]),), ) - fragmentarium.create(fragment_1) - fragmentarium.create(fragment_2) + fragmentarium.create_many(fragment_1) + fragmentarium.create_many(fragment_2) get_result = client.simulate_get(f"/fragments/{faulty_fragment_id}/match") assert get_result.status == falcon.HTTP_UNPROCESSABLE_ENTITY diff --git a/ebl/tests/fragmentarium/test_fragment_repository.py b/ebl/tests/fragmentarium/test_fragment_repository.py index d2eee9bda..71b82e6a4 100644 --- a/ebl/tests/fragmentarium/test_fragment_repository.py +++ b/ebl/tests/fragmentarium/test_fragment_repository.py @@ -70,7 +70,7 @@ def test_create(database, fragment_repository): fragment = LemmatizedFragmentFactory.build() - fragment_id = fragment_repository.create(fragment) + fragment_id = fragment_repository.create_many(fragment) assert fragment_id == str(fragment.number) assert database[COLLECTION].find_one( @@ -136,7 +136,7 @@ def test_query_by_museum_number_references( reference = ReferenceFactory.build(with_document=True) fragment = LemmatizedFragmentFactory.build(references=(reference,)) database[COLLECTION].insert_one(FragmentSchema(exclude=["joins"]).dump(fragment)) - bibliography_repository.create(reference.document) + bibliography_repository.create_many(reference.document) assert fragment_repository.query_by_museum_number(fragment.number) == fragment @@ -149,7 +149,7 @@ def test_find_random(fragment_repository): fragment = FragmentFactory.build() transliterated_fragment = TransliteratedFragmentFactory.build() for a_fragment in fragment, transliterated_fragment: - fragment_repository.create(a_fragment) + fragment_repository.create_many(a_fragment) assert fragment_repository.query_random_by_transliterated() == [ transliterated_fragment @@ -199,7 +199,7 @@ def test_folio_pager_exception(fragment_repository): ) def test_query_next_and_previous_fragment(museum_numbers, fragment_repository): for fragmentNumber in museum_numbers: - fragment_repository.create( + fragment_repository.create_many( FragmentFactory.build(number=MuseumNumber.of(fragmentNumber)) ) for museum_number in museum_numbers: @@ -214,7 +214,7 @@ def test_query_next_and_previous_fragment(museum_numbers, fragment_repository): def test_update_transliteration_with_record(fragment_repository, user): fragment = FragmentFactory.build() - fragment_repository.create(fragment) + fragment_repository.create_many(fragment) updated_fragment = fragment.update_transliteration( TransliterationUpdate(parse_atf_lark("$ (the transliteration)"), "notes"), user ) @@ -233,7 +233,7 @@ def test_update_update_transliteration_not_found(fragment_repository): def test_update_genres(fragment_repository): fragment = FragmentFactory.build(genres=tuple()) - fragment_repository.create(fragment) + fragment_repository.create_many(fragment) updated_fragment = fragment.set_genres( (Genre(["ARCHIVAL", "Administrative"], False),) ) @@ -245,7 +245,7 @@ def test_update_genres(fragment_repository): def test_update_lemmatization(fragment_repository): transliterated_fragment = TransliteratedFragmentFactory.build() - fragment_repository.create(transliterated_fragment) + fragment_repository.create_many(transliterated_fragment) tokens = [list(line) for line in transliterated_fragment.text.lemmatization.tokens] tokens[1][3] = LemmatizationToken(tokens[1][3].value, ("aklu I",)) lemmatization = Lemmatization(tokens) @@ -405,8 +405,8 @@ def test_empty_result_search_reference_id_and_pages( @pytest.mark.parametrize("signs,is_match", SEARCH_SIGNS_DATA) def test_search_signs(signs, is_match, fragment_repository): transliterated_fragment = TransliteratedFragmentFactory.build() - fragment_repository.create(transliterated_fragment) - fragment_repository.create(FragmentFactory.build()) + fragment_repository.create_many(transliterated_fragment) + fragment_repository.create_many(FragmentFactory.build()) result = fragment_repository.query_by_transliteration(TransliterationQuery(signs)) expected = [transliterated_fragment] if is_match else [] @@ -457,7 +457,7 @@ def test_find_transliterated_line_to_vec(database, fragment_repository): def test_update_references(fragment_repository): reference = ReferenceFactory.build() fragment = FragmentFactory.build() - fragment_repository.create(fragment) + fragment_repository.create_many(fragment) references = (reference,) updated_fragment = fragment.set_references(references) diff --git a/ebl/tests/fragmentarium/test_fragment_updater.py b/ebl/tests/fragmentarium/test_fragment_updater.py index 3961b835c..cfe6f5a46 100644 --- a/ebl/tests/fragmentarium/test_fragment_updater.py +++ b/ebl/tests/fragmentarium/test_fragment_updater.py @@ -50,7 +50,7 @@ def test_update_transliteration( .query_by_museum_number(number) .thenReturn(transliterated_fragment) ) - when(changelog).create( + when(changelog).create_many( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(transliterated_fragment)}, @@ -109,7 +109,7 @@ def test_update_genres(fragment_updater, user, fragment_repository, changelog, w expected_fragment = fragment.set_genres(genres) (when(fragment_repository).query_by_museum_number(number).thenReturn(fragment)) - when(changelog).create( + when(changelog).create_many( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(fragment)}, @@ -136,7 +136,7 @@ def test_update_lemmatization( .query_by_museum_number(number) .thenReturn(transliterated_fragment) ) - when(changelog).create( + when(changelog).create_many( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(transliterated_fragment)}, @@ -179,7 +179,7 @@ def test_update_references( .thenReturn(expected_fragment) ) when(fragment_repository).update_references(expected_fragment).thenReturn() - when(changelog).create( + when(changelog).create_many( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(fragment)}, diff --git a/ebl/tests/fragmentarium/test_fragments_route.py b/ebl/tests/fragmentarium/test_fragments_route.py index 325e6cefe..f07bba409 100644 --- a/ebl/tests/fragmentarium/test_fragments_route.py +++ b/ebl/tests/fragmentarium/test_fragments_route.py @@ -7,7 +7,7 @@ def test_get(client, fragmentarium, user): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) result = client.simulate_get(f"/fragments/{transliterated_fragment.number}") assert result.json == create_response_dto( diff --git a/ebl/tests/fragmentarium/test_fragments_search_route.py b/ebl/tests/fragmentarium/test_fragments_search_route.py index 016ce215f..b9ca51da0 100644 --- a/ebl/tests/fragmentarium/test_fragments_search_route.py +++ b/ebl/tests/fragmentarium/test_fragments_search_route.py @@ -26,7 +26,7 @@ def expected_fragment_info_dto(fragment, lines=tuple()): ) def test_search_fragment(get_number, client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) result = client.simulate_get("/fragments", params={"number": get_number(fragment)}) assert result.status == falcon.HTTP_OK @@ -43,8 +43,8 @@ def test_search_fragment_not_found(client): def test_search_references(client, fragmentarium, bibliography, user): bib_entry_1 = BibliographyEntryFactory.build(id="RN.0", pages="254") bib_entry_2 = BibliographyEntryFactory.build(id="RN.1") - bibliography.create(bib_entry_1, user) - bibliography.create(bib_entry_2, user) + bibliography.create_many(bib_entry_1, user) + bibliography.create_many(bib_entry_2, user) fragment = FragmentFactory.build( references=( @@ -52,7 +52,7 @@ def test_search_references(client, fragmentarium, bibliography, user): ReferenceFactory.build(id="RN.1"), ) ) - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) result = client.simulate_get( "/fragments", params={"id": fragment.references[0].id, "pages": fragment.references[0].pages}, @@ -76,7 +76,7 @@ def test_search_references_invalid_query(client, fragmentarium): fragment = FragmentFactory.build( references=(ReferenceFactory.build(), ReferenceFactory.build()) ) - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) reference_id = fragment.references[0].id reference_pages = "should be a number" result = client.simulate_get( @@ -87,9 +87,9 @@ def test_search_references_invalid_query(client, fragmentarium): def test_search_signs(client, fragmentarium, sign_repository, signs): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) result = client.simulate_get("/fragments", params={"transliteration": "ma-tu₂"}) @@ -111,7 +111,7 @@ def test_search_signs_invalid(client, fragmentarium, sign_repository, signs): def test_random(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) result = client.simulate_get("/fragments", params={"random": True}) @@ -124,7 +124,7 @@ def test_interesting(client, fragmentarium): interesting_fragment = InterestingFragmentFactory.build( number=MuseumNumber("K", "1") ) - fragmentarium.create(interesting_fragment) + fragmentarium.create_many(interesting_fragment) result = client.simulate_get("/fragments", params={"interesting": True}) @@ -135,7 +135,7 @@ def test_interesting(client, fragmentarium): def test_latest(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) result = client.simulate_get("/fragments", params={"latest": True}) @@ -146,7 +146,7 @@ def test_latest(client, fragmentarium): def test_needs_revision(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) result = client.simulate_get("/fragments", params={"needsRevision": True}) diff --git a/ebl/tests/fragmentarium/test_lemmatization_route.py b/ebl/tests/fragmentarium/test_lemmatization_route.py index 2cb04f8c1..d59af599c 100644 --- a/ebl/tests/fragmentarium/test_lemmatization_route.py +++ b/ebl/tests/fragmentarium/test_lemmatization_route.py @@ -28,7 +28,7 @@ def test_deserialize_lemmatization(): def test_update_lemmatization(client, fragmentarium, user, database): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) tokens = [list(line) for line in transliterated_fragment.text.lemmatization.tokens] tokens[1][3] = LemmatizationToken(tokens[1][3].value, ("aklu I",)) lemmatization = Lemmatization(tokens) @@ -89,7 +89,7 @@ def test_update_lemmatization_invalid_number(client): ) def test_update_lemmatization_invalid_entity(client, fragmentarium, body): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) url = f"/fragments/{fragment.number}/lemmatization" post_result = client.simulate_post(url, body=body) @@ -99,7 +99,7 @@ def test_update_lemmatization_invalid_entity(client, fragmentarium, body): def test_update_lemmatization_atf_change(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) dto = LemmatizationSchema().dump(transliterated_fragment.text.lemmatization) dto["lemmatization"][0][0]["value"] = "ana" url = f"/fragments/{transliterated_fragment.number}/lemmatization" diff --git a/ebl/tests/fragmentarium/test_pager_route.py b/ebl/tests/fragmentarium/test_pager_route.py index 769316f4e..d755c03a2 100644 --- a/ebl/tests/fragmentarium/test_pager_route.py +++ b/ebl/tests/fragmentarium/test_pager_route.py @@ -9,7 +9,7 @@ def test_get_fragment_pager(client, fragmentarium): fragment_1 = FragmentFactory.build(number=MuseumNumber("X", "1")) fragment_2 = FragmentFactory.build(number=MuseumNumber("X", "2")) for fragment in [fragment_0, fragment_1, fragment_2]: - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) result = client.simulate_get(f"/fragments/{fragment_1.number}/pager") assert result.json == {"next": "X.2", "previous": "X.0"} @@ -21,10 +21,10 @@ def test_get_fragment_pager_cache(cached_client, fragmentarium): fragment_0 = FragmentFactory.build(number=MuseumNumber("X", "0")) fragment_1 = FragmentFactory.build(number=MuseumNumber("X", "3")) for fragment in [fragment_0, fragment_1]: - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) first_result = cached_client.simulate_get(f"/fragments/{fragment_1.number}/pager") - fragmentarium.create(FragmentFactory.build(number=MuseumNumber("X", "2"))) + fragmentarium.create_many(FragmentFactory.build(number=MuseumNumber("X", "2"))) second_result = cached_client.simulate_get(f"/fragments/{fragment_1.number}/pager") assert first_result.json == second_result.json @@ -39,7 +39,7 @@ def test_get_fragment_pager_invalid_id(client): def test_get_folio_pager(client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) result = client.simulate_get(f"/fragments/{fragment.number}/pager/WGL/1") assert result.json == { @@ -57,7 +57,7 @@ def test_get_folio_pager_invalid_id(client): def test_get_folio_pager_no_access_to_folio(client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) result = client.simulate_get(f"/fragments/{fragment.number}/pager/XXX/1") assert result.status == falcon.HTTP_FORBIDDEN diff --git a/ebl/tests/fragmentarium/test_references_route.py b/ebl/tests/fragmentarium/test_references_route.py index 4fe78cb17..3b4bc7e2e 100644 --- a/ebl/tests/fragmentarium/test_references_route.py +++ b/ebl/tests/fragmentarium/test_references_route.py @@ -15,9 +15,9 @@ def test_update_references(client, fragmentarium, bibliography, user): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) reference = ReferenceFactory.build(with_document=True) - bibliography.create(reference.document, ANY_USER) + bibliography.create_many(reference.document, ANY_USER) references = [ReferenceSchema().dump(reference)] body = json.dumps({"references": references}) url = f"/fragments/{fragment.number}/references" @@ -81,7 +81,7 @@ def test_update_references_invalid_museum_number(client): ) def test_update_references_invalid_reference(client, fragmentarium, body): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) url = f"/fragments/{fragment.number}/references" post_result = client.simulate_post(url, body=body) @@ -92,7 +92,7 @@ def test_update_references_invalid_reference(client, fragmentarium, body): def test_update_references_invalid_id(client, fragmentarium): reference = ReferenceFactory.build(with_document=True) fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) body = json.dumps({"references": [ReferenceSchema().dump(reference)]}) url = f"/fragments/{fragment.number}/references" post_result = client.simulate_post(url, body=body) diff --git a/ebl/tests/fragmentarium/test_statistics_route.py b/ebl/tests/fragmentarium/test_statistics_route.py index d43c47c23..42bd85d88 100644 --- a/ebl/tests/fragmentarium/test_statistics_route.py +++ b/ebl/tests/fragmentarium/test_statistics_route.py @@ -17,7 +17,7 @@ def test_get_statistics(guest_client): def test_get_statistics_cache(cached_client, fragmentarium): first_result = cached_client.simulate_get(PATH) transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create(transliterated_fragment) + fragmentarium.create_many(transliterated_fragment) second_result = cached_client.simulate_get(PATH) assert second_result.json == first_result.json diff --git a/ebl/tests/fragmentarium/test_transliteration_query_factory.py b/ebl/tests/fragmentarium/test_transliteration_query_factory.py index 4010bdfd2..c50bb3765 100644 --- a/ebl/tests/fragmentarium/test_transliteration_query_factory.py +++ b/ebl/tests/fragmentarium/test_transliteration_query_factory.py @@ -9,7 +9,7 @@ def test_create_query(sign_repository, signs): for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) factory = TransliterationQueryFactory(sign_repository) atf = "šu\ngid₂" diff --git a/ebl/tests/fragmentarium/test_transliteration_update_factory.py b/ebl/tests/fragmentarium/test_transliteration_update_factory.py index fa0a35577..4427291db 100644 --- a/ebl/tests/fragmentarium/test_transliteration_update_factory.py +++ b/ebl/tests/fragmentarium/test_transliteration_update_factory.py @@ -11,7 +11,7 @@ def test_create(sign_repository, signs): for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) factory = TransliterationUpdateFactory(sign_repository) atf = Atf("1. šu gid₂") diff --git a/ebl/tests/fragmentarium/test_transliterations_route.py b/ebl/tests/fragmentarium/test_transliterations_route.py index d4696a759..659e58e37 100644 --- a/ebl/tests/fragmentarium/test_transliterations_route.py +++ b/ebl/tests/fragmentarium/test_transliterations_route.py @@ -15,7 +15,7 @@ @freeze_time("2018-09-07 15:41:24.032") def test_update_transliteration(client, fragmentarium, user, database): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) updates = {"transliteration": "$ (the transliteration)", "notes": "some notes"} body = json.dumps(updates) url = f"/fragments/{fragment.number}/transliteration" @@ -65,13 +65,13 @@ def test_update_transliteration_merge_lemmatization( ): for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) lemmatized_fragment = LemmatizedFragmentFactory.build() - fragmentarium.create(lemmatized_fragment) + fragmentarium.create_many(lemmatized_fragment) lines = lemmatized_fragment.text.atf.split("\n") lines[1] = new_transliteration updates = {"transliteration": "\n".join(lines), "notes": lemmatized_fragment.notes} - updated_transliteration = transliteration_factory.create( + updated_transliteration = transliteration_factory.create_many( updates["transliteration"], updates["notes"] ) expected_json = create_response_dto( @@ -96,7 +96,7 @@ def test_update_transliteration_merge_lemmatization( def test_update_transliteration_invalid_atf(client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) updates = {"transliteration": "1. kururu", "notes": ""} body = json.dumps(updates) url = f"/fragments/{fragment.number}/transliteration" @@ -160,7 +160,7 @@ def test_update_transliteration_invalid(client): ) def test_update_transliteration_invalid_entity(client, fragmentarium, body): fragment = FragmentFactory.build() - fragmentarium.create(fragment) + fragmentarium.create_many(fragment) url = f"/fragments/{fragment.number}/transliteration" post_result = client.simulate_post(url, body=body) diff --git a/ebl/tests/lemmatization/test_lemma_search_route.py b/ebl/tests/lemmatization/test_lemma_search_route.py index 75091985e..ea1572e67 100644 --- a/ebl/tests/lemmatization/test_lemma_search_route.py +++ b/ebl/tests/lemmatization/test_lemma_search_route.py @@ -13,8 +13,8 @@ def test_search_fragment( ): expected_word = {**word, "_id": lemma} lemmatized_fragment = LemmatizedFragmentFactory.build() - fragmentarium.create(lemmatized_fragment) - dictionary.create(expected_word) + fragmentarium.create_many(lemmatized_fragment) + dictionary.create_many(expected_word) result = client.simulate_get( "/lemmas", params={"word": query_word, "isNormalized": is_normalized} ) diff --git a/ebl/tests/lemmatization/test_suggestion_finder.py b/ebl/tests/lemmatization/test_suggestion_finder.py index d4079c70c..e78767c2d 100644 --- a/ebl/tests/lemmatization/test_suggestion_finder.py +++ b/ebl/tests/lemmatization/test_suggestion_finder.py @@ -58,24 +58,24 @@ def test_query_lemmas(fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create(lemmatized_fragment) - fragment_repository.create(ANOTHER_LEMMATIZED_FRAGMENT) + fragment_repository.create_many(lemmatized_fragment) + fragment_repository.create_many(ANOTHER_LEMMATIZED_FRAGMENT) assert lemma_repository.query_lemmas("GI₆", False) == [["ginâ I"]] def test_query_lemmas_normalized(fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create(lemmatized_fragment) - fragment_repository.create(ANOTHER_LEMMATIZED_FRAGMENT) + fragment_repository.create_many(lemmatized_fragment) + fragment_repository.create_many(ANOTHER_LEMMATIZED_FRAGMENT) assert lemma_repository.query_lemmas("ana", True) == [["normalized I"]] def test_query_lemmas_multiple(fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create(lemmatized_fragment) - fragment_repository.create(ANOTHER_LEMMATIZED_FRAGMENT) + fragment_repository.create_many(lemmatized_fragment) + fragment_repository.create_many(ANOTHER_LEMMATIZED_FRAGMENT) assert lemma_repository.query_lemmas("ana", False) == [["ana II"], ["ana I"]] @@ -154,7 +154,7 @@ def test_query_lemmas_ignores_in_value( ), signs="DIŠ", ) - fragment_repository.create(fragment) + fragment_repository.create_many(fragment) assert lemma_repository.query_lemmas("ana", False) == expected @@ -162,7 +162,7 @@ def test_query_lemmas_ignores_in_value( @pytest.mark.parametrize("is_normalized", [False, True]) def test_query_lemmas_not_found(is_normalized, fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create(lemmatized_fragment) + fragment_repository.create_many(lemmatized_fragment) assert lemma_repository.query_lemmas("aklu", is_normalized) == [] diff --git a/ebl/tests/signs/test_sign_images_route.py b/ebl/tests/signs/test_sign_images_route.py index 5f67f721d..5d16a9804 100644 --- a/ebl/tests/signs/test_sign_images_route.py +++ b/ebl/tests/signs/test_sign_images_route.py @@ -23,14 +23,14 @@ def test_signs_get( fragment = TransliteratedFragmentFactory.build( number=MuseumNumber.of("K.2"), text=text_with_labels ) - fragment_repository.create(fragment) + fragment_repository.create_many(fragment) annotation_data = AnnotationDataFactory.build(sign_name="signName", path=[2, 0, 0]) cropped_sign = CroppedSignFactory.build(script=fragment.script) annotation = AnnotationFactory.build( data=annotation_data, cropped_sign=cropped_sign ) - cropped_sign_images_repository.create( + cropped_sign_images_repository.create_many( [ CroppedSignImage( annotation.cropped_sign.image_id, Base64("test-base64-string") diff --git a/ebl/tests/signs/test_sign_repository.py b/ebl/tests/signs/test_sign_repository.py index 990c9c5f7..9f956a329 100644 --- a/ebl/tests/signs/test_sign_repository.py +++ b/ebl/tests/signs/test_sign_repository.py @@ -153,7 +153,7 @@ def sign_si_2(mongo_sign_si_2): def test_create(database, sign_repository, sign_igi): - sign_name = sign_repository.create(sign_igi) + sign_name = sign_repository.create_many(sign_igi) assert database[COLLECTION].find_one({"_id": sign_name}) == SignSchema().dump( sign_igi diff --git a/ebl/tests/signs/test_sign_search_route.py b/ebl/tests/signs/test_sign_search_route.py index bbc962122..95a44fe18 100644 --- a/ebl/tests/signs/test_sign_search_route.py +++ b/ebl/tests/signs/test_sign_search_route.py @@ -7,7 +7,7 @@ def test_signs_get(client, sign_repository): sign = Sign(SignName("test")) - sign_repository.create(sign) + sign_repository.create_many(sign) result = client.simulate_get(f"/signs/{sign.name}") assert result.json == SignDtoSchema().dump(sign) @@ -87,7 +87,7 @@ def test_signs_not_found(client): ) def test_signs_search_route(params, expected, client, sign_repository, signs): for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) get_result = client.simulate_get("/signs", params=params) assert get_result.status == falcon.HTTP_OK assert get_result.json == expected @@ -95,6 +95,6 @@ def test_signs_search_route(params, expected, client, sign_repository, signs): def test_signs_search_route_error(client, sign_repository, signs): for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) get_result = client.simulate_get("/signs", params={"signs": "P₂"}) assert get_result.status == falcon.HTTP_UNPROCESSABLE_ENTITY diff --git a/ebl/tests/test_changelog.py b/ebl/tests/test_changelog.py index 1e5cbeb61..b256442d0 100644 --- a/ebl/tests/test_changelog.py +++ b/ebl/tests/test_changelog.py @@ -10,6 +10,6 @@ @freeze_time("2018-09-07 15:41:24.032") def test_create(database, changelog, user, make_changelog_entry): - entry_id = changelog.create(RESOURCE_TYPE, user.profile, OLD, NEW) + entry_id = changelog.create_many(RESOURCE_TYPE, user.profile, OLD, NEW) expected = make_changelog_entry(RESOURCE_TYPE, RESOURCE_ID, OLD, NEW) assert database[COLLECTION].find_one({"_id": entry_id}, {"_id": 0}) == expected diff --git a/ebl/tests/test_mongo_collection.py b/ebl/tests/test_mongo_collection.py index 1b4456e6e..0c3c82fa5 100644 --- a/ebl/tests/test_mongo_collection.py +++ b/ebl/tests/test_mongo_collection.py @@ -9,6 +9,15 @@ def collection(database): return MongoCollection(database, "collection") +def test_create_many_and_find_by_id(collection): + documents = [{"data": "payload-1"}, {"data": "payload-2"}] + + insert_ids = collection.insert_many(documents) + assert [ + collection.find_one_by_id(insert_id) for insert_id in insert_ids + ] == documents + + def test_create_and_find_by_id(collection): document = {"data": "payload"} insert_id = collection.insert_one(document) diff --git a/ebl/tests/transliteration/test_signs_visitor.py b/ebl/tests/transliteration/test_signs_visitor.py index d81b396bb..36ae225dd 100644 --- a/ebl/tests/transliteration/test_signs_visitor.py +++ b/ebl/tests/transliteration/test_signs_visitor.py @@ -58,7 +58,7 @@ ) def test_signs_visitor(text: str, expected: Sequence[str], sign_repository, signs): for sign in signs: - sign_repository.create(sign) + sign_repository.create_many(sign) visitor = SignsVisitor(sign_repository) parse_line(f"1. {text}").accept(visitor) From de27acfa55eb26bbbc31a4f6b55d8147b036909d Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 18:22:44 +0100 Subject: [PATCH 13/23] migration --- .../application/annotations_service.py | 16 +++++++++++++-- .../mongo_annotations_repository.py | 2 +- ebl/fragmentarium/migrate_annotations.py | 20 +++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 ebl/fragmentarium/migrate_annotations.py diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 2d33ae494..17ed5af3a 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -22,7 +22,7 @@ from ebl.fragmentarium.domain.annotation import ( Annotations, Annotation, - BoundingBox, + BoundingBox, AnnotationValueType, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.transliteration.domain.line_label import LineLabel @@ -125,10 +125,13 @@ def _cropped_image_from_annotations( ) ), None, - ) + ) if annotation.data.type != AnnotationValueType.BLANK else "" + cropped_image_base64 = self._cropped_image_from_annotation( annotation, image ) + + image_id = str(bson.ObjectId()) cropped_sign_images.append(CroppedSignImage(image_id, cropped_image_base64)) @@ -167,3 +170,12 @@ def update(self, annotations: Annotations, user: User) -> Annotations: self._annotations_repository.create_or_update(annotations_with_image_ids) self._cropped_sign_images_repository.create(cropped_sign_images) return annotations_with_image_ids + + def migrate(self, annotations: Annotations) -> Annotations: + ( + annotations_with_image_ids, + cropped_sign_images, + ) = self._cropped_image_from_annotations(annotations) + self._annotations_repository.create_or_update(annotations_with_image_ids) + self._cropped_sign_images_repository.create(cropped_sign_images) + return annotations_with_image_ids diff --git a/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py b/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py index f34b68a68..b9e95a685 100644 --- a/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py +++ b/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py @@ -11,7 +11,7 @@ from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.mongo_collection import MongoCollection -COLLECTION = "annotations" +COLLECTION = "annotations_copy" def has_none_values(dictionary: dict) -> bool: diff --git a/ebl/fragmentarium/migrate_annotations.py b/ebl/fragmentarium/migrate_annotations.py new file mode 100644 index 000000000..099e617a7 --- /dev/null +++ b/ebl/fragmentarium/migrate_annotations.py @@ -0,0 +1,20 @@ +from ebl.app import create_context +from ebl.fragmentarium.application.annotations_service import AnnotationsService + +if __name__ == "__main__": + context = create_context() + annotations = context.annotations_repository.retrieve_all_non_empty() + service = AnnotationsService( + context.ebl_ai_client, + context.annotations_repository, + context.photo_repository, + context.changelog, + context.fragment_repository, + context.photo_repository, + context.cropped_sign_images_repository, + ) + print(f"Lenght {len(annotations)}") + for counter, annotation in enumerate(annotations): + service.migrate(annotation) + + print("Update annotations completed!") From 00037a2687e3f09301c1cb2f14742160c149dd4d Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 18:25:20 +0100 Subject: [PATCH 14/23] format --- .../application/annotations_service.py | 28 +++++++++++-------- ebl/fragmentarium/migrate_annotations.py | 1 + 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 17ed5af3a..f26ca6785 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -22,7 +22,8 @@ from ebl.fragmentarium.domain.annotation import ( Annotations, Annotation, - BoundingBox, AnnotationValueType, + BoundingBox, + AnnotationValueType, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.transliteration.domain.line_label import LineLabel @@ -116,22 +117,25 @@ def _cropped_image_from_annotations( for annotation in annotations.annotations: script = fragment.script labels = fragment.text.labels - label = next( - ( - label - for label in labels - if self._is_matching_number( - label.line_number, annotation.data.path[0] - ) - ), - None, - ) if annotation.data.type != AnnotationValueType.BLANK else "" + label = ( + next( + ( + label + for label in labels + if self._is_matching_number( + label.line_number, annotation.data.path[0] + ) + ), + None, + ) + if annotation.data.type != AnnotationValueType.BLANK + else None + ) cropped_image_base64 = self._cropped_image_from_annotation( annotation, image ) - image_id = str(bson.ObjectId()) cropped_sign_images.append(CroppedSignImage(image_id, cropped_image_base64)) diff --git a/ebl/fragmentarium/migrate_annotations.py b/ebl/fragmentarium/migrate_annotations.py index 099e617a7..bc45c8d4b 100644 --- a/ebl/fragmentarium/migrate_annotations.py +++ b/ebl/fragmentarium/migrate_annotations.py @@ -15,6 +15,7 @@ ) print(f"Lenght {len(annotations)}") for counter, annotation in enumerate(annotations): + print(counter) service.migrate(annotation) print("Update annotations completed!") From 5c6609eff7d17a9670e7b02f7aa513d956994ca1 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 18:27:27 +0100 Subject: [PATCH 15/23] use real repo name --- .../infrastructure/mongo_annotations_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py b/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py index b9e95a685..f34b68a68 100644 --- a/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py +++ b/ebl/fragmentarium/infrastructure/mongo_annotations_repository.py @@ -11,7 +11,7 @@ from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.mongo_collection import MongoCollection -COLLECTION = "annotations_copy" +COLLECTION = "annotations" def has_none_values(dictionary: dict) -> bool: From 98b0b25bc71182b758c13eb811aa78796e67968b Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 18:39:31 +0100 Subject: [PATCH 16/23] revert refactor --- ebl/bibliography/web/bibliography_entries.py | 2 +- ebl/corpus/application/corpus.py | 2 +- .../application/annotations_service.py | 24 +++++++++++-------- ebl/fragmentarium/web/transliterations.py | 2 +- ebl/tests/bibliography/test_bibliography.py | 20 +++++++--------- .../test_bibliography_repository.py | 8 +++---- .../bibliography/test_bibliography_route.py | 2 +- ebl/tests/corpus/support.py | 4 ++-- .../test_chapter_display_lines_route.py | 8 +++---- .../corpus/test_chapter_display_route.py | 4 ++-- ebl/tests/corpus/test_corpus.py | 4 ++-- .../corpus/test_mongo_text_repository.py | 18 +++++++------- ebl/tests/corpus/test_texts_route.py | 6 ++--- ebl/tests/dictionary/test_dictionary.py | 18 +++++++------- ebl/tests/dictionary/test_word_repositor.py | 4 ++-- ebl/tests/dictionary/test_words_route.py | 2 +- .../fragmentarium/test_annotations_route.py | 2 +- .../fragmentarium/test_annotations_service.py | 2 +- .../test_fragment_genre_route.py | 4 ++-- .../test_fragment_matcher_route.py | 8 +++---- .../fragmentarium/test_fragment_repository.py | 20 ++++++++-------- .../fragmentarium/test_fragment_updater.py | 8 +++---- .../fragmentarium/test_fragments_route.py | 2 +- .../test_fragments_search_route.py | 22 ++++++++--------- .../fragmentarium/test_lemmatization_route.py | 6 ++--- ebl/tests/fragmentarium/test_pager_route.py | 10 ++++---- .../fragmentarium/test_references_route.py | 8 +++---- .../fragmentarium/test_statistics_route.py | 2 +- .../test_transliteration_query_factory.py | 2 +- .../test_transliteration_update_factory.py | 2 +- .../test_transliterations_route.py | 12 +++++----- .../lemmatization/test_lemma_search_route.py | 4 ++-- .../lemmatization/test_suggestion_finder.py | 16 ++++++------- ebl/tests/signs/test_sign_images_route.py | 4 ++-- ebl/tests/signs/test_sign_repository.py | 2 +- ebl/tests/signs/test_sign_search_route.py | 6 ++--- ebl/tests/test_changelog.py | 2 +- .../transliteration/test_signs_visitor.py | 2 +- 38 files changed, 137 insertions(+), 137 deletions(-) diff --git a/ebl/bibliography/web/bibliography_entries.py b/ebl/bibliography/web/bibliography_entries.py index cda9ffea0..fb00e8372 100644 --- a/ebl/bibliography/web/bibliography_entries.py +++ b/ebl/bibliography/web/bibliography_entries.py @@ -18,7 +18,7 @@ def on_get(self, req: Request, resp: Response) -> None: @validate(CSL_JSON_SCHEMA) def on_post(self, req: Request, resp: Response) -> None: bibliography_entry = req.media - self._bibliography.create_many(bibliography_entry, req.context.user) + self._bibliography.create(bibliography_entry, req.context.user) resp.status = falcon.HTTP_CREATED resp.location = f'/bibliography/{bibliography_entry["id"]}' resp.media = bibliography_entry diff --git a/ebl/corpus/application/corpus.py b/ebl/corpus/application/corpus.py index efd156bb9..ea768ecd9 100644 --- a/ebl/corpus/application/corpus.py +++ b/ebl/corpus/application/corpus.py @@ -206,4 +206,4 @@ def _hydrate_references(self, chapter: Chapter) -> Chapter: def _create_changelog(self, old: Chapter, new: Chapter, user: User) -> None: old_dict: dict = {**ChapterSchema().dump(old), "_id": old.id_.to_tuple()} new_dict: dict = {**ChapterSchema().dump(new), "_id": new.id_.to_tuple()} - self._changelog.create_many(COLLECTION, user.profile, old_dict, new_dict) + self._changelog.create(COLLECTION, user.profile, old_dict, new_dict) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 4be62d397..d068ff622 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -23,6 +23,7 @@ Annotations, Annotation, BoundingBox, + AnnotationValueType, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.transliteration.domain.line_label import LineLabel @@ -116,21 +117,24 @@ def _cropped_image_from_annotations( for annotation in annotations.annotations: script = fragment.script labels = fragment.text.labels - label = next( - ( - label - for label in labels - if self._is_matching_number( - label.line_number, annotation.data.path[0] - ) - ), - None, + label = ( + next( + ( + label + for label in labels + if self._is_matching_number( + label.line_number, annotation.data.path[0] + ) + ), + None, + ) + if annotation.data.type != AnnotationValueType.BLANK + else None ) cropped_image_base64 = self._cropped_image_from_annotation( annotation, image ) image_id = str(bson.ObjectId()) - cropped_sign_images.append(CroppedSignImage(image_id, cropped_image_base64)) cropped_annotations.append( attr.evolve( diff --git a/ebl/fragmentarium/web/transliterations.py b/ebl/fragmentarium/web/transliterations.py index 6ad3a8ac2..4de894581 100644 --- a/ebl/fragmentarium/web/transliterations.py +++ b/ebl/fragmentarium/web/transliterations.py @@ -45,7 +45,7 @@ def on_post(self, req: Request, resp: Response, number: str) -> None: def _create_transliteration(self, media): try: - return self._transliteration_factory.create_many( + return self._transliteration_factory.create( Atf(media["transliteration"]), media["notes"] ) except ValueError as error: diff --git a/ebl/tests/bibliography/test_bibliography.py b/ebl/tests/bibliography/test_bibliography.py index 81cc43714..1ed23d91a 100644 --- a/ebl/tests/bibliography/test_bibliography.py +++ b/ebl/tests/bibliography/test_bibliography.py @@ -69,7 +69,7 @@ def test_create( bibliography_entry = BibliographyEntryFactory.build() ( when(changelog) - .create_many( + .create( COLLECTION, user.profile, {"_id": bibliography_entry["id"]}, @@ -77,9 +77,9 @@ def test_create( ) .thenReturn() ) - (when(bibliography_repository).create_many(bibliography_entry).thenReturn()) + (when(bibliography_repository).create(bibliography_entry).thenReturn()) - bibliography.create_many(bibliography_entry, user) + bibliography.create(bibliography_entry, user) def test_create_duplicate( @@ -93,7 +93,7 @@ def test_create_duplicate( bibliography_entry = BibliographyEntryFactory.build() ( when(changelog) - .create_many( + .create( COLLECTION, user.profile, {"_id": bibliography_entry["id"]}, @@ -101,13 +101,9 @@ def test_create_duplicate( ) .thenReturn() ) - ( - when(bibliography_repository) - .create_many(bibliography_entry) - .thenRaise(DuplicateError) - ) + (when(bibliography_repository).create(bibliography_entry).thenRaise(DuplicateError)) with pytest.raises(DuplicateError): - bibliography.create_many(bibliography_entry, user) + bibliography.create(bibliography_entry, user) def test_entry_not_found(bibliography, bibliography_repository, when): @@ -137,7 +133,7 @@ def test_update( ) ( when(changelog) - .create_many( + .create( COLLECTION, user.profile, create_mongo_bibliography_entry(), @@ -175,7 +171,7 @@ def test_validate_references_invalid( valid_reference = ReferenceFactory.build(with_document=True) first_invalid = ReferenceFactory.build(with_document=True) second_invalid = ReferenceFactory.build(with_document=True) - bibliography.create_many(valid_reference.document, user) + bibliography.create(valid_reference.document, user) (when(bibliography).find(valid_reference.id).thenReturn(valid_reference)) (when(bibliography).find(first_invalid.id).thenRaise(NotFoundError)) (when(bibliography).find(second_invalid.id).thenRaise(NotFoundError)) diff --git a/ebl/tests/bibliography/test_bibliography_repository.py b/ebl/tests/bibliography/test_bibliography_repository.py index b6d03128b..1e93d4a4f 100644 --- a/ebl/tests/bibliography/test_bibliography_repository.py +++ b/ebl/tests/bibliography/test_bibliography_repository.py @@ -9,7 +9,7 @@ def test_create(database, bibliography_repository, create_mongo_bibliography_entry): bibliography_entry = BibliographyEntryFactory.build() - bibliography_repository.create_many(bibliography_entry) + bibliography_repository.create(bibliography_entry) assert ( database[COLLECTION].find_one({"_id": bibliography_entry["id"]}) @@ -19,9 +19,9 @@ def test_create(database, bibliography_repository, create_mongo_bibliography_ent def test_create_duplicate(bibliography_repository): bibliography_entry = BibliographyEntryFactory.build() - bibliography_repository.create_many(bibliography_entry) + bibliography_repository.create(bibliography_entry) with pytest.raises(DuplicateError): - bibliography_repository.create_many(bibliography_entry) + bibliography_repository.create(bibliography_entry) def test_find(database, bibliography_repository, create_mongo_bibliography_entry): @@ -43,7 +43,7 @@ def test_entry_not_found(bibliography_repository): def test_update(bibliography_repository): bibliography_entry = BibliographyEntryFactory.build() updated_entry = pydash.omit({**bibliography_entry, "title": "New Title"}, "PMID") - bibliography_repository.create_many(bibliography_entry) + bibliography_repository.create(bibliography_entry) bibliography_repository.update(updated_entry) assert ( diff --git a/ebl/tests/bibliography/test_bibliography_route.py b/ebl/tests/bibliography/test_bibliography_route.py index 77dfece2a..e0e6caeb6 100644 --- a/ebl/tests/bibliography/test_bibliography_route.py +++ b/ebl/tests/bibliography/test_bibliography_route.py @@ -15,7 +15,7 @@ @pytest.fixture def saved_entry(bibliography, user): bibliography_entry = BibliographyEntryFactory.build() - bibliography.create_many(bibliography_entry, user) + bibliography.create(bibliography_entry, user) return bibliography_entry diff --git a/ebl/tests/corpus/support.py b/ebl/tests/corpus/support.py index 836b23ad0..927ed773e 100644 --- a/ebl/tests/corpus/support.py +++ b/ebl/tests/corpus/support.py @@ -9,12 +9,12 @@ def allow_references(chapter, bibliography): for manuscript in chapter.manuscripts: for reference in manuscript.references: - bibliography.create_many(reference.document, ANY_USER) + bibliography.create(reference.document, ANY_USER) def allow_signs(signs, sign_list): for sign in signs: - sign_list.create_many(sign) + sign_list.create(sign) def create_chapter_dto(chapter: Chapter) -> dict: diff --git a/ebl/tests/corpus/test_chapter_display_lines_route.py b/ebl/tests/corpus/test_chapter_display_lines_route.py index 0a0b20fef..7d934a72f 100644 --- a/ebl/tests/corpus/test_chapter_display_lines_route.py +++ b/ebl/tests/corpus/test_chapter_display_lines_route.py @@ -28,7 +28,7 @@ def url(chapter: Chapter) -> str: def test_get(client, text_repository, text, chapter, bibliography, url): - text_repository.create_many(text) + text_repository.create(text) text_repository.create_chapter(chapter) allow_references(chapter, bibliography) schema = LineDetailsSchema(context={"manuscripts": chapter.manuscripts}) @@ -40,7 +40,7 @@ def test_get(client, text_repository, text, chapter, bibliography, url): def test_chapter_not_found(client, text_repository, text, chapter, url): - text_repository.create_many(text) + text_repository.create(text) result = client.simulate_get(f"{url}/0") @@ -48,7 +48,7 @@ def test_chapter_not_found(client, text_repository, text, chapter, url): def test_line_not_found(client, text_repository, chapter, bibliography, url): - text_repository.create_many(text) + text_repository.create(text) text_repository.create_chapter(chapter) allow_references(chapter, bibliography) @@ -58,7 +58,7 @@ def test_line_not_found(client, text_repository, chapter, bibliography, url): def test_invalid_line(client, text_repository, chapter, bibliography, url): - text_repository.create_many(text) + text_repository.create(text) text_repository.create_chapter(chapter) allow_references(chapter, bibliography) diff --git a/ebl/tests/corpus/test_chapter_display_route.py b/ebl/tests/corpus/test_chapter_display_route.py index 11e0fbc75..5bf4e111c 100644 --- a/ebl/tests/corpus/test_chapter_display_route.py +++ b/ebl/tests/corpus/test_chapter_display_route.py @@ -29,7 +29,7 @@ def url(chapter: Chapter) -> str: def test_get(client, text_repository, text, chapter, url): - text_repository.create_many(text) + text_repository.create(text) text_repository.create_chapter(chapter) chapter_display = ChapterDisplay.of_chapter(text, chapter) @@ -40,7 +40,7 @@ def test_get(client, text_repository, text, chapter, url): def test_text_not_found(client, text_repository, text, chapter, url): - text_repository.create_many(text) + text_repository.create(text) result = client.simulate_get(url) diff --git a/ebl/tests/corpus/test_corpus.py b/ebl/tests/corpus/test_corpus.py index 010e97f87..2af50134d 100644 --- a/ebl/tests/corpus/test_corpus.py +++ b/ebl/tests/corpus/test_corpus.py @@ -59,7 +59,7 @@ def expect_invalid_references(bibliography, when) -> None: def expect_signs(signs, sign_repository) -> None: for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) def expect_chapter_update( @@ -77,7 +77,7 @@ def expect_chapter_update( when(text_repository).update(CHAPTER.id_, updated_chapter).thenReturn( updated_chapter ) - when(changelog).create_many( + when(changelog).create( CHAPTERS_COLLECTION, user.profile, {**ChapterSchema().dump(old_chapter), "_id": old_chapter.id_.to_tuple()}, diff --git a/ebl/tests/corpus/test_mongo_text_repository.py b/ebl/tests/corpus/test_mongo_text_repository.py index a7d230750..1a29ab37e 100644 --- a/ebl/tests/corpus/test_mongo_text_repository.py +++ b/ebl/tests/corpus/test_mongo_text_repository.py @@ -53,7 +53,7 @@ def when_chapter_in_collection(database, chapter=CHAPTER) -> None: def test_creating_text(database, text_repository) -> None: - text_repository.create_many(TEXT) + text_repository.create(TEXT) inserted_text = database[TEXTS_COLLECTION].find_one( {"category": TEXT.category, "index": TEXT.index}, projection={"_id": False} @@ -78,18 +78,18 @@ def test_creating_chapter(database, text_repository) -> None: def test_it_is_not_possible_to_create_duplicate_texts(text_repository) -> None: text_repository.create_indexes() - text_repository.create_many(TEXT) + text_repository.create(TEXT) with pytest.raises(DuplicateError): - text_repository.create_many(TEXT) + text_repository.create(TEXT) def test_it_is_not_possible_to_create_duplicate_chapters(text_repository) -> None: text_repository.create_indexes() - text_repository.create_many(CHAPTER) + text_repository.create(CHAPTER) with pytest.raises(DuplicateError): - text_repository.create_many(CHAPTER) + text_repository.create(CHAPTER) def test_finding_text( @@ -107,9 +107,9 @@ def test_finding_text( chapter = attr.evolve(CHAPTER, uncertain_fragments=(UNCERTAIN_FRAGMET,)) when_text_in_collection(database, text) when_chapter_in_collection(database, chapter) - fragment_repository.create_many(Fragment(UNCERTAIN_FRAGMET)) + fragment_repository.create(Fragment(UNCERTAIN_FRAGMET)) for reference in text.references: - bibliography_repository.create_many(reference.document) + bibliography_repository.create(reference.document) assert text_repository.find(text.id) == text @@ -133,7 +133,7 @@ def test_listing_texts(database, text_repository, bibliography_repository) -> No when_chapter_in_collection(database) when_chapter_in_collection(database, another_chapter) for reference in TEXT.references: - bibliography_repository.create_many(reference.document) + bibliography_repository.create(reference.document) assert text_repository.list() == [TEXT, another_text] @@ -225,7 +225,7 @@ def test_query_manuscripts_with_joins_by_chapter_no_joins( def test_query_manuscripts_with_joins_is_in_fragmentarium( database, text_repository, fragment_repository ) -> None: - fragment_repository.create_many(FragmentFactory.build(number=MUSEUM_NUMBER)) + fragment_repository.create(FragmentFactory.build(number=MUSEUM_NUMBER)) when_chapter_in_collection(database) assert text_repository.query_manuscripts_with_joins_by_chapter(CHAPTER.id_) == [ diff --git a/ebl/tests/corpus/test_texts_route.py b/ebl/tests/corpus/test_texts_route.py index e19b65012..d088b4aa9 100644 --- a/ebl/tests/corpus/test_texts_route.py +++ b/ebl/tests/corpus/test_texts_route.py @@ -14,7 +14,7 @@ def create_dto(text): def test_get_text(client, bibliography, sign_repository, signs, text_repository): text = TextFactory.build(chapters=tuple(), references=tuple()) - text_repository.create_many(text) + text_repository.create(text) get_result = client.simulate_get( f"/texts/{text.genre.value}/{text.category}/{text.index}" @@ -45,8 +45,8 @@ def test_invalid_index(client): def test_listing_texts(client, bibliography, sign_repository, signs, text_repository): first_text = TextFactory.build(chapters=tuple(), references=tuple()) second_text = TextFactory.build(chapters=tuple(), references=tuple()) - text_repository.create_many(first_text) - text_repository.create_many(second_text) + text_repository.create(first_text) + text_repository.create(second_text) get_result = client.simulate_get("/texts") diff --git a/ebl/tests/dictionary/test_dictionary.py b/ebl/tests/dictionary/test_dictionary.py index d0f133cd6..bca260caa 100644 --- a/ebl/tests/dictionary/test_dictionary.py +++ b/ebl/tests/dictionary/test_dictionary.py @@ -8,7 +8,7 @@ def test_create_and_find(database, dictionary, word): - word_id = dictionary.create_many(word) + word_id = dictionary.create(word) assert dictionary.find(word_id) == word @@ -20,8 +20,8 @@ def test_word_not_found(dictionary): def test_search_finds_all_homonyms(dictionary, word): another_word = {**word, "_id": "part1 part2 II", "homonym": "II"} - dictionary.create_many(word) - dictionary.create_many(another_word) + dictionary.create(word) + dictionary.create(another_word) assert dictionary.search(" ".join(word["lemma"])) == [word, another_word] @@ -33,16 +33,16 @@ def test_search_finds_by_meaning(dictionary, word): "homonym": "II", "meaning": "not matching", } - dictionary.create_many(word) - dictionary.create_many(another_word) + dictionary.create(word) + dictionary.create(another_word) assert dictionary.search(word["meaning"][1:4]) == [word] def test_search_finds_duplicates(dictionary, word): another_word = {**word, "_id": "part1 part2 II", "homonym": "II"} - dictionary.create_many(word) - dictionary.create_many(another_word) + dictionary.create(word) + dictionary.create(another_word) assert dictionary.search(word["meaning"][1:4]) == [word, another_word] @@ -53,7 +53,7 @@ def test_search_not_found(dictionary): def test_update(dictionary, word, user): new_lemma = ["new"] - word_id = dictionary.create_many(word) + word_id = dictionary.create(word) updated_word = pydash.defaults({"lemma": new_lemma}, word) dictionary.update(updated_word, user) @@ -63,7 +63,7 @@ def test_update(dictionary, word, user): @freeze_time("2018-09-07 15:41:24.032") def test_changelog(dictionary, word, user, database, make_changelog_entry): - word_id = dictionary.create_many(word) + word_id = dictionary.create(word) updated_word = pydash.defaults({"lemma": ["new"]}, word) dictionary.update(updated_word, user) diff --git a/ebl/tests/dictionary/test_word_repositor.py b/ebl/tests/dictionary/test_word_repositor.py index f03863350..90b77bbc4 100644 --- a/ebl/tests/dictionary/test_word_repositor.py +++ b/ebl/tests/dictionary/test_word_repositor.py @@ -7,7 +7,7 @@ def test_create(database, word_repository, word): - word_id = word_repository.create_many(word) + word_id = word_repository.create(word) assert database[COLLECTION].find_one({"_id": word_id}) == word @@ -63,7 +63,7 @@ def test_search_not_found(word_repository): def test_update(word_repository, word): new_lemma = ["new"] - word_id = word_repository.create_many(word) + word_id = word_repository.create(word) updated_word = pydash.defaults({"lemma": new_lemma}, word) word_repository.update(updated_word) diff --git a/ebl/tests/dictionary/test_words_route.py b/ebl/tests/dictionary/test_words_route.py index 7747ddabb..a0c04e9cf 100644 --- a/ebl/tests/dictionary/test_words_route.py +++ b/ebl/tests/dictionary/test_words_route.py @@ -7,7 +7,7 @@ @pytest.fixture def saved_word(dictionary, word): word = {**word} - dictionary.create_many(word) + dictionary.create(word) return word diff --git a/ebl/tests/fragmentarium/test_annotations_route.py b/ebl/tests/fragmentarium/test_annotations_route.py index d61299ec9..aa0c94553 100644 --- a/ebl/tests/fragmentarium/test_annotations_route.py +++ b/ebl/tests/fragmentarium/test_annotations_route.py @@ -75,7 +75,7 @@ def test_update(client, fragment_repository, photo_repository): fragment_number=number, annotations=[annotation_1, annotation_2] ) fragment = TransliteratedFragmentFactory.build(number=number) - fragment_repository.create_many(fragment) + fragment_repository.create(fragment) photo_repository._create(create_test_photo(number)) body = AnnotationsSchema().dump(annotations) diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index e3ebce94b..af9de2fe8 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -177,7 +177,7 @@ def test_update( annotations ) when(annotations_repository).create_or_update(updated_annotations).thenReturn() - when(changelog).create_many( + when(changelog).create( "annotations", user.profile, {"_id": str(fragment_number), **SCHEMA.dump(annotations)}, diff --git a/ebl/tests/fragmentarium/test_fragment_genre_route.py b/ebl/tests/fragmentarium/test_fragment_genre_route.py index a8c20eb5c..3eb4e4f6d 100644 --- a/ebl/tests/fragmentarium/test_fragment_genre_route.py +++ b/ebl/tests/fragmentarium/test_fragment_genre_route.py @@ -31,7 +31,7 @@ ) def test_update_genres(client, fragmentarium, user, database, parameters): fragment = FragmentFactory.build(genres=parameters["currentGenres"]) - fragment_number = fragmentarium.create_many(fragment) + fragment_number = fragmentarium.create(fragment) updates = {"genres": GenreSchema().dump(parameters["newGenres"], many=True)} post_result = client.simulate_post( f"/fragments/{fragment_number}/genres", body=json.dumps(updates) @@ -59,7 +59,7 @@ def test_update_genres(client, fragmentarium, user, database, parameters): def test_update_invalid_genres(client, fragmentarium, user, database): fragment = FragmentFactory.build(genres=tuple()) - fragment_number = fragmentarium.create_many(fragment) + fragment_number = fragmentarium.create(fragment) updates = {"genres": [{"category": ["asd", "wtz"], "uncertain": False}]} post_result = client.simulate_post( diff --git a/ebl/tests/fragmentarium/test_fragment_matcher_route.py b/ebl/tests/fragmentarium/test_fragment_matcher_route.py index fae154601..3391c697e 100644 --- a/ebl/tests/fragmentarium/test_fragment_matcher_route.py +++ b/ebl/tests/fragmentarium/test_fragment_matcher_route.py @@ -19,8 +19,8 @@ def test_fragment_matcher_route(client, fragmentarium, user): {"museumNumber": "X.326", "script": fragment_2.script, "score": 5} ], } - fragmentarium.create_many(fragment_1) - fragmentarium.create_many(fragment_2) + fragmentarium.create(fragment_1) + fragmentarium.create(fragment_2) get_result = client.simulate_get(f"/fragments/{fragment_id}/match") assert get_result.status == falcon.HTTP_OK assert get_result.json == expected_score @@ -33,7 +33,7 @@ def test_fragment_matcher_route_error(client, fragmentarium, user): number=MuseumNumber.of("X.1"), line_to_vec=(LineToVecEncoding.from_list([1, 1, 2]),), ) - fragmentarium.create_many(fragment_1) - fragmentarium.create_many(fragment_2) + fragmentarium.create(fragment_1) + fragmentarium.create(fragment_2) get_result = client.simulate_get(f"/fragments/{faulty_fragment_id}/match") assert get_result.status == falcon.HTTP_UNPROCESSABLE_ENTITY diff --git a/ebl/tests/fragmentarium/test_fragment_repository.py b/ebl/tests/fragmentarium/test_fragment_repository.py index 71b82e6a4..d2eee9bda 100644 --- a/ebl/tests/fragmentarium/test_fragment_repository.py +++ b/ebl/tests/fragmentarium/test_fragment_repository.py @@ -70,7 +70,7 @@ def test_create(database, fragment_repository): fragment = LemmatizedFragmentFactory.build() - fragment_id = fragment_repository.create_many(fragment) + fragment_id = fragment_repository.create(fragment) assert fragment_id == str(fragment.number) assert database[COLLECTION].find_one( @@ -136,7 +136,7 @@ def test_query_by_museum_number_references( reference = ReferenceFactory.build(with_document=True) fragment = LemmatizedFragmentFactory.build(references=(reference,)) database[COLLECTION].insert_one(FragmentSchema(exclude=["joins"]).dump(fragment)) - bibliography_repository.create_many(reference.document) + bibliography_repository.create(reference.document) assert fragment_repository.query_by_museum_number(fragment.number) == fragment @@ -149,7 +149,7 @@ def test_find_random(fragment_repository): fragment = FragmentFactory.build() transliterated_fragment = TransliteratedFragmentFactory.build() for a_fragment in fragment, transliterated_fragment: - fragment_repository.create_many(a_fragment) + fragment_repository.create(a_fragment) assert fragment_repository.query_random_by_transliterated() == [ transliterated_fragment @@ -199,7 +199,7 @@ def test_folio_pager_exception(fragment_repository): ) def test_query_next_and_previous_fragment(museum_numbers, fragment_repository): for fragmentNumber in museum_numbers: - fragment_repository.create_many( + fragment_repository.create( FragmentFactory.build(number=MuseumNumber.of(fragmentNumber)) ) for museum_number in museum_numbers: @@ -214,7 +214,7 @@ def test_query_next_and_previous_fragment(museum_numbers, fragment_repository): def test_update_transliteration_with_record(fragment_repository, user): fragment = FragmentFactory.build() - fragment_repository.create_many(fragment) + fragment_repository.create(fragment) updated_fragment = fragment.update_transliteration( TransliterationUpdate(parse_atf_lark("$ (the transliteration)"), "notes"), user ) @@ -233,7 +233,7 @@ def test_update_update_transliteration_not_found(fragment_repository): def test_update_genres(fragment_repository): fragment = FragmentFactory.build(genres=tuple()) - fragment_repository.create_many(fragment) + fragment_repository.create(fragment) updated_fragment = fragment.set_genres( (Genre(["ARCHIVAL", "Administrative"], False),) ) @@ -245,7 +245,7 @@ def test_update_genres(fragment_repository): def test_update_lemmatization(fragment_repository): transliterated_fragment = TransliteratedFragmentFactory.build() - fragment_repository.create_many(transliterated_fragment) + fragment_repository.create(transliterated_fragment) tokens = [list(line) for line in transliterated_fragment.text.lemmatization.tokens] tokens[1][3] = LemmatizationToken(tokens[1][3].value, ("aklu I",)) lemmatization = Lemmatization(tokens) @@ -405,8 +405,8 @@ def test_empty_result_search_reference_id_and_pages( @pytest.mark.parametrize("signs,is_match", SEARCH_SIGNS_DATA) def test_search_signs(signs, is_match, fragment_repository): transliterated_fragment = TransliteratedFragmentFactory.build() - fragment_repository.create_many(transliterated_fragment) - fragment_repository.create_many(FragmentFactory.build()) + fragment_repository.create(transliterated_fragment) + fragment_repository.create(FragmentFactory.build()) result = fragment_repository.query_by_transliteration(TransliterationQuery(signs)) expected = [transliterated_fragment] if is_match else [] @@ -457,7 +457,7 @@ def test_find_transliterated_line_to_vec(database, fragment_repository): def test_update_references(fragment_repository): reference = ReferenceFactory.build() fragment = FragmentFactory.build() - fragment_repository.create_many(fragment) + fragment_repository.create(fragment) references = (reference,) updated_fragment = fragment.set_references(references) diff --git a/ebl/tests/fragmentarium/test_fragment_updater.py b/ebl/tests/fragmentarium/test_fragment_updater.py index cfe6f5a46..3961b835c 100644 --- a/ebl/tests/fragmentarium/test_fragment_updater.py +++ b/ebl/tests/fragmentarium/test_fragment_updater.py @@ -50,7 +50,7 @@ def test_update_transliteration( .query_by_museum_number(number) .thenReturn(transliterated_fragment) ) - when(changelog).create_many( + when(changelog).create( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(transliterated_fragment)}, @@ -109,7 +109,7 @@ def test_update_genres(fragment_updater, user, fragment_repository, changelog, w expected_fragment = fragment.set_genres(genres) (when(fragment_repository).query_by_museum_number(number).thenReturn(fragment)) - when(changelog).create_many( + when(changelog).create( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(fragment)}, @@ -136,7 +136,7 @@ def test_update_lemmatization( .query_by_museum_number(number) .thenReturn(transliterated_fragment) ) - when(changelog).create_many( + when(changelog).create( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(transliterated_fragment)}, @@ -179,7 +179,7 @@ def test_update_references( .thenReturn(expected_fragment) ) when(fragment_repository).update_references(expected_fragment).thenReturn() - when(changelog).create_many( + when(changelog).create( "fragments", user.profile, {"_id": str(number), **SCHEMA.dump(fragment)}, diff --git a/ebl/tests/fragmentarium/test_fragments_route.py b/ebl/tests/fragmentarium/test_fragments_route.py index f07bba409..325e6cefe 100644 --- a/ebl/tests/fragmentarium/test_fragments_route.py +++ b/ebl/tests/fragmentarium/test_fragments_route.py @@ -7,7 +7,7 @@ def test_get(client, fragmentarium, user): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) result = client.simulate_get(f"/fragments/{transliterated_fragment.number}") assert result.json == create_response_dto( diff --git a/ebl/tests/fragmentarium/test_fragments_search_route.py b/ebl/tests/fragmentarium/test_fragments_search_route.py index b9ca51da0..016ce215f 100644 --- a/ebl/tests/fragmentarium/test_fragments_search_route.py +++ b/ebl/tests/fragmentarium/test_fragments_search_route.py @@ -26,7 +26,7 @@ def expected_fragment_info_dto(fragment, lines=tuple()): ) def test_search_fragment(get_number, client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) result = client.simulate_get("/fragments", params={"number": get_number(fragment)}) assert result.status == falcon.HTTP_OK @@ -43,8 +43,8 @@ def test_search_fragment_not_found(client): def test_search_references(client, fragmentarium, bibliography, user): bib_entry_1 = BibliographyEntryFactory.build(id="RN.0", pages="254") bib_entry_2 = BibliographyEntryFactory.build(id="RN.1") - bibliography.create_many(bib_entry_1, user) - bibliography.create_many(bib_entry_2, user) + bibliography.create(bib_entry_1, user) + bibliography.create(bib_entry_2, user) fragment = FragmentFactory.build( references=( @@ -52,7 +52,7 @@ def test_search_references(client, fragmentarium, bibliography, user): ReferenceFactory.build(id="RN.1"), ) ) - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) result = client.simulate_get( "/fragments", params={"id": fragment.references[0].id, "pages": fragment.references[0].pages}, @@ -76,7 +76,7 @@ def test_search_references_invalid_query(client, fragmentarium): fragment = FragmentFactory.build( references=(ReferenceFactory.build(), ReferenceFactory.build()) ) - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) reference_id = fragment.references[0].id reference_pages = "should be a number" result = client.simulate_get( @@ -87,9 +87,9 @@ def test_search_references_invalid_query(client, fragmentarium): def test_search_signs(client, fragmentarium, sign_repository, signs): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) result = client.simulate_get("/fragments", params={"transliteration": "ma-tu₂"}) @@ -111,7 +111,7 @@ def test_search_signs_invalid(client, fragmentarium, sign_repository, signs): def test_random(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) result = client.simulate_get("/fragments", params={"random": True}) @@ -124,7 +124,7 @@ def test_interesting(client, fragmentarium): interesting_fragment = InterestingFragmentFactory.build( number=MuseumNumber("K", "1") ) - fragmentarium.create_many(interesting_fragment) + fragmentarium.create(interesting_fragment) result = client.simulate_get("/fragments", params={"interesting": True}) @@ -135,7 +135,7 @@ def test_interesting(client, fragmentarium): def test_latest(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) result = client.simulate_get("/fragments", params={"latest": True}) @@ -146,7 +146,7 @@ def test_latest(client, fragmentarium): def test_needs_revision(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) result = client.simulate_get("/fragments", params={"needsRevision": True}) diff --git a/ebl/tests/fragmentarium/test_lemmatization_route.py b/ebl/tests/fragmentarium/test_lemmatization_route.py index d59af599c..2cb04f8c1 100644 --- a/ebl/tests/fragmentarium/test_lemmatization_route.py +++ b/ebl/tests/fragmentarium/test_lemmatization_route.py @@ -28,7 +28,7 @@ def test_deserialize_lemmatization(): def test_update_lemmatization(client, fragmentarium, user, database): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) tokens = [list(line) for line in transliterated_fragment.text.lemmatization.tokens] tokens[1][3] = LemmatizationToken(tokens[1][3].value, ("aklu I",)) lemmatization = Lemmatization(tokens) @@ -89,7 +89,7 @@ def test_update_lemmatization_invalid_number(client): ) def test_update_lemmatization_invalid_entity(client, fragmentarium, body): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) url = f"/fragments/{fragment.number}/lemmatization" post_result = client.simulate_post(url, body=body) @@ -99,7 +99,7 @@ def test_update_lemmatization_invalid_entity(client, fragmentarium, body): def test_update_lemmatization_atf_change(client, fragmentarium): transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) dto = LemmatizationSchema().dump(transliterated_fragment.text.lemmatization) dto["lemmatization"][0][0]["value"] = "ana" url = f"/fragments/{transliterated_fragment.number}/lemmatization" diff --git a/ebl/tests/fragmentarium/test_pager_route.py b/ebl/tests/fragmentarium/test_pager_route.py index d755c03a2..769316f4e 100644 --- a/ebl/tests/fragmentarium/test_pager_route.py +++ b/ebl/tests/fragmentarium/test_pager_route.py @@ -9,7 +9,7 @@ def test_get_fragment_pager(client, fragmentarium): fragment_1 = FragmentFactory.build(number=MuseumNumber("X", "1")) fragment_2 = FragmentFactory.build(number=MuseumNumber("X", "2")) for fragment in [fragment_0, fragment_1, fragment_2]: - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) result = client.simulate_get(f"/fragments/{fragment_1.number}/pager") assert result.json == {"next": "X.2", "previous": "X.0"} @@ -21,10 +21,10 @@ def test_get_fragment_pager_cache(cached_client, fragmentarium): fragment_0 = FragmentFactory.build(number=MuseumNumber("X", "0")) fragment_1 = FragmentFactory.build(number=MuseumNumber("X", "3")) for fragment in [fragment_0, fragment_1]: - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) first_result = cached_client.simulate_get(f"/fragments/{fragment_1.number}/pager") - fragmentarium.create_many(FragmentFactory.build(number=MuseumNumber("X", "2"))) + fragmentarium.create(FragmentFactory.build(number=MuseumNumber("X", "2"))) second_result = cached_client.simulate_get(f"/fragments/{fragment_1.number}/pager") assert first_result.json == second_result.json @@ -39,7 +39,7 @@ def test_get_fragment_pager_invalid_id(client): def test_get_folio_pager(client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) result = client.simulate_get(f"/fragments/{fragment.number}/pager/WGL/1") assert result.json == { @@ -57,7 +57,7 @@ def test_get_folio_pager_invalid_id(client): def test_get_folio_pager_no_access_to_folio(client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) result = client.simulate_get(f"/fragments/{fragment.number}/pager/XXX/1") assert result.status == falcon.HTTP_FORBIDDEN diff --git a/ebl/tests/fragmentarium/test_references_route.py b/ebl/tests/fragmentarium/test_references_route.py index 3b4bc7e2e..4fe78cb17 100644 --- a/ebl/tests/fragmentarium/test_references_route.py +++ b/ebl/tests/fragmentarium/test_references_route.py @@ -15,9 +15,9 @@ def test_update_references(client, fragmentarium, bibliography, user): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) reference = ReferenceFactory.build(with_document=True) - bibliography.create_many(reference.document, ANY_USER) + bibliography.create(reference.document, ANY_USER) references = [ReferenceSchema().dump(reference)] body = json.dumps({"references": references}) url = f"/fragments/{fragment.number}/references" @@ -81,7 +81,7 @@ def test_update_references_invalid_museum_number(client): ) def test_update_references_invalid_reference(client, fragmentarium, body): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) url = f"/fragments/{fragment.number}/references" post_result = client.simulate_post(url, body=body) @@ -92,7 +92,7 @@ def test_update_references_invalid_reference(client, fragmentarium, body): def test_update_references_invalid_id(client, fragmentarium): reference = ReferenceFactory.build(with_document=True) fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) body = json.dumps({"references": [ReferenceSchema().dump(reference)]}) url = f"/fragments/{fragment.number}/references" post_result = client.simulate_post(url, body=body) diff --git a/ebl/tests/fragmentarium/test_statistics_route.py b/ebl/tests/fragmentarium/test_statistics_route.py index 42bd85d88..d43c47c23 100644 --- a/ebl/tests/fragmentarium/test_statistics_route.py +++ b/ebl/tests/fragmentarium/test_statistics_route.py @@ -17,7 +17,7 @@ def test_get_statistics(guest_client): def test_get_statistics_cache(cached_client, fragmentarium): first_result = cached_client.simulate_get(PATH) transliterated_fragment = TransliteratedFragmentFactory.build() - fragmentarium.create_many(transliterated_fragment) + fragmentarium.create(transliterated_fragment) second_result = cached_client.simulate_get(PATH) assert second_result.json == first_result.json diff --git a/ebl/tests/fragmentarium/test_transliteration_query_factory.py b/ebl/tests/fragmentarium/test_transliteration_query_factory.py index c50bb3765..4010bdfd2 100644 --- a/ebl/tests/fragmentarium/test_transliteration_query_factory.py +++ b/ebl/tests/fragmentarium/test_transliteration_query_factory.py @@ -9,7 +9,7 @@ def test_create_query(sign_repository, signs): for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) factory = TransliterationQueryFactory(sign_repository) atf = "šu\ngid₂" diff --git a/ebl/tests/fragmentarium/test_transliteration_update_factory.py b/ebl/tests/fragmentarium/test_transliteration_update_factory.py index 4427291db..fa0a35577 100644 --- a/ebl/tests/fragmentarium/test_transliteration_update_factory.py +++ b/ebl/tests/fragmentarium/test_transliteration_update_factory.py @@ -11,7 +11,7 @@ def test_create(sign_repository, signs): for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) factory = TransliterationUpdateFactory(sign_repository) atf = Atf("1. šu gid₂") diff --git a/ebl/tests/fragmentarium/test_transliterations_route.py b/ebl/tests/fragmentarium/test_transliterations_route.py index 659e58e37..d4696a759 100644 --- a/ebl/tests/fragmentarium/test_transliterations_route.py +++ b/ebl/tests/fragmentarium/test_transliterations_route.py @@ -15,7 +15,7 @@ @freeze_time("2018-09-07 15:41:24.032") def test_update_transliteration(client, fragmentarium, user, database): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) updates = {"transliteration": "$ (the transliteration)", "notes": "some notes"} body = json.dumps(updates) url = f"/fragments/{fragment.number}/transliteration" @@ -65,13 +65,13 @@ def test_update_transliteration_merge_lemmatization( ): for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) lemmatized_fragment = LemmatizedFragmentFactory.build() - fragmentarium.create_many(lemmatized_fragment) + fragmentarium.create(lemmatized_fragment) lines = lemmatized_fragment.text.atf.split("\n") lines[1] = new_transliteration updates = {"transliteration": "\n".join(lines), "notes": lemmatized_fragment.notes} - updated_transliteration = transliteration_factory.create_many( + updated_transliteration = transliteration_factory.create( updates["transliteration"], updates["notes"] ) expected_json = create_response_dto( @@ -96,7 +96,7 @@ def test_update_transliteration_merge_lemmatization( def test_update_transliteration_invalid_atf(client, fragmentarium): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) updates = {"transliteration": "1. kururu", "notes": ""} body = json.dumps(updates) url = f"/fragments/{fragment.number}/transliteration" @@ -160,7 +160,7 @@ def test_update_transliteration_invalid(client): ) def test_update_transliteration_invalid_entity(client, fragmentarium, body): fragment = FragmentFactory.build() - fragmentarium.create_many(fragment) + fragmentarium.create(fragment) url = f"/fragments/{fragment.number}/transliteration" post_result = client.simulate_post(url, body=body) diff --git a/ebl/tests/lemmatization/test_lemma_search_route.py b/ebl/tests/lemmatization/test_lemma_search_route.py index ea1572e67..75091985e 100644 --- a/ebl/tests/lemmatization/test_lemma_search_route.py +++ b/ebl/tests/lemmatization/test_lemma_search_route.py @@ -13,8 +13,8 @@ def test_search_fragment( ): expected_word = {**word, "_id": lemma} lemmatized_fragment = LemmatizedFragmentFactory.build() - fragmentarium.create_many(lemmatized_fragment) - dictionary.create_many(expected_word) + fragmentarium.create(lemmatized_fragment) + dictionary.create(expected_word) result = client.simulate_get( "/lemmas", params={"word": query_word, "isNormalized": is_normalized} ) diff --git a/ebl/tests/lemmatization/test_suggestion_finder.py b/ebl/tests/lemmatization/test_suggestion_finder.py index e78767c2d..d4079c70c 100644 --- a/ebl/tests/lemmatization/test_suggestion_finder.py +++ b/ebl/tests/lemmatization/test_suggestion_finder.py @@ -58,24 +58,24 @@ def test_query_lemmas(fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create_many(lemmatized_fragment) - fragment_repository.create_many(ANOTHER_LEMMATIZED_FRAGMENT) + fragment_repository.create(lemmatized_fragment) + fragment_repository.create(ANOTHER_LEMMATIZED_FRAGMENT) assert lemma_repository.query_lemmas("GI₆", False) == [["ginâ I"]] def test_query_lemmas_normalized(fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create_many(lemmatized_fragment) - fragment_repository.create_many(ANOTHER_LEMMATIZED_FRAGMENT) + fragment_repository.create(lemmatized_fragment) + fragment_repository.create(ANOTHER_LEMMATIZED_FRAGMENT) assert lemma_repository.query_lemmas("ana", True) == [["normalized I"]] def test_query_lemmas_multiple(fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create_many(lemmatized_fragment) - fragment_repository.create_many(ANOTHER_LEMMATIZED_FRAGMENT) + fragment_repository.create(lemmatized_fragment) + fragment_repository.create(ANOTHER_LEMMATIZED_FRAGMENT) assert lemma_repository.query_lemmas("ana", False) == [["ana II"], ["ana I"]] @@ -154,7 +154,7 @@ def test_query_lemmas_ignores_in_value( ), signs="DIŠ", ) - fragment_repository.create_many(fragment) + fragment_repository.create(fragment) assert lemma_repository.query_lemmas("ana", False) == expected @@ -162,7 +162,7 @@ def test_query_lemmas_ignores_in_value( @pytest.mark.parametrize("is_normalized", [False, True]) def test_query_lemmas_not_found(is_normalized, fragment_repository, lemma_repository): lemmatized_fragment = LemmatizedFragmentFactory.build() - fragment_repository.create_many(lemmatized_fragment) + fragment_repository.create(lemmatized_fragment) assert lemma_repository.query_lemmas("aklu", is_normalized) == [] diff --git a/ebl/tests/signs/test_sign_images_route.py b/ebl/tests/signs/test_sign_images_route.py index 5d16a9804..5f67f721d 100644 --- a/ebl/tests/signs/test_sign_images_route.py +++ b/ebl/tests/signs/test_sign_images_route.py @@ -23,14 +23,14 @@ def test_signs_get( fragment = TransliteratedFragmentFactory.build( number=MuseumNumber.of("K.2"), text=text_with_labels ) - fragment_repository.create_many(fragment) + fragment_repository.create(fragment) annotation_data = AnnotationDataFactory.build(sign_name="signName", path=[2, 0, 0]) cropped_sign = CroppedSignFactory.build(script=fragment.script) annotation = AnnotationFactory.build( data=annotation_data, cropped_sign=cropped_sign ) - cropped_sign_images_repository.create_many( + cropped_sign_images_repository.create( [ CroppedSignImage( annotation.cropped_sign.image_id, Base64("test-base64-string") diff --git a/ebl/tests/signs/test_sign_repository.py b/ebl/tests/signs/test_sign_repository.py index 9f956a329..990c9c5f7 100644 --- a/ebl/tests/signs/test_sign_repository.py +++ b/ebl/tests/signs/test_sign_repository.py @@ -153,7 +153,7 @@ def sign_si_2(mongo_sign_si_2): def test_create(database, sign_repository, sign_igi): - sign_name = sign_repository.create_many(sign_igi) + sign_name = sign_repository.create(sign_igi) assert database[COLLECTION].find_one({"_id": sign_name}) == SignSchema().dump( sign_igi diff --git a/ebl/tests/signs/test_sign_search_route.py b/ebl/tests/signs/test_sign_search_route.py index 95a44fe18..bbc962122 100644 --- a/ebl/tests/signs/test_sign_search_route.py +++ b/ebl/tests/signs/test_sign_search_route.py @@ -7,7 +7,7 @@ def test_signs_get(client, sign_repository): sign = Sign(SignName("test")) - sign_repository.create_many(sign) + sign_repository.create(sign) result = client.simulate_get(f"/signs/{sign.name}") assert result.json == SignDtoSchema().dump(sign) @@ -87,7 +87,7 @@ def test_signs_not_found(client): ) def test_signs_search_route(params, expected, client, sign_repository, signs): for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) get_result = client.simulate_get("/signs", params=params) assert get_result.status == falcon.HTTP_OK assert get_result.json == expected @@ -95,6 +95,6 @@ def test_signs_search_route(params, expected, client, sign_repository, signs): def test_signs_search_route_error(client, sign_repository, signs): for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) get_result = client.simulate_get("/signs", params={"signs": "P₂"}) assert get_result.status == falcon.HTTP_UNPROCESSABLE_ENTITY diff --git a/ebl/tests/test_changelog.py b/ebl/tests/test_changelog.py index b256442d0..1e5cbeb61 100644 --- a/ebl/tests/test_changelog.py +++ b/ebl/tests/test_changelog.py @@ -10,6 +10,6 @@ @freeze_time("2018-09-07 15:41:24.032") def test_create(database, changelog, user, make_changelog_entry): - entry_id = changelog.create_many(RESOURCE_TYPE, user.profile, OLD, NEW) + entry_id = changelog.create(RESOURCE_TYPE, user.profile, OLD, NEW) expected = make_changelog_entry(RESOURCE_TYPE, RESOURCE_ID, OLD, NEW) assert database[COLLECTION].find_one({"_id": entry_id}, {"_id": 0}) == expected diff --git a/ebl/tests/transliteration/test_signs_visitor.py b/ebl/tests/transliteration/test_signs_visitor.py index 36ae225dd..d81b396bb 100644 --- a/ebl/tests/transliteration/test_signs_visitor.py +++ b/ebl/tests/transliteration/test_signs_visitor.py @@ -58,7 +58,7 @@ ) def test_signs_visitor(text: str, expected: Sequence[str], sign_repository, signs): for sign in signs: - sign_repository.create_many(sign) + sign_repository.create(sign) visitor = SignsVisitor(sign_repository) parse_line(f"1. {text}").accept(visitor) From 7efafb1a2524b7f2424e4c34170f7440400a3b73 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 18:46:45 +0100 Subject: [PATCH 17/23] revert changes to chapter --- .github/workflows/main.yml | 4 +- README.md | 10 ++--- ebl/corpus/application/display_schemas.py | 13 +------ ebl/corpus/domain/chapter_display.py | 17 +++----- ebl/corpus/infrastructure/queries.py | 41 ++++++++++---------- ebl/tests/corpus/test_chapter_display.py | 7 ++-- ebl/tests/corpus/test_chapter_display_schema | 2 - 7 files changed, 35 insertions(+), 59 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aa8603247..a5665b6df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,7 +54,7 @@ jobs: if: ${{ startsWith(matrix.python-version, 'pypy') }} env: CI: true - run: poetry run pytest + run: poetry run pytest -n auto - name: Unit Tests with Coverage uses: paambaati/codeclimate-action@v3.0.0 @@ -62,7 +62,7 @@ jobs: env: CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} with: - coverageCommand: poetry run pytest --cov=ebl --cov-report term --cov-report xml + coverageCommand: poetry run pytest --cov=ebl --cov-report term --cov-report xml -n auto - uses: edge/simple-slack-notify@v1.1.1 if: failure() diff --git a/README.md b/README.md index f2110dcc9..4e8d22151 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,7 @@ Requirements: pip install poetry poetry install ``` - -If `libcst` installation fails (because a binary wheel is not available for Linux/Windows + Pypy) you may need to install +If `libcst` installation fails (because a binary wheel is not available for Linux/Windows + Pypy) you may need to install the [rust compiler](https://www.rust-lang.org/tools/install) to solve it. The following are needed to run application: @@ -87,7 +86,7 @@ poetry run flake8 poetry run pyre check poetry run pytest poetry run pytest -n auto # Run tests in parallel. -poetry run --cov=ebl --cov-report term --cov-report xml # Run tests with coverage (slow in PyPy). +poetry run --cov=ebl --cov-report term --cov-report xml -n auto # Run tests in parallel with coverage (slow in PyPy). ``` See [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) documentation @@ -245,9 +244,8 @@ SENTRY_ENVIRONMENT= CACHE_CONFIG= ``` -Poetry does not support .env-files. The environment variables need to be configured in the shell. -Alternatively and external program can be used to handle the file e.g. [direnv](https://direnv.net/) -or [Set-PsEnv](https://github.com/rajivharris/Set-PsEnv). +Poetry does not support .env-files. The environment variables need to be configured in the shell. +Alternatively and external program can be used to handle the file e.g. [direnv](https://direnv.net/) or [Set-PsEnv](https://github.com/rajivharris/Set-PsEnv). ### Locally diff --git a/ebl/corpus/application/display_schemas.py b/ebl/corpus/application/display_schemas.py index 95954252f..30543e77a 100644 --- a/ebl/corpus/application/display_schemas.py +++ b/ebl/corpus/application/display_schemas.py @@ -5,14 +5,9 @@ from ebl.corpus.domain.chapter_display import ChapterDisplay, LineDisplay from ebl.corpus.domain.record import Record from ebl.transliteration.application.line_number_schemas import OneOfLineNumberSchema -from ebl.transliteration.application.line_schemas import ( - NoteLineSchema, - TranslationLineSchema, -) from ebl.transliteration.application.note_line_part_schemas import ( OneOfNoteLinePartSchema, ) -from ebl.transliteration.application.one_of_line_schema import ParallelLineSchema from ebl.transliteration.application.token_schemas import OneOfTokenSchema @@ -29,11 +24,7 @@ class LineDisplaySchema(Schema): ) reconstruction = fields.List(fields.Nested(OneOfTokenSchema), load_default=tuple()) translation = fields.List( - fields.Nested(TranslationLineSchema), load_default=tuple(), allow_none=True - ) - note = fields.Nested(NoteLineSchema, allow_none=True, load_default=None) - parallel_lines = fields.Nested( - ParallelLineSchema, data_key="parallelLines", many=True, load_default=tuple() + fields.Nested(OneOfNoteLinePartSchema), load_default=tuple(), allow_none=True ) @post_load @@ -45,8 +36,6 @@ def make_line(self, data: dict, **kwargs) -> LineDisplay: tuple(data["intertext"] or []), tuple(data["reconstruction"]), tuple(data["translation"] or []), - data["note"], - tuple(data["parallel_lines"]), ) diff --git a/ebl/corpus/domain/chapter_display.py b/ebl/corpus/domain/chapter_display.py index 901d3eb5f..b68e70af1 100644 --- a/ebl/corpus/domain/chapter_display.py +++ b/ebl/corpus/domain/chapter_display.py @@ -1,13 +1,10 @@ -from typing import Optional, Sequence - +from typing import Iterable, Sequence import attr from ebl.corpus.domain.chapter import ChapterId, Chapter from ebl.corpus.domain.line import Line from ebl.corpus.domain.record import Record from ebl.corpus.domain.text import Text -from ebl.transliteration.domain.note_line import NoteLine -from ebl.transliteration.domain.parallel_line import ParallelLine from ebl.transliteration.domain.translation_line import ( DEFAULT_LANGUAGE, TranslationLine, @@ -18,7 +15,7 @@ def get_default_translation( - translations: Sequence[TranslationLine], + translations: Iterable[TranslationLine], ) -> Sequence[MarkupPart]: return next( ( @@ -37,13 +34,11 @@ class LineDisplay: is_beginning_of_section: bool intertext: Sequence[MarkupPart] reconstruction: Sequence[Token] - translation: Sequence[TranslationLine] - note: Optional[NoteLine] - parallel_lines: Sequence[ParallelLine] + translation: Sequence[MarkupPart] @property def title(self) -> Sequence[MarkupPart]: - return to_title(get_default_translation(self.translation)) + return to_title(self.translation) @staticmethod def of_line(line: Line) -> "LineDisplay": @@ -54,9 +49,7 @@ def of_line(line: Line) -> "LineDisplay": line.is_beginning_of_section, first_variant.intertext, first_variant.reconstruction, - line.translation, - first_variant.note, - first_variant.parallel_lines, + get_default_translation(line.translation), ) diff --git a/ebl/corpus/infrastructure/queries.py b/ebl/corpus/infrastructure/queries.py index ca40f18ec..3e858b1e1 100644 --- a/ebl/corpus/infrastructure/queries.py +++ b/ebl/corpus/infrastructure/queries.py @@ -3,6 +3,7 @@ from ebl.corpus.domain.chapter import ChapterId from ebl.corpus.infrastructure.collections import CHAPTERS_COLLECTION from ebl.fragmentarium.infrastructure.queries import is_in_fragmentarium +from ebl.transliteration.domain.translation_line import DEFAULT_LANGUAGE def chapter_id_query(id_: ChapterId) -> dict: @@ -123,50 +124,48 @@ def aggregate_chapter_display(id_: ChapterId) -> List[dict]: "number": "$$line.number", "isSecondLineOfParallelism": "$$line.isSecondLineOfParallelism", "isBeginningOfSection": "$$line.isBeginningOfSection", - "translation": "$$line.translation", - "intertext": { - "$arrayElemAt": [ - { - "$map": { - "input": "$$line.variants", - "as": "variant", - "in": "$$variant.intertext", - } - }, - 0, - ] - }, - "reconstruction": { + "translation": { "$arrayElemAt": [ { "$map": { - "input": "$$line.variants", - "as": "variant", - "in": "$$variant.reconstruction", + "input": { + "$filter": { + "input": "$$line.translation", + "as": "translation", + "cond": { + "eq": [ + "$$translation.language", + DEFAULT_LANGUAGE, + ] + }, + } + }, + "as": "en_translation", + "in": "$$en_translation.parts", } }, 0, ] }, - "note": { + "intertext": { "$arrayElemAt": [ { "$map": { "input": "$$line.variants", "as": "variant", - "in": "$$variant.note", + "in": "$$variant.intertext", } }, 0, ] }, - "parallelLines": { + "reconstruction": { "$arrayElemAt": [ { "$map": { "input": "$$line.variants", "as": "variant", - "in": "$$variant.parallelLines", + "in": "$$variant.reconstruction", } }, 0, diff --git a/ebl/tests/corpus/test_chapter_display.py b/ebl/tests/corpus/test_chapter_display.py index 1a5c716ac..e6ad09b77 100644 --- a/ebl/tests/corpus/test_chapter_display.py +++ b/ebl/tests/corpus/test_chapter_display.py @@ -9,9 +9,10 @@ def test_line_display_of_line() -> None: + expected_translation = (StringPart("foo"),) translation_lines = ( TranslationLine((StringPart("bar"),), "de", None), - TranslationLine((StringPart("foo"),), DEFAULT_LANGUAGE, None), + TranslationLine(expected_translation, DEFAULT_LANGUAGE, None), ) line = LineFactory.build(translation=translation_lines) @@ -23,9 +24,7 @@ def test_line_display_of_line() -> None: line.is_beginning_of_section, line.variants[0].intertext, line.variants[0].reconstruction, - translation_lines, - line.variants[0].note, - line.variants[0].parallel_lines, + expected_translation, ) assert line_display.title == make_title(translation_lines) diff --git a/ebl/tests/corpus/test_chapter_display_schema b/ebl/tests/corpus/test_chapter_display_schema index 609f97704..b1631e757 100644 --- a/ebl/tests/corpus/test_chapter_display_schema +++ b/ebl/tests/corpus/test_chapter_display_schema @@ -7,7 +7,6 @@ from ebl.tests.factories.corpus import ( TextFactory, ) from ebl.transliteration.application.line_number_schemas import OneOfLineNumberSchema -from ebl.transliteration.application.line_schemas import NoteLineSchema from ebl.transliteration.application.note_line_part_schemas import ( OneOfNoteLinePartSchema, ) @@ -44,7 +43,6 @@ def to_dict(chapter: ChapterDisplay, missing_intertext: bool = False, missing_tr "translation": None if missing_translation else OneOfNoteLinePartSchema().dump( line.translation, many=True ), - "note": line.note and NoteLineSchema().dump(line.none) } for line in chapter.lines ], From fe1ffa10dfda6c0b8d14374596e383ed6d8796ed Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 19:25:50 +0100 Subject: [PATCH 18/23] fix tests --- .../application/annotations_schema.py | 4 ++- .../application/annotations_service.py | 13 +++++----- ebl/fragmentarium/web/annotations.py | 2 +- .../fragmentarium/test_annotations_schema.py | 25 +------------------ .../fragmentarium/test_annotations_service.py | 11 +++----- ebl/tests/signs/test_sign_images_route.py | 2 +- 6 files changed, 15 insertions(+), 42 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_schema.py b/ebl/fragmentarium/application/annotations_schema.py index 154ebbc5e..0ecb3053b 100644 --- a/ebl/fragmentarium/application/annotations_schema.py +++ b/ebl/fragmentarium/application/annotations_schema.py @@ -38,7 +38,9 @@ def make_data(self, data, **kwargs): class AnnotationSchema(Schema): geometry = fields.Nested(GeometrySchema(), required=True) data = fields.Nested(AnnotationDataSchema(), required=True) - cropped_sign = fields.Nested(CroppedSignSchema(), data_key="croppedSign") + cropped_sign = fields.Nested( + CroppedSignSchema(), load_default=None, data_key="croppedSign" + ) @post_load def make_annotation(self, data, **kwargs): diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index d068ff622..f5eb335c8 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -157,17 +157,16 @@ def update(self, annotations: Annotations, user: User) -> Annotations: ) _id = str(annotations.fragment_number) schema = AnnotationsSchema() - self._changelog.create( - "annotations", - user.profile, - {"_id": _id, **schema.dump(old_annotations)}, - {"_id": _id, **schema.dump(annotations)}, - ) - ( annotations_with_image_ids, cropped_sign_images, ) = self._cropped_image_from_annotations(annotations) self._annotations_repository.create_or_update(annotations_with_image_ids) self._cropped_sign_images_repository.create_many(cropped_sign_images) + self._changelog.create( + "annotations", + user.profile, + {"_id": _id, **schema.dump(old_annotations)}, + {"_id": _id, **schema.dump(annotations_with_image_ids)}, + ) return annotations_with_image_ids diff --git a/ebl/fragmentarium/web/annotations.py b/ebl/fragmentarium/web/annotations.py index 7bc5d7d7b..03f4540fb 100644 --- a/ebl/fragmentarium/web/annotations.py +++ b/ebl/fragmentarium/web/annotations.py @@ -12,7 +12,7 @@ def __init__(self, annotation_service: AnnotationsService): self._annotation_service = annotation_service @falcon.before(require_scope, "annotate:fragments") - @validate(AnnotationsSchema(), AnnotationsSchema()) + @validate(AnnotationsSchema()) def on_post(self, req: falcon.Request, resp: falcon.Response, number: str): if number == req.media.get("fragmentNumber"): annotations = self._annotation_service.update( diff --git a/ebl/tests/fragmentarium/test_annotations_schema.py b/ebl/tests/fragmentarium/test_annotations_schema.py index 8f5d56b8a..371e1b726 100644 --- a/ebl/tests/fragmentarium/test_annotations_schema.py +++ b/ebl/tests/fragmentarium/test_annotations_schema.py @@ -27,15 +27,10 @@ AnnotationData(ID, VALUE, TYPE, PATH, SIGN_NAME), CroppedSign(IMAGE_ID, SCRIPT, LABEL), ) -ANNOTATION_NO_CROPPED_SIGN = Annotation( - Geometry(X, Y, WIDTH, HEIGHT), - AnnotationData(ID, VALUE, TYPE, PATH, SIGN_NAME), - None, -) MUSEUM_NUMBER = MuseumNumber("K", "1") ANNOTATIONS = Annotations(MUSEUM_NUMBER, [ANNOTATION]) -ANNOTATIONS_NO_CROPPED_SIGN = Annotations(MUSEUM_NUMBER, [ANNOTATION_NO_CROPPED_SIGN]) + SERIALIZED = { "fragmentNumber": str(MUSEUM_NUMBER), @@ -54,28 +49,10 @@ ], } -SERIALIZED_NO_SIGN = { - "fragmentNumber": str(MUSEUM_NUMBER), - "annotations": [ - { - "geometry": {"x": X, "y": Y, "width": WIDTH, "height": HEIGHT}, - "data": { - "id": ID, - "value": VALUE, - "type": "HasSign", - "signName": SIGN_NAME, - "path": PATH, - }, - } - ], -} - def test_load(): assert AnnotationsSchema().load(SERIALIZED) == ANNOTATIONS - assert AnnotationsSchema().load(SERIALIZED_NO_SIGN) == ANNOTATIONS_NO_CROPPED_SIGN def test_dump(): assert AnnotationsSchema().dump(ANNOTATIONS) == SERIALIZED - assert AnnotationsSchema().dump(ANNOTATIONS_NO_CROPPED_SIGN) == SERIALIZED_NO_SIGN diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index af9de2fe8..02dbf34ea 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -176,13 +176,8 @@ def test_update( when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( annotations ) - when(annotations_repository).create_or_update(updated_annotations).thenReturn() - when(changelog).create( - "annotations", - user.profile, - {"_id": str(fragment_number), **SCHEMA.dump(annotations)}, - {"_id": str(fragment_number), **SCHEMA.dump(updated_annotations)}, - ).thenReturn() + when(annotations_repository).create_or_update(...).thenReturn() + when(changelog).create(...).thenReturn() ( when(fragment_repository) .query_by_museum_number(annotations.fragment_number) @@ -200,4 +195,4 @@ def test_update( ): assert result_annotation.geometry == annotation.geometry assert result_annotation.data == annotation.data - assert result_annotation.cropped_sign is not None + assert result_annotation.cropped_sign != annotation.cropped_sign diff --git a/ebl/tests/signs/test_sign_images_route.py b/ebl/tests/signs/test_sign_images_route.py index 5f67f721d..986d4e9bc 100644 --- a/ebl/tests/signs/test_sign_images_route.py +++ b/ebl/tests/signs/test_sign_images_route.py @@ -30,7 +30,7 @@ def test_signs_get( annotation = AnnotationFactory.build( data=annotation_data, cropped_sign=cropped_sign ) - cropped_sign_images_repository.create( + cropped_sign_images_repository.create_many( [ CroppedSignImage( annotation.cropped_sign.image_id, Base64("test-base64-string") From 9a13f51417e85a6afa9d366f300821562dfb369c Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 20:17:02 +0100 Subject: [PATCH 19/23] fix pyre --- ebl/fragmentarium/application/annotations_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index f5eb335c8..4bafd8b27 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -6,7 +6,7 @@ import attr import bson.objectid from PIL import Image -from singledispatchmethod import singledispatchmethod +from functools import singledispatchmethod from ebl.changelog import Changelog from ebl.ebl_ai_client import EblAiClient From a3980f9bdccb6c0e1a7383815055b4b1be6ff4c3 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Mon, 21 Mar 2022 20:41:10 +0100 Subject: [PATCH 20/23] fix pyre --- ebl/fragmentarium/application/annotations_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 4bafd8b27..733bef00b 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -97,7 +97,7 @@ def _is_matching_number(self, line_number: AbstractLineNumber, number: int) -> b def _(self, line_number: LineNumber, number: int): return number == line_number.number - @_is_matching_number.register(LineNumberRange) # pyre-ignore[56] + @_is_matching_number.register(LineNumberRange) def _(self, line_number: LineNumberRange, number: int): return line_number.start.number <= number <= line_number.end.number From db20ef112477bbe0bfb24163c5a43738f538943d Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Tue, 29 Mar 2022 15:11:57 +0200 Subject: [PATCH 21/23] pr changes --- .../application/annotations_service.py | 141 ++++++------------ .../application/cropped_sign_image.py | 8 +- ebl/fragmentarium/domain/annotation.py | 20 ++- ebl/mongo_collection.py | 3 +- .../fragmentarium/test_annotations_service.py | 130 ++++++---------- ebl/tests/transliteration/test_line_label.py | 30 ++++ ebl/tests/transliteration/test_line_number.py | 14 ++ ebl/transliteration/domain/line_label.py | 17 +++ ebl/transliteration/domain/line_number.py | 10 ++ 9 files changed, 191 insertions(+), 182 deletions(-) create mode 100644 ebl/tests/transliteration/test_line_label.py diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index 733bef00b..af7f689ad 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -1,41 +1,30 @@ -import base64 -import io from io import BytesIO from typing import Tuple, Sequence import attr -import bson.objectid from PIL import Image -from functools import singledispatchmethod from ebl.changelog import Changelog from ebl.ebl_ai_client import EblAiClient from ebl.files.application.file_repository import FileRepository from ebl.fragmentarium.application.annotations_repository import AnnotationsRepository from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema -from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSign -from ebl.fragmentarium.application.fragment_repository import FragmentRepository +from ebl.fragmentarium.application.cropped_sign_image import CroppedSign from ebl.fragmentarium.application.cropped_sign_images_repository import ( CroppedSignImage, CroppedSignImagesRepository, ) +from ebl.fragmentarium.application.fragment_repository import FragmentRepository from ebl.fragmentarium.domain.annotation import ( Annotations, - Annotation, - BoundingBox, AnnotationValueType, ) from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.transliteration.domain.line_label import LineLabel -from ebl.transliteration.domain.line_number import ( - LineNumber, - LineNumberRange, - AbstractLineNumber, -) from ebl.users.domain.user import User -@attr.attrs(auto_attribs=True, frozen=True) +@attr.attrs(auto_attribs=True) class AnnotationsService: _ebl_ai_client: EblAiClient _annotations_repository: AnnotationsRepository @@ -56,56 +45,55 @@ def generate_annotations( def find(self, number: MuseumNumber) -> Annotations: return self._annotations_repository.query_by_museum_number(number) - def _format_label(self, label: LineLabel) -> str: - line_number = label.line_number - column = label.column - surface = label.surface - object = label.object - line_atf = line_number.atf if line_number else "" - column_abbr = column.abbreviation if column else "" - surface_abbr = surface.abbreviation if surface else "" - object_abbr = object.abbreviation if object else "" - return " ".join( - filter( - bool, - [column_abbr, surface_abbr, object_abbr, line_atf.replace(".", "")], - ) - ) - - def _cropped_image_from_annotation( - self, annotation: Annotation, image: Image.Image - ) -> Base64: - bounding_box = BoundingBox.from_annotations( - image.size[0], image.size[1], [annotation] - )[0] - area = ( - bounding_box.top_left_x, - bounding_box.top_left_y, - bounding_box.top_left_x + bounding_box.width, - bounding_box.top_left_y + bounding_box.height, - ) - cropped_image = image.crop(area) - buf = io.BytesIO() - cropped_image.save(buf, format="PNG") - return Base64(base64.b64encode(buf.getvalue()).decode("utf-8")) + def _label_by_line_number( + self, line_number_to_match: int, labels: Sequence[LineLabel] + ) -> str: + matching_label = None + for label in labels: + label_line_number = label.line_number + if label_line_number and label_line_number.is_matching_number( + line_number_to_match + ): + matching_label = label + return matching_label.formatted_label if matching_label else "" - @singledispatchmethod - def _is_matching_number(self, line_number: AbstractLineNumber, number: int) -> bool: - raise ValueError("No default for overloading") + def _cropped_image_from_annotations_helper( + self, + annotations: Annotations, + image: Image.Image, + script: str, + labels: Sequence[LineLabel], + ) -> Tuple[Annotations, Sequence[CroppedSignImage]]: + cropped_sign_images = [] + updated_cropped_annotations = [] - @_is_matching_number.register(LineNumber) - def _(self, line_number: LineNumber, number: int): - return number == line_number.number + for annotation in annotations.annotations: + label = ( + self._label_by_line_number(annotation.data.path[0], labels) + if annotation.data.type != AnnotationValueType.BLANK + else "" + ) + cropped_image = annotation.crop_image(image) + cropped_sign_image = CroppedSignImage.create(cropped_image) + cropped_sign_images.append(cropped_sign_image) - @_is_matching_number.register(LineNumberRange) - def _(self, line_number: LineNumberRange, number: int): - return line_number.start.number <= number <= line_number.end.number + updated_cropped_annotation = attr.evolve( + annotation, + cropped_sign=CroppedSign( + cropped_sign_image.image_id, + script, + label, + ), + ) + updated_cropped_annotations.append(updated_cropped_annotation) + return ( + attr.evolve(annotations, annotations=updated_cropped_annotations), + cropped_sign_images, + ) def _cropped_image_from_annotations( self, annotations: Annotations ) -> Tuple[Annotations, Sequence[CroppedSignImage]]: - cropped_sign_images = [] - cropped_annotations = [] fragment = self._fragments_repository.query_by_museum_number( annotations.fragment_number ) @@ -114,41 +102,8 @@ def _cropped_image_from_annotations( ) image_bytes = fragment_image.read() image = Image.open(BytesIO(image_bytes), mode="r") - for annotation in annotations.annotations: - script = fragment.script - labels = fragment.text.labels - label = ( - next( - ( - label - for label in labels - if self._is_matching_number( - label.line_number, annotation.data.path[0] - ) - ), - None, - ) - if annotation.data.type != AnnotationValueType.BLANK - else None - ) - cropped_image_base64 = self._cropped_image_from_annotation( - annotation, image - ) - image_id = str(bson.ObjectId()) - cropped_sign_images.append(CroppedSignImage(image_id, cropped_image_base64)) - cropped_annotations.append( - attr.evolve( - annotation, - cropped_sign=CroppedSign( - image_id, - script, - self._format_label(label) if label else "", - ), - ) - ) - return ( - attr.evolve(annotations, annotations=cropped_annotations), - cropped_sign_images, + return self._cropped_image_from_annotations_helper( + annotations, image, fragment.script, fragment.text.labels ) def update(self, annotations: Annotations, user: User) -> Annotations: @@ -161,8 +116,10 @@ def update(self, annotations: Annotations, user: User) -> Annotations: annotations_with_image_ids, cropped_sign_images, ) = self._cropped_image_from_annotations(annotations) + self._annotations_repository.create_or_update(annotations_with_image_ids) self._cropped_sign_images_repository.create_many(cropped_sign_images) + self._changelog.create( "annotations", user.profile, diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index a4da34aa7..52036e113 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -1,6 +1,8 @@ +import uuid from typing import NewType -from marshmallow import Schema, fields, post_load, post_dump + import attr +from marshmallow import Schema, fields, post_load, post_dump from ebl.fragmentarium.domain.museum_number import MuseumNumber @@ -12,6 +14,10 @@ class CroppedSignImage: image_id: str image: Base64 + @classmethod + def create(cls, image: Base64) -> "CroppedSignImage": + return cls(str(uuid.uuid4()), image) + class CroppedSignImageSchema(Schema): image_id = fields.Str(required=True) diff --git a/ebl/fragmentarium/domain/annotation.py b/ebl/fragmentarium/domain/annotation.py index 74dc86c2e..c9c88812e 100644 --- a/ebl/fragmentarium/domain/annotation.py +++ b/ebl/fragmentarium/domain/annotation.py @@ -1,10 +1,13 @@ +import base64 +import io +from PIL import Image from enum import Enum from typing import Sequence, Optional from uuid import uuid4 import attr -from ebl.fragmentarium.application.cropped_sign_image import CroppedSign +from ebl.fragmentarium.application.cropped_sign_image import CroppedSign, Base64 from ebl.fragmentarium.domain.museum_number import MuseumNumber @@ -41,6 +44,21 @@ class Annotation: data: AnnotationData cropped_sign: Optional[CroppedSign] + def crop_image(self, image: Image.Image) -> Base64: + bounding_box = BoundingBox.from_annotations( + image.size[0], image.size[1], [self] + )[0] + area = ( + bounding_box.top_left_x, + bounding_box.top_left_y, + bounding_box.top_left_x + bounding_box.width, + bounding_box.top_left_y + bounding_box.height, + ) + cropped_image = image.crop(area) + buf = io.BytesIO() + cropped_image.save(buf, format="PNG") + return Base64(base64.b64encode(buf.getvalue()).decode("utf-8")) + @classmethod def from_prediction(cls, geometry: Geometry) -> "Annotation": data = AnnotationData(uuid4().hex, "", AnnotationValueType.PREDICTED, [], "") diff --git a/ebl/mongo_collection.py b/ebl/mongo_collection.py index 5759d7542..874bab36a 100644 --- a/ebl/mongo_collection.py +++ b/ebl/mongo_collection.py @@ -1,7 +1,6 @@ from typing import Any, cast, Sequence import inflect -from bson import ObjectId from pymongo.collection import Collection from pymongo.database import Database from pymongo.errors import DuplicateKeyError @@ -20,7 +19,7 @@ def __init__(self, database: Database, collection: str): self.__collection = collection self.__resource_noun = singlar(collection) - def insert_many(self, documents: Sequence[dict]) -> Sequence[ObjectId]: + def insert_many(self, documents: Sequence[dict]): return self.__get_collection().insert_many(documents).inserted_ids def insert_one(self, document): diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index 02dbf34ea..2525677a3 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -1,83 +1,30 @@ +import attr + from ebl.ebl_ai_client import EblAiClient from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema from ebl.fragmentarium.application.annotations_service import AnnotationsService +from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSignImage from ebl.fragmentarium.domain.annotation import Annotations from ebl.fragmentarium.domain.museum_number import MuseumNumber - -import pytest - from ebl.tests.conftest import create_test_photo from ebl.tests.factories.annotation import ( AnnotationsFactory, AnnotationFactory, AnnotationDataFactory, + CroppedSignFactory, ) from ebl.tests.factories.fragment import TransliteratedFragmentFactory -from ebl.transliteration.domain import atf -from ebl.transliteration.domain.labels import ColumnLabel, SurfaceLabel, ObjectLabel -from ebl.transliteration.domain.line_label import LineLabel -from ebl.transliteration.domain.line_number import LineNumber, LineNumberRange + SCHEMA = AnnotationsSchema() -@pytest.mark.parametrize( - "line_label, expected", - [ - ( - LineLabel( - ColumnLabel.from_int(1), - SurfaceLabel([], atf.Surface.SURFACE, "Stone wig"), - ObjectLabel([], atf.Object.OBJECT, "Stone wig"), - LineNumber(2), - ), - "i Stone wig Stone wig 2", - ), - ( - LineLabel( - None, None, None, LineNumberRange(LineNumber(1, True), LineNumber(3)) - ), - "1'-3", - ), - ], -) -def test_format_line_label(line_label, expected, annotations_service): - assert annotations_service._format_label(line_label) == expected - - -@pytest.mark.parametrize( - "line_label, line_number, expected", - [ - ( - LineNumber(2), - 2, - True, - ), - ( - LineNumber(2), - 1, - False, - ), - ( - LineNumberRange(LineNumber(1, True), LineNumber(3)), - 2, - True, - ), - ( - LineNumberRange(LineNumber(1, True), LineNumber(3)), - 4, - False, - ), - ], -) -def test_line_label_match_line_number( - line_label, - line_number, - expected, - annotations_service, -): - assert annotations_service._is_matching_number(line_label, line_number) == expected +def test_label_by_line_number(text_with_labels, annotations_service): + assert ( + annotations_service._label_by_line_number(2, text_with_labels.labels) + == "i Stone wig Stone wig 2" + ) def test_cropped_images_from_sign( @@ -163,36 +110,47 @@ def test_update( annotations_repository, photo_repository, fragment_repository, + cropped_sign_images_repository, when, user, changelog, text_with_labels, ): fragment_number = MuseumNumber("K", "1") - annotations = AnnotationsFactory.build(fragment_number=fragment_number) - updated_annotations = AnnotationsFactory.build(fragment_number=fragment_number) - fragment = TransliteratedFragmentFactory.build(text=text_with_labels) - when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( - annotations + old_annotations = AnnotationsFactory.build(fragment_number=fragment_number) + + annotation = AnnotationFactory.build(cropped_sign=None) + annotations = AnnotationsFactory.build( + fragment_number=fragment_number, annotations=[annotation] ) - when(annotations_repository).create_or_update(...).thenReturn() - when(changelog).create(...).thenReturn() - ( - when(fragment_repository) - .query_by_museum_number(annotations.fragment_number) - .thenReturn(fragment) + + expected_cropped_sign_images = [CroppedSignImage("test-id", Base64("test-image"))] + annotation_cropped_sign = attr.evolve( + annotation, cropped_sign=CroppedSignFactory.build() ) - ( - when(photo_repository) - .query_by_file_name(f"{annotations.fragment_number}.jpg") - .thenReturn(create_test_photo("K.2")) + expected_annotations = attr.evolve( + annotations, annotations=[annotation_cropped_sign] + ) + + when(annotations_service)._cropped_image_from_annotations(annotations).thenReturn( + (expected_annotations, expected_cropped_sign_images) + ) + + when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( + old_annotations ) - result = annotations_service.update(updated_annotations, user) - assert result.fragment_number == updated_annotations.fragment_number - for result_annotation, annotation in zip( - result.annotations, updated_annotations.annotations - ): - assert result_annotation.geometry == annotation.geometry - assert result_annotation.data == annotation.data - assert result_annotation.cropped_sign != annotation.cropped_sign + when(annotations_repository).create_or_update(expected_annotations).thenReturn() + when(cropped_sign_images_repository).create_many( + expected_cropped_sign_images + ).thenReturn() + schema = AnnotationsSchema() + when(changelog).create( + "annotations", + user.profile, + {"_id": str(fragment_number), **schema.dump(old_annotations)}, + {"_id": str(fragment_number), **schema.dump(expected_annotations)}, + ).thenReturn() + + result = annotations_service.update(annotations, user) + assert result == expected_annotations diff --git a/ebl/tests/transliteration/test_line_label.py b/ebl/tests/transliteration/test_line_label.py new file mode 100644 index 000000000..f3c8cd849 --- /dev/null +++ b/ebl/tests/transliteration/test_line_label.py @@ -0,0 +1,30 @@ +import pytest + +from ebl.transliteration.domain import atf +from ebl.transliteration.domain.labels import ColumnLabel, SurfaceLabel, ObjectLabel +from ebl.transliteration.domain.line_label import LineLabel +from ebl.transliteration.domain.line_number import LineNumber, LineNumberRange + + +@pytest.mark.parametrize( + "line_label, expected", + [ + ( + LineLabel( + ColumnLabel.from_int(1), + SurfaceLabel([], atf.Surface.SURFACE, "Stone wig"), + ObjectLabel([], atf.Object.OBJECT, "Stone wig"), + LineNumber(2), + ), + "i Stone wig Stone wig 2", + ), + ( + LineLabel( + None, None, None, LineNumberRange(LineNumber(1, True), LineNumber(3)) + ), + "1'-3", + ), + ], +) +def test_format_line_label(line_label, expected, annotations_service): + assert line_label.formatted_label == expected diff --git a/ebl/tests/transliteration/test_line_number.py b/ebl/tests/transliteration/test_line_number.py index d59210a06..a813a7d9b 100644 --- a/ebl/tests/transliteration/test_line_number.py +++ b/ebl/tests/transliteration/test_line_number.py @@ -47,3 +47,17 @@ def test_line_number_range(start: LineNumber, end: LineNumber) -> None: assert line_number.atf == f"{label}." assert line_number.label == label assert line_number.is_beginning_of_side == start.is_beginning_of_side + + +@pytest.mark.parametrize( + "line_number, matching_number, expected", + [ + (LineNumber(1), 1, True), + (LineNumber(2), 1, False), + (LineNumberRange(LineNumber(1), LineNumber(3)), 1, True), + (LineNumberRange(LineNumber(1), LineNumber(3)), 2, True), + (LineNumberRange(LineNumber(1), LineNumber(3)), 4, False), + ], +) +def test_is_line_matching_number(line_number, matching_number, expected): + assert line_number.is_matching_number(matching_number) == expected diff --git a/ebl/transliteration/domain/line_label.py b/ebl/transliteration/domain/line_label.py index 5298e3e4a..8effe5476 100644 --- a/ebl/transliteration/domain/line_label.py +++ b/ebl/transliteration/domain/line_label.py @@ -26,3 +26,20 @@ def set_object(self, object: Optional[ObjectLabel]) -> "LineLabel": def set_line_number(self, line_number: Optional[AbstractLineNumber]) -> "LineLabel": return attr.evolve(self, line_number=line_number) + + @property + def formatted_label(self) -> str: + line_number = self.line_number + column = self.column + surface = self.surface + object = self.object + line_atf = line_number.atf if line_number else "" + column_abbr = column.abbreviation if column else "" + surface_abbr = surface.abbreviation if surface else "" + object_abbr = object.abbreviation if object else "" + return " ".join( + filter( + bool, + [column_abbr, surface_abbr, object_abbr, line_atf.replace(".", "")], + ) + ) diff --git a/ebl/transliteration/domain/line_number.py b/ebl/transliteration/domain/line_number.py index 7b9992e15..146278511 100644 --- a/ebl/transliteration/domain/line_number.py +++ b/ebl/transliteration/domain/line_number.py @@ -19,6 +19,10 @@ def is_beginning_of_side(self) -> bool: def atf(self) -> str: return f"{self.label}." + @abstractmethod + def is_matching_number(self, number: int) -> bool: + ... + @attr.s(auto_attribs=True, frozen=True) class LineNumber(AbstractLineNumber): @@ -38,6 +42,9 @@ def label(self) -> str: def is_beginning_of_side(self) -> bool: return self.number == 1 and not self.has_prime and self.prefix_modifier is None + def is_matching_number(self, number: int) -> bool: + return number == self.number + @attr.s(auto_attribs=True, frozen=True) class LineNumberRange(AbstractLineNumber): @@ -51,3 +58,6 @@ def label(self) -> str: @property def is_beginning_of_side(self) -> bool: return self.start.is_beginning_of_side + + def is_matching_number(self, number: int) -> bool: + return self.start.number <= number <= self.end.number From d74b8963c4c28183ad5d2b87b8d07932785b7ce6 Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Tue, 29 Mar 2022 15:28:03 +0200 Subject: [PATCH 22/23] merge with master --- ebl/app.py | 1 - .../application/annotations_service.py | 3 ++- ebl/fragmentarium/application/cropped_sign_image.py | 2 +- ebl/fragmentarium/domain/annotation.py | 2 +- ebl/tests/conftest.py | 13 ++++++------- ebl/tests/fragmentarium/test_annotations_route.py | 3 ++- ebl/tests/fragmentarium/test_annotations_service.py | 4 ++-- ebl/tests/signs/test_sign_images_route.py | 3 ++- 8 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ebl/app.py b/ebl/app.py index 89760c6f3..769b91801 100644 --- a/ebl/app.py +++ b/ebl/app.py @@ -26,7 +26,6 @@ from ebl.fragmentarium.infrastructure.cropped_sign_images_repository import ( MongoCroppedSignImagesRepository, ) -from ebl.fragmentarium.infrastructure.fragment_repository import MongoFragmentRepository from ebl.fragmentarium.infrastructure.mongo_annotations_repository import ( MongoAnnotationsRepository, ) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index af7f689ad..ab66fbd03 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -19,8 +19,9 @@ Annotations, AnnotationValueType, ) -from ebl.fragmentarium.domain.museum_number import MuseumNumber + from ebl.transliteration.domain.line_label import LineLabel +from ebl.transliteration.domain.museum_number import MuseumNumber from ebl.users.domain.user import User diff --git a/ebl/fragmentarium/application/cropped_sign_image.py b/ebl/fragmentarium/application/cropped_sign_image.py index 52036e113..5863b1395 100644 --- a/ebl/fragmentarium/application/cropped_sign_image.py +++ b/ebl/fragmentarium/application/cropped_sign_image.py @@ -4,7 +4,7 @@ import attr from marshmallow import Schema, fields, post_load, post_dump -from ebl.fragmentarium.domain.museum_number import MuseumNumber +from ebl.transliteration.domain.museum_number import MuseumNumber Base64 = NewType("Base64", str) diff --git a/ebl/fragmentarium/domain/annotation.py b/ebl/fragmentarium/domain/annotation.py index c9c88812e..79ec6fb73 100644 --- a/ebl/fragmentarium/domain/annotation.py +++ b/ebl/fragmentarium/domain/annotation.py @@ -8,7 +8,7 @@ import attr from ebl.fragmentarium.application.cropped_sign_image import CroppedSign, Base64 -from ebl.fragmentarium.domain.museum_number import MuseumNumber +from ebl.transliteration.domain.museum_number import MuseumNumber @attr.attrs(auto_attribs=True, frozen=True) diff --git a/ebl/tests/conftest.py b/ebl/tests/conftest.py index f7418cfdf..bf3f6d173 100644 --- a/ebl/tests/conftest.py +++ b/ebl/tests/conftest.py @@ -10,6 +10,7 @@ import pytest from PIL import Image from dictdiffer import diff + from falcon import testing from falcon_auth import NoneAuthBackend from falcon_caching import Cache @@ -38,19 +39,15 @@ from ebl.fragmentarium.application.transliteration_update_factory import ( TransliterationUpdateFactory, ) -from ebl.transliteration.application.parallel_line_injector import ParallelLineInjector -from ebl.transliteration.domain.museum_number import MuseumNumber -from ebl.fragmentarium.infrastructure.mongo_fragment_repository import ( - MongoFragmentRepository, -) -from ebl.fragmentarium.domain.museum_number import MuseumNumber from ebl.fragmentarium.infrastructure.cropped_sign_images_repository import ( MongoCroppedSignImagesRepository, ) -from ebl.fragmentarium.infrastructure.fragment_repository import MongoFragmentRepository from ebl.fragmentarium.infrastructure.mongo_annotations_repository import ( MongoAnnotationsRepository, ) +from ebl.fragmentarium.infrastructure.mongo_fragment_repository import ( + MongoFragmentRepository, +) from ebl.lemmatization.infrastrcuture.mongo_suggestions_finder import ( MongoLemmaRepository, ) @@ -59,10 +56,12 @@ SignSchema, ) from ebl.tests.factories.bibliography import BibliographyEntryFactory +from ebl.transliteration.application.parallel_line_injector import ParallelLineInjector from ebl.transliteration.domain import atf from ebl.transliteration.domain.at_line import ColumnAtLine, SurfaceAtLine, ObjectAtLine from ebl.transliteration.domain.labels import ColumnLabel, SurfaceLabel, ObjectLabel from ebl.transliteration.domain.line_number import LineNumber +from ebl.transliteration.domain.museum_number import MuseumNumber from ebl.transliteration.domain.sign import Sign, SignListRecord, Value from ebl.transliteration.domain.sign_tokens import Reading from ebl.transliteration.domain.text import Text diff --git a/ebl/tests/fragmentarium/test_annotations_route.py b/ebl/tests/fragmentarium/test_annotations_route.py index aa0c94553..f4ad88285 100644 --- a/ebl/tests/fragmentarium/test_annotations_route.py +++ b/ebl/tests/fragmentarium/test_annotations_route.py @@ -5,10 +5,11 @@ from ebl.fragmentarium.application.annotations_schema import AnnotationsSchema from ebl.fragmentarium.domain.annotation import Annotations -from ebl.fragmentarium.domain.museum_number import MuseumNumber + from ebl.tests.conftest import create_test_photo from ebl.tests.factories.annotation import AnnotationsFactory, AnnotationFactory from ebl.tests.factories.fragment import TransliteratedFragmentFactory +from ebl.transliteration.domain.museum_number import MuseumNumber def test_find_annotations(client): diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index 2525677a3..7efb77f77 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -5,7 +5,7 @@ from ebl.fragmentarium.application.annotations_service import AnnotationsService from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSignImage from ebl.fragmentarium.domain.annotation import Annotations -from ebl.fragmentarium.domain.museum_number import MuseumNumber + from ebl.tests.conftest import create_test_photo from ebl.tests.factories.annotation import ( @@ -15,7 +15,7 @@ CroppedSignFactory, ) from ebl.tests.factories.fragment import TransliteratedFragmentFactory - +from ebl.transliteration.domain.museum_number import MuseumNumber SCHEMA = AnnotationsSchema() diff --git a/ebl/tests/signs/test_sign_images_route.py b/ebl/tests/signs/test_sign_images_route.py index 986d4e9bc..f980b05f9 100644 --- a/ebl/tests/signs/test_sign_images_route.py +++ b/ebl/tests/signs/test_sign_images_route.py @@ -1,7 +1,7 @@ import falcon from ebl.fragmentarium.application.cropped_sign_image import CroppedSignImage, Base64 -from ebl.fragmentarium.domain.museum_number import MuseumNumber + from ebl.tests.factories.annotation import ( AnnotationsFactory, AnnotationFactory, @@ -9,6 +9,7 @@ CroppedSignFactory, ) from ebl.tests.factories.fragment import TransliteratedFragmentFactory +from ebl.transliteration.domain.museum_number import MuseumNumber def test_signs_get( From 2da6a16f6ed76036c41092d30092b057aa01560c Mon Sep 17 00:00:00 2001 From: yCobanoglu Date: Wed, 30 Mar 2022 23:02:49 +0200 Subject: [PATCH 23/23] improve test --- .../application/annotations_service.py | 2 +- .../fragmentarium/test_annotations_service.py | 38 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ebl/fragmentarium/application/annotations_service.py b/ebl/fragmentarium/application/annotations_service.py index ab66fbd03..790182ffa 100644 --- a/ebl/fragmentarium/application/annotations_service.py +++ b/ebl/fragmentarium/application/annotations_service.py @@ -25,7 +25,7 @@ from ebl.users.domain.user import User -@attr.attrs(auto_attribs=True) +@attr.attrs(auto_attribs=True, frozen=True) class AnnotationsService: _ebl_ai_client: EblAiClient _annotations_repository: AnnotationsRepository diff --git a/ebl/tests/fragmentarium/test_annotations_service.py b/ebl/tests/fragmentarium/test_annotations_service.py index 7efb77f77..3b5c4ee18 100644 --- a/ebl/tests/fragmentarium/test_annotations_service.py +++ b/ebl/tests/fragmentarium/test_annotations_service.py @@ -5,8 +5,6 @@ from ebl.fragmentarium.application.annotations_service import AnnotationsService from ebl.fragmentarium.application.cropped_sign_image import Base64, CroppedSignImage from ebl.fragmentarium.domain.annotation import Annotations - - from ebl.tests.conftest import create_test_photo from ebl.tests.factories.annotation import ( AnnotationsFactory, @@ -117,32 +115,46 @@ def test_update( text_with_labels, ): fragment_number = MuseumNumber("K", "1") + fragment = TransliteratedFragmentFactory( + number=fragment_number, text=text_with_labels + ) old_annotations = AnnotationsFactory.build(fragment_number=fragment_number) - annotation = AnnotationFactory.build(cropped_sign=None) + data = AnnotationDataFactory.build(path=[2, 0, 0]) + annotation = AnnotationFactory.build(cropped_sign=None, data=data) annotations = AnnotationsFactory.build( fragment_number=fragment_number, annotations=[annotation] ) - expected_cropped_sign_images = [CroppedSignImage("test-id", Base64("test-image"))] + when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( + old_annotations + ) + image = create_test_photo("K.2") + when(fragment_repository).query_by_museum_number(fragment_number).thenReturn( + fragment + ) + ( + when(photo_repository) + .query_by_file_name(f"{annotations.fragment_number}.jpg") + .thenReturn(image) + ) + + expected_cropped_sign_image = CroppedSignImage("test-id", Base64("test-image")) annotation_cropped_sign = attr.evolve( - annotation, cropped_sign=CroppedSignFactory.build() + annotation, + cropped_sign=CroppedSignFactory.build( + image_id="test-id", script=fragment.script, label="i Stone wig Stone wig 2" + ), ) expected_annotations = attr.evolve( annotations, annotations=[annotation_cropped_sign] ) + when(CroppedSignImage).create(...).thenReturn(expected_cropped_sign_image) - when(annotations_service)._cropped_image_from_annotations(annotations).thenReturn( - (expected_annotations, expected_cropped_sign_images) - ) - - when(annotations_repository).query_by_museum_number(fragment_number).thenReturn( - old_annotations - ) when(annotations_repository).create_or_update(expected_annotations).thenReturn() when(cropped_sign_images_repository).create_many( - expected_cropped_sign_images + [expected_cropped_sign_image] ).thenReturn() schema = AnnotationsSchema() when(changelog).create(