From 90dd43289a50afa898ee30964df83da2f5086e24 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 6 Dec 2023 14:38:00 +0100 Subject: [PATCH 01/37] Update GenerateAnnotationPatch with test cases from biigle/maia --- src/Jobs/GenerateAnnotationPatch.php | 14 +++--- .../Jobs/GenerateImageAnnotationPatchTest.php | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/src/Jobs/GenerateAnnotationPatch.php b/src/Jobs/GenerateAnnotationPatch.php index eac153b3..cc8a726a 100644 --- a/src/Jobs/GenerateAnnotationPatch.php +++ b/src/Jobs/GenerateAnnotationPatch.php @@ -180,14 +180,12 @@ protected function getPatchRect(array $points, Shape $shape, $thumbWidth, $thumb $right = -INF; $top = INF; $bottom = -INF; - foreach ($points as $index => $value) { - if ($index % 2 === 0) { - $left = min($left, $value); - $right = max($right, $value); - } else { - $top = min($top, $value); - $bottom = max($bottom, $value); - } + $pointCount = count($points); + for ($i = 0; $i < $pointCount; $i += 2) { + $left = min($left, $points[$i]); + $top = min($top, $points[$i + 1]); + $right = max($right, $points[$i]); + $bottom = max($bottom, $points[$i + 1]); } } diff --git a/tests/Jobs/GenerateImageAnnotationPatchTest.php b/tests/Jobs/GenerateImageAnnotationPatchTest.php index aae70e43..fd9af7a0 100644 --- a/tests/Jobs/GenerateImageAnnotationPatchTest.php +++ b/tests/Jobs/GenerateImageAnnotationPatchTest.php @@ -223,6 +223,52 @@ public function testHandleMinDimension() $job->handleFile($annotation->image, 'abc'); } + public function testHandleContainedNegativeProblematic() + { + config(['thumbnails.height' => 100, 'thumbnails.width' => 100]); + Storage::fake('test'); + $image = $this->getImageMock(); + $image->width = 25; + $image->height = 25; + $annotation = ImageAnnotationTest::create([ + 'points' => [10, 10, 15], + 'shape_id' => Shape::circleId(), + ]); + $job = new GenerateImageAnnotationPatchStub($annotation); + $job->mock = $image; + + $image->shouldReceive('crop') + ->with(0, 0, 25, 25) + ->once() + ->andReturn($image); + + $image->shouldReceive('writeToBuffer')->once()->andReturn('abc123'); + $job->handleFile($annotation->image, 'abc'); + } + + public function testHandleContainedPositiveProblematic() + { + config(['thumbnails.height' => 100, 'thumbnails.width' => 100]); + Storage::fake('test'); + $image = $this->getImageMock(); + $image->width = 25; + $image->height = 25; + $annotation = ImageAnnotationTest::create([ + 'points' => [15, 15, 15], + 'shape_id' => Shape::circleId(), + ]); + $job = new GenerateImageAnnotationPatchStub($annotation); + $job->mock = $image; + + $image->shouldReceive('crop') + ->with(0, 0, 25, 25) + ->once() + ->andReturn($image); + + $image->shouldReceive('writeToBuffer')->once()->andReturn('abc123'); + $job->handleFile($annotation->image, 'abc'); + } + public function testHandleError() { FileCache::shouldReceive('get')->andThrow(new Exception('error')); From 9e406a14d6af7489b45ab4bb37b9796ebdfab827 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Fri, 8 Dec 2023 15:00:17 +0100 Subject: [PATCH 02/37] Implement trait to compute an annotation bounding box This can be used by biigle/maia, too. --- src/Jobs/GenerateAnnotationPatch.php | 134 ++-------------------- src/Traits/ComputesAnnotationBox.php | 161 +++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 125 deletions(-) create mode 100644 src/Traits/ComputesAnnotationBox.php diff --git a/src/Jobs/GenerateAnnotationPatch.php b/src/Jobs/GenerateAnnotationPatch.php index cc8a726a..3f14bd6c 100644 --- a/src/Jobs/GenerateAnnotationPatch.php +++ b/src/Jobs/GenerateAnnotationPatch.php @@ -5,6 +5,7 @@ use Biigle\Contracts\Annotation; use Biigle\FileCache\Exceptions\FileLockedException; use Biigle\Jobs\Job; +use Biigle\Modules\Largo\Traits\ComputesAnnotationBox; use Biigle\Shape; use Biigle\VideoAnnotation; use Biigle\VolumeFile; @@ -20,7 +21,7 @@ abstract class GenerateAnnotationPatch extends Job implements ShouldQueue { - use SerializesModels, InteractsWithQueue; + use SerializesModels, InteractsWithQueue, ComputesAnnotationBox; /** * The number of times the job may be attempted. @@ -145,121 +146,6 @@ protected function shouldRetryAfterException(Exception $e) ); } - /** - * Calculate the bounding rectangle of the patch to extract. - * - * @param array $points - * @param Shape $Shape - * @param int $thumbWidth - * @param int $thumbHeight - * - * @return array Containing width, height, top and left - */ - protected function getPatchRect(array $points, Shape $shape, $thumbWidth, $thumbHeight) - { - $padding = config('largo.patch_padding'); - - switch ($shape->id) { - case Shape::pointId(): - $pointPadding = config('largo.point_padding'); - $left = $points[0] - $pointPadding; - $right = $points[0] + $pointPadding; - $top = $points[1] - $pointPadding; - $bottom = $points[1] + $pointPadding; - break; - - case Shape::circleId(): - $left = $points[0] - $points[2]; - $right = $points[0] + $points[2]; - $top = $points[1] - $points[2]; - $bottom = $points[1] + $points[2]; - break; - - default: - $left = INF; - $right = -INF; - $top = INF; - $bottom = -INF; - $pointCount = count($points); - for ($i = 0; $i < $pointCount; $i += 2) { - $left = min($left, $points[$i]); - $top = min($top, $points[$i + 1]); - $right = max($right, $points[$i]); - $bottom = max($bottom, $points[$i + 1]); - } - } - - $left -= $padding; - $right += $padding; - $top -= $padding; - $bottom += $padding; - - $width = $right - $left; - $height = $bottom - $top; - - // Ensure the minimum width so the annotation patch is not "zoomed in". - if ($width < $thumbWidth) { - $delta = ($thumbWidth - $width) / 2.0; - $left -= $delta; - $right += $delta; - $width = $thumbWidth; - } - - // Ensure the minimum height so the annotation patch is not "zoomed in". - if ($height < $thumbHeight) { - $delta = ($thumbHeight - $height) / 2.0; - $top -= $delta; - $bottom += $delta; - $height = $thumbHeight; - } - - $widthRatio = $width / $thumbWidth; - $heightRatio = $height / $thumbHeight; - - // increase the size of the patch so its aspect ratio is the same than the - // ratio of the thumbnail dimensions - if ($widthRatio > $heightRatio) { - $newHeight = round($thumbHeight * $widthRatio); - $top -= round(($newHeight - $height) / 2); - $height = $newHeight; - } else { - $newWidth = round($thumbWidth * $heightRatio); - $left -= round(($newWidth - $width) / 2); - $width = $newWidth; - } - - return [ - 'width' => intval(round($width)), - 'height' => intval(round($height)), - 'left' => intval(round($left)), - 'top' => intval(round($top)), - ]; - } - - /** - * Adjust the position and size of the patch rectangle so it is contained in the - * image. - * - * @param array $rect - * @param Image $image - * - * @return array - */ - protected function makeRectContained($rect, $image) - { - // Order of min max is importans so the point gets no negative coordinates. - $rect['left'] = min($image->width - $rect['width'], $rect['left']); - $rect['left'] = max(0, $rect['left']); - $rect['top'] = min($image->height - $rect['height'], $rect['top']); - $rect['top'] = max(0, $rect['top']); - - // Adjust dimensions of rect if it is larger than the image. - $rect['width'] = min($image->width, $rect['width']); - $rect['height'] = min($image->height, $rect['height']); - - return $rect; - } - /** * Get the annotation patch as buffer. * @@ -277,16 +163,14 @@ protected function getAnnotationPatch($image, $points, $shape) if ($shape->id === Shape::wholeFrameId()) { $image = $image->resize(floatval($thumbWidth) / $image->width); } else { - $rect = $this->getPatchRect($points, $shape, $thumbWidth, $thumbHeight); - $rect = $this->makeRectContained($rect, $image); + $padding = config('largo.patch_padding'); + $pointPadding = config('largo.point_padding'); + + $box = $this->getAnnotationBoundingBox($points, $shape, $pointPadding, $padding); + $box = $this->ensureBoxAspectRatio($box, $thumbWidth, $thumbHeight); + $box = $this->makeBoxContained($box, $image->width, $image->height); - $image = $image->crop( - $rect['left'], - $rect['top'], - $rect['width'], - $rect['height'] - ) - ->resize(floatval($thumbWidth) / $rect['width']); + $image = $image->crop(...$box)->resize(floatval($thumbWidth) / $box[2]); } return $image->writeToBuffer('.'.config('largo.patch_format'), [ diff --git a/src/Traits/ComputesAnnotationBox.php b/src/Traits/ComputesAnnotationBox.php new file mode 100644 index 00000000..6d46f101 --- /dev/null +++ b/src/Traits/ComputesAnnotationBox.php @@ -0,0 +1,161 @@ +id) { + Shape::pointId() => $this->getPointBoundingBox($points, $pointPadding), + Shape::circleId() => $this->getCircleBoundingBox($points), + // An ellipse will not be handled correctly by this but I didn't bother + // because this shape is almost never used anyway. + default => $this->getPolygonBoundingBox($points), + }; + + if ($boxPadding > 0) { + $box = [ + $box[0] - $boxPadding, + $box[1] - $boxPadding, + $box[2] + $boxPadding, + $box[3] + $boxPadding, + ]; + } + + return $this->makeBoxIntegers([ + $box[0], // left + $box[1], // top + $box[2] - $box[0], // width + $box[3] - $box[1], // height + ]); + } + + /** + * Modify a bounding box so it adheres to the aspect ratio given by width and height. + */ + public function ensureBoxAspectRatio(array $box, int $aspectWidth, int $aspectHeight): array + { + [$left, $top, $width, $height] = $box; + + // Ensure the minimum width so the annotation patch is not "zoomed in". + if ($width < $aspectWidth) { + $left -= ($aspectWidth - $width) / 2.0; + $width = $aspectWidth; + } + + // Ensure the minimum height so the annotation patch is not "zoomed in". + if ($height < $aspectHeight) { + $top -= ($aspectHeight - $height) / 2.0; + $height = $aspectHeight; + } + + $widthRatio = $width / $aspectWidth; + $heightRatio = $height / $aspectHeight; + + // Increase the size of the patch so its aspect ratio is the same than the + // ratio of the given dimensions. + if ($widthRatio > $heightRatio) { + $newHeight = round($aspectHeight * $widthRatio); + $top -= round(($newHeight - $height) / 2); + $height = $newHeight; + } else { + $newWidth = round($aspectWidth * $heightRatio); + $left -= round(($newWidth - $width) / 2); + $width = $newWidth; + } + + return $this->makeBoxIntegers([$left, $top, $width, $height]); + } + + /** + * Adjust the position and size of the box so it is contained in a box with the given + * dimensions. + */ + public function makeBoxContained(array $box, ?int $maxWidth, ?int $maxHeight) + { + [$left, $top, $width, $height] = $box; + + if (!is_null($maxWidth)) { + $left = min($maxWidth - $width, $left); + // Adjust dimensions of rect if it is larger than the image. + $width = min($maxWidth, $width); + } + + if (!is_null($maxHeight)) { + $top = min($maxHeight - $height, $top); + // Adjust dimensions of rect if it is larger than the image. + $height = min($maxHeight, $height); + } + + // Order of min max is importans so the point gets no negative coordinates. + $left = max(0, $left); + $top = max(0, $top); + + return [$left, $top, $width, $height]; + } + + /** + * Get the bounding box of a point annotation. + */ + protected function getPointBoundingBox(array $points, int $padding): array + { + return [ + $points[0] - $padding, + $points[1] - $padding, + $points[0] + $padding, + $points[1] + $padding, + ]; + } + + /** + * Get the bounding box of a circle annotation. + */ + protected function getCircleBoundingBox(array $points): array + { + return [ + $points[0] - $points[2], + $points[1] - $points[2], + $points[0] + $points[2], + $points[1] + $points[2], + ]; + } + + /** + * Get the bounding box of an annotation that is no point, circle or whole frame. + */ + protected function getPolygonBoundingBox(array $points): array + { + $minX = INF; + $minY = INF; + $maxX = -INF; + $maxY = -INF; + + for ($i = 0; $i < count($points); $i += 2) { + $minX = min($minX, $points[$i]); + $minY = min($minY, $points[$i + 1]); + $maxX = max($maxX, $points[$i]); + $maxY = max($maxY, $points[$i + 1]); + } + + return [$minX, $minY, $maxX, $maxY]; + } + + /** + * Round and cast box values to int. + */ + protected function makeBoxIntegers(array $box): array + { + return array_map(fn ($v) => intval(round($v)), $box); + } +} From 33ebbeec1483a62203f31c110c287c665c8bd8c0 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Fri, 8 Dec 2023 15:37:49 +0100 Subject: [PATCH 03/37] Add ExtractFeatures script from biigle/maia --- README.md | 1 + requirements.txt | 4 ++ src/Traits/ComputesAnnotationBox.php | 1 + src/config/largo.php | 10 +++++ src/resources/scripts/ExtractFeatures.py | 53 ++++++++++++++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 requirements.txt create mode 100644 src/resources/scripts/ExtractFeatures.py diff --git a/README.md b/README.md index bf07fe49..0c88eba6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This is the BIIGLE module to review image annotations in a regular grid. This module is already included in [`biigle/biigle`](https://github.com/biigle/biigle). 1. Run `composer require biigle/largo`. +2. Install the Python dependencies with `pip install -r requirements.txt`. 2. Add `Biigle\Modules\Largo\LargoServiceProvider::class` to the `providers` array in `config/app.php`. 3. Run `php artisan vendor:publish --tag=public` to publish the public assets of this module. 4. Configure a storage disk for the Largo annotation patches and set the `LARGO_PATCH_STORAGE_DISK` variable to the name of this storage disk in the `.env` file. The content of the storage disk should be publicly accessible. Example for a local disk: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..f27801b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +Pillow==10.0.1 +torch==2.0.* +torchvision==0.15.* +xformers==0.0.18 diff --git a/src/Traits/ComputesAnnotationBox.php b/src/Traits/ComputesAnnotationBox.php index 6d46f101..a2954ec8 100644 --- a/src/Traits/ComputesAnnotationBox.php +++ b/src/Traits/ComputesAnnotationBox.php @@ -12,6 +12,7 @@ trait ComputesAnnotationBox public function getAnnotationBoundingBox( array $points, Shape $shape, + // This results in the 224x224 expected by ExtractFeatures.py. int $pointPadding = 112, int $boxPadding = 0 ): array diff --git a/src/config/largo.php b/src/config/largo.php index 35caed40..d5fba540 100644 --- a/src/config/largo.php +++ b/src/config/largo.php @@ -39,4 +39,14 @@ | Specifies which queue should be used for the job to save a Largo session. */ 'apply_session_queue' => env('LARGO_APPLY_SESSION_QUEUE', 'default'), + + /* + | Path to the extract features script. + */ + 'extract_features_script' => __DIR__.'/../resources/scripts/ExtractFeatures.py', + + /* + | Path to the directory to use as Torch Hub cache. + */ + 'torch_hub_path' => storage_path('largo_cache'), ]; diff --git a/src/resources/scripts/ExtractFeatures.py b/src/resources/scripts/ExtractFeatures.py new file mode 100644 index 00000000..7a55540e --- /dev/null +++ b/src/resources/scripts/ExtractFeatures.py @@ -0,0 +1,53 @@ +from PIL import Image +import csv +import json +import numpy as np +import sys +import torch +import torchvision.transforms as T + +# input_json = { +# cached_filename: { +# annotation_model_id: [left, top, right, bottom], +# }, +# } + +with open(sys.argv[1], 'r') as f: + input_json = json.load(f) + +if torch.cuda.is_available(): + device = torch.device('cuda') + dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14').cuda() +else: + device = torch.device('cpu') + dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14') + +dinov2_vits14.to(device) + +transform = T.Compose([ + T.Resize((224, 224), interpolation=T.InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), +]) + +def normalize_image_mode(image): + if image.mode == 'RGBA' or image.mode == 'L' or image.mode == 'P': + image = image.convert('RGB') + + if image.mode =='I': + # I images (32 bit signed integer) need to be rescaled manually before converting. + image = Image.fromarray(((np.array(image)/(2**16))*2**8).astype(np.uint8)).convert('RGB') + + return image + +with open(sys.argv[2], 'w') as f: + writer = csv.writer(f) + with torch.no_grad(): + for image_path, annotations in input_json.items(): + image = Image.open(image_path) + image = normalize_image_mode(image) + for model_id, box in annotations.items(): + image_crop = image.crop(box) + image_crop_t = transform(image_crop).unsqueeze(0).to(device) + features = dinov2_vits14(image_crop_t) + writer.writerow([model_id, json.dumps(features[0].tolist())]) From 4c1addcaebdc63231e8f95d1f08f9764f7b80d11 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Fri, 15 Dec 2023 15:23:12 +0100 Subject: [PATCH 04/37] Implement feature vector models --- ...ageAnnotationLabelFeatureVectorFactory.php | 38 ++++++ ...deoAnnotationLabelFeatureVectorFactory.php | 38 ++++++ ...eate_annotation_feature_vectors_tables.php | 118 ++++++++++++++++++ src/ImageAnnotationLabelFeatureVector.php | 53 ++++++++ src/LargoServiceProvider.php | 1 + src/VideoAnnotationLabelFeatureVector.php | 53 ++++++++ .../ImageAnnotationLabelFeatureVectorTest.php | 61 +++++++++ .../VideoAnnotationLabelFeatureVectorTest.php | 61 +++++++++ 8 files changed, 423 insertions(+) create mode 100644 src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php create mode 100644 src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php create mode 100644 src/Database/migrations/2023_12_12_090800_create_annotation_feature_vectors_tables.php create mode 100644 src/ImageAnnotationLabelFeatureVector.php create mode 100644 src/VideoAnnotationLabelFeatureVector.php create mode 100644 tests/ImageAnnotationLabelFeatureVectorTest.php create mode 100644 tests/VideoAnnotationLabelFeatureVectorTest.php diff --git a/src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php b/src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php new file mode 100644 index 00000000..e32ae067 --- /dev/null +++ b/src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php @@ -0,0 +1,38 @@ + ImageAnnotationLabel::factory(), + 'annotation_id' => ImageAnnotation::factory(), + 'label_id' => Label::factory(), + 'label_tree_id' => LabelTree::factory(), + 'volume_id' => Volume::factory(), + 'vector' => range(0, 383), + ]; + } +} diff --git a/src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php b/src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php new file mode 100644 index 00000000..4e80e588 --- /dev/null +++ b/src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php @@ -0,0 +1,38 @@ + VideoAnnotationLabel::factory(), + 'annotation_id' => VideoAnnotation::factory(), + 'label_id' => Label::factory(), + 'label_tree_id' => LabelTree::factory(), + 'volume_id' => Volume::factory(), + 'vector' => range(0, 383), + ]; + } +} diff --git a/src/Database/migrations/2023_12_12_090800_create_annotation_feature_vectors_tables.php b/src/Database/migrations/2023_12_12_090800_create_annotation_feature_vectors_tables.php new file mode 100644 index 00000000..c43c176d --- /dev/null +++ b/src/Database/migrations/2023_12_12_090800_create_annotation_feature_vectors_tables.php @@ -0,0 +1,118 @@ +unsignedBigInteger('id'); + $table->foreign('id') + ->references('id') + ->on('image_annotation_labels') + ->onDelete('cascade'); + + $table->foreignId('annotation_id') + ->constrained(table: 'image_annotations') + ->onDelete('cascade'); + + $table->foreignId('label_id') + ->constrained() + ->onDelete('restrict'); + + // This is added to be used by LabelBOT in the future. + // We still have to think of an efficient indexing strategy so indexes + // are added later. + $table->foreignId('label_tree_id') + ->constrained() + ->onDelete('restrict'); + + $table->foreignId('volume_id') + ->constrained() + ->onDelete('cascade'); + + $table->vector('vector', 384); + + // For Largo queries. + $table->index(['label_id', 'volume_id']); + }); + + Schema::create('video_annotation_label_feature_vectors', function (Blueprint $table) { + $table->unsignedBigInteger('id'); + $table->foreign('id') + ->references('id') + ->on('video_annotation_labels') + ->onDelete('cascade'); + + $table->foreignId('annotation_id') + ->constrained(table: 'video_annotations') + ->onDelete('cascade'); + + $table->foreignId('label_id') + ->constrained() + ->onDelete('restrict'); + + // This is added to be used by LabelBOT in the future. + // We still have to think of an efficient indexing strategy so indexes + // are added later. + $table->foreignId('label_tree_id') + ->constrained() + ->onDelete('restrict'); + + $table->foreignId('volume_id') + ->constrained() + ->onDelete('cascade'); + + $table->vector('vector', 384); + + // For Largo queries. + $table->index(['label_id', 'volume_id']); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::connection('pgvector') + ->dropIfExists('image_annotation_label_feature_vectors'); + Schema::connection('pgvector') + ->dropIfExists('video_annotation_label_feature_vectors'); + } +}; diff --git a/src/ImageAnnotationLabelFeatureVector.php b/src/ImageAnnotationLabelFeatureVector.php new file mode 100644 index 00000000..a673d08a --- /dev/null +++ b/src/ImageAnnotationLabelFeatureVector.php @@ -0,0 +1,53 @@ + Vector::class, + ]; + + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'id', + 'annotation_id', + 'label_id', + 'label_tree_id', + 'volume_id', + 'vector', + ]; + + /** + * Create a new factory instance for the model. + * + * @return \Illuminate\Database\Eloquent\Factories\Factory + */ + protected static function newFactory() + { + return ImageAnnotationLabelFeatureVectorFactory::new(); + } +} diff --git a/src/LargoServiceProvider.php b/src/LargoServiceProvider.php index 5d86ace7..320e8eb3 100644 --- a/src/LargoServiceProvider.php +++ b/src/LargoServiceProvider.php @@ -28,6 +28,7 @@ class LargoServiceProvider extends ServiceProvider public function boot(Modules $modules, Router $router) { $this->loadViewsFrom(__DIR__.'/resources/views', 'largo'); + $this->loadMigrationsFrom(__DIR__.'/Database/migrations'); $this->publishes([ __DIR__.'/public/assets' => public_path('vendor/largo'), diff --git a/src/VideoAnnotationLabelFeatureVector.php b/src/VideoAnnotationLabelFeatureVector.php new file mode 100644 index 00000000..4509501d --- /dev/null +++ b/src/VideoAnnotationLabelFeatureVector.php @@ -0,0 +1,53 @@ + Vector::class, + ]; + + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'id', + 'annotation_id', + 'label_id', + 'label_tree_id', + 'volume_id', + 'vector', + ]; + + /** + * Create a new factory instance for the model. + * + * @return \Illuminate\Database\Eloquent\Factories\Factory + */ + protected static function newFactory() + { + return VideoAnnotationLabelFeatureVectorFactory::new(); + } +} diff --git a/tests/ImageAnnotationLabelFeatureVectorTest.php b/tests/ImageAnnotationLabelFeatureVectorTest.php new file mode 100644 index 00000000..37f256fd --- /dev/null +++ b/tests/ImageAnnotationLabelFeatureVectorTest.php @@ -0,0 +1,61 @@ +create(); + $this->assertNotNull($v->id); + $this->assertNotNull($v->annotation_id); + $this->assertNotNull($v->label_id); + $this->assertNotNull($v->label_tree_id); + $this->assertNotNull($v->volume_id); + $this->assertNotNull($v->vector); + } + + public function testDeleteAnnotationLabelCascade() + { + $v = ImageAnnotationLabelFeatureVector::factory()->create(); + ImageAnnotationLabel::where('id', $v->id)->delete(); + $this->assertNull($v->fresh()); + } + + public function testDeleteAnnotationCascade() + { + $v = ImageAnnotationLabelFeatureVector::factory()->create(); + ImageAnnotation::where('id', $v->annotation_id)->delete(); + $this->assertNull($v->fresh()); + } + + public function testDeleteLabelRestrict() + { + $v = ImageAnnotationLabelFeatureVector::factory()->create(); + $this->expectException(QueryException::class); + Label::where('id', $v->label_id)->delete(); + } + + public function testDeleteLabelTreeRestrict() + { + $v = ImageAnnotationLabelFeatureVector::factory()->create(); + $this->expectException(QueryException::class); + LabelTree::where('id', $v->label_tree_id)->delete(); + } + + public function testDeleteVolumeCascade() + { + $v = ImageAnnotationLabelFeatureVector::factory()->create(); + Volume::where('id', $v->volume_id)->delete(); + $this->assertNull($v->fresh()); + } +} diff --git a/tests/VideoAnnotationLabelFeatureVectorTest.php b/tests/VideoAnnotationLabelFeatureVectorTest.php new file mode 100644 index 00000000..450a877a --- /dev/null +++ b/tests/VideoAnnotationLabelFeatureVectorTest.php @@ -0,0 +1,61 @@ +create(); + $this->assertNotNull($v->id); + $this->assertNotNull($v->annotation_id); + $this->assertNotNull($v->label_id); + $this->assertNotNull($v->label_tree_id); + $this->assertNotNull($v->volume_id); + $this->assertNotNull($v->vector); + } + + public function testDeleteAnnotationLabelCascade() + { + $v = VideoAnnotationLabelFeatureVector::factory()->create(); + VideoAnnotationLabel::where('id', $v->id)->delete(); + $this->assertNull($v->fresh()); + } + + public function testDeleteAnnotationCascade() + { + $v = VideoAnnotationLabelFeatureVector::factory()->create(); + VideoAnnotation::where('id', $v->annotation_id)->delete(); + $this->assertNull($v->fresh()); + } + + public function testDeleteLabelRestrict() + { + $v = VideoAnnotationLabelFeatureVector::factory()->create(); + $this->expectException(QueryException::class); + Label::where('id', $v->label_id)->delete(); + } + + public function testDeleteLabelTreeRestrict() + { + $v = VideoAnnotationLabelFeatureVector::factory()->create(); + $this->expectException(QueryException::class); + LabelTree::where('id', $v->label_tree_id)->delete(); + } + + public function testDeleteVolumeCascade() + { + $v = VideoAnnotationLabelFeatureVector::factory()->create(); + Volume::where('id', $v->volume_id)->delete(); + $this->assertNull($v->fresh()); + } +} From fb7348054188b66eb611cb810ee37f3f19227585 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Fri, 15 Dec 2023 15:34:18 +0100 Subject: [PATCH 05/37] Fix migration timestamp The migration to create the vector extension had a later timestamp but it needs to be executed before. --- ...023_12_15_153400_create_annotation_feature_vectors_tables.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Database/migrations/{2023_12_12_090800_create_annotation_feature_vectors_tables.php => 2023_12_15_153400_create_annotation_feature_vectors_tables.php} (100%) diff --git a/src/Database/migrations/2023_12_12_090800_create_annotation_feature_vectors_tables.php b/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php similarity index 100% rename from src/Database/migrations/2023_12_12_090800_create_annotation_feature_vectors_tables.php rename to src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php From 0896365b17744b3f36fa9f6a2dd4a787f7719923 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 20 Dec 2023 13:10:39 +0100 Subject: [PATCH 06/37] Implement GenerateFeatureVectors job --- src/Jobs/GenerateFeatureVectors.php | 97 +++++++++++++++++++++++++++++ src/config/largo.php | 5 ++ 2 files changed, 102 insertions(+) create mode 100644 src/Jobs/GenerateFeatureVectors.php diff --git a/src/Jobs/GenerateFeatureVectors.php b/src/Jobs/GenerateFeatureVectors.php new file mode 100644 index 00000000..74d26c45 --- /dev/null +++ b/src/Jobs/GenerateFeatureVectors.php @@ -0,0 +1,97 @@ +groupBy('image_id'); + $input = []; + + foreach ($images as $index => $image) { + $path = $paths[$index]; + $imageAnnotations = $annotations[$image->id]; + $boxes = []; + foreach ($imageAnnotations as $a) { + $box = $this->getAnnotationBoundingBox($a->points, $a->shape, self::POINT_PADDING); + $box = $this->makeBoxContained($box, $image->width, $image->height); + $zeroSize = $box[2] === 0 && $box[3] === 0; + + if (!$zeroSize) { + // Convert width and height to "right" and "bottom" coordinates. + $box[2] = $box[0] + $box[2]; + $box[3] = $box[1] + $box[3]; + + $boxes[$a->id] = $box; + } + } + + if (!empty($boxes)) { + $input[$path] = $boxes; + } + } + + return $input; + } + + /** + * Run the Python command. + * + * @param string $command + */ + protected function python(array $input, string $outputPath) + { + $python = config('largo.python'); + $script = config('largo.extract_features_script'); + $inputPath = tempnam(sys_get_temp_dir(), 'largo_feature_vector_input'); + File::put($inputPath, json_encode($input)); + try { + $result = Process::forever() + ->env(['TORCH_HOME' => config('largo.torch_hub_path')]) + ->run("{$python} -u {$script} {$inputPath} {$outputPath}") + ->throw(); + } finally { + File::delete($inputPath); + } + } + + /** + * Generator to read the output CSV row by row. + */ + protected function readOuputCsv(string $path): \Generator + { + $file = new SplFileObject($path); + while (!$file->eof()) { + $csv = $file->fgetcsv(); + if (count($csv) === 2) { + yield $csv; + } + } + } +} diff --git a/src/config/largo.php b/src/config/largo.php index d5fba540..fea01eed 100644 --- a/src/config/largo.php +++ b/src/config/largo.php @@ -49,4 +49,9 @@ | Path to the directory to use as Torch Hub cache. */ 'torch_hub_path' => storage_path('largo_cache'), + + /* + | Path to the Python executable. + */ + 'python' => env('LARGO_PYTHON', '/usr/bin/python3'), ]; From 27ce50c2ff99edc52e631b5c8b9051d024036a06 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 20 Dec 2023 16:28:56 +0100 Subject: [PATCH 07/37] Implement generating feature vectors in GenerateAnnotationPatch jobs --- src/Jobs/GenerateAnnotationPatch.php | 71 +++++- src/Jobs/GenerateFeatureVectors.php | 229 +++++++++++++++--- src/Jobs/GenerateImageAnnotationPatch.php | 23 +- src/Jobs/GenerateVideoAnnotationPatch.php | 33 ++- src/Traits/ComputesAnnotationBox.php | 162 ------------- .../Jobs/GenerateImageAnnotationPatchTest.php | 87 +++++++ ...p => GenerateVideoAnnotationPatchTest.php} | 91 ++++++- 7 files changed, 491 insertions(+), 205 deletions(-) delete mode 100644 src/Traits/ComputesAnnotationBox.php rename tests/Jobs/{GenerateVideoAnnotationPatchesTest.php => GenerateVideoAnnotationPatchTest.php} (73%) diff --git a/src/Jobs/GenerateAnnotationPatch.php b/src/Jobs/GenerateAnnotationPatch.php index 3f14bd6c..281434c1 100644 --- a/src/Jobs/GenerateAnnotationPatch.php +++ b/src/Jobs/GenerateAnnotationPatch.php @@ -4,7 +4,6 @@ use Biigle\Contracts\Annotation; use Biigle\FileCache\Exceptions\FileLockedException; -use Biigle\Jobs\Job; use Biigle\Modules\Largo\Traits\ComputesAnnotationBox; use Biigle\Shape; use Biigle\VideoAnnotation; @@ -12,16 +11,18 @@ use Exception; use FileCache; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Jcupitt\Vips\Image; -use Log; +use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Log; use Str; +use Jcupitt\Vips\Image; -abstract class GenerateAnnotationPatch extends Job implements ShouldQueue +abstract class GenerateAnnotationPatch extends GenerateFeatureVectors { - use SerializesModels, InteractsWithQueue, ComputesAnnotationBox; + use SerializesModels, InteractsWithQueue; /** * The number of times the job may be attempted. @@ -178,4 +179,64 @@ protected function getAnnotationPatch($image, $points, $shape) 'strip' => true, ]); } + + /** + * Generates a feature vector for the annotation of this job and either creates a new + * feature vector model or updates the existing ones for the annotation. + */ + protected function generateFeatureVector(VolumeFile $file, string $path): void + { + $boxes = $this->generateFileInput($file, collect([$this->annotation])); + if (empty($boxes)) { + return; + } + + // shm is available because this will only be executed in a Docker container. + // The files are small here so this should be a fast way for communication. + // Input/output files are used to also allow larger use cases with thousands of + // feature vectors (e.g. in biigle/maia). + $inputPath = tempnam('/dev/shm', 'largo_feature_vector_input'); + $outputPath = tempnam('/dev/shm', 'largo_feature_vector_output'); + + try { + File::put($inputPath, json_encode([$path => $boxes])); + $this->python($inputPath, $outputPath); + $output = $this->readOuputCsv($outputPath)->current(); + + $shouldUpdate = $this->getFeatureVectorQuery()->exists(); + if ($shouldUpdate) { + $this->getFeatureVectorQuery()->update(['vector' => $output[1]]); + } else { + $annotationLabel = $this->annotation + ->labels() + ->orderBy('id', 'asc') + ->with('label') + ->first(); + + if (!is_null($annotationLabel)) { + $this->createFeatureVector([ + 'id' => $annotationLabel->id, + 'annotation_id' => $this->annotation->id, + 'label_id' => $annotationLabel->label_id, + 'label_tree_id' => $annotationLabel->label->label_tree_id, + 'volume_id' => $file->volume_id, + 'vector' => $output[1], + ]); + } + } + } finally { + File::delete($outputPath); + File::delete($inputPath); + } + } + + /** + * Get a query for the feature vectors associated with the annotation of this job. + */ + abstract protected function getFeatureVectorQuery(): Builder; + + /** + * Create a new feature vector model for the annotation of this job. + */ + abstract protected function createFeatureVector(array $attributes): void; } diff --git a/src/Jobs/GenerateFeatureVectors.php b/src/Jobs/GenerateFeatureVectors.php index 74d26c45..7d683886 100644 --- a/src/Jobs/GenerateFeatureVectors.php +++ b/src/Jobs/GenerateFeatureVectors.php @@ -3,18 +3,17 @@ namespace Biigle\Modules\Largo\Jobs; use Biigle\Jobs\Job; -use Biigle\Modules\Largo\Traits\ComputesAnnotationBox; +use Biigle\Shape; +use Biigle\VideoAnnotation; +use Biigle\VolumeFile; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Process; use SplFileObject; abstract class GenerateFeatureVectors extends Job implements ShouldQueue { - use ComputesAnnotationBox; - /** * The "radius" of the bounding box around a point annotation. * @@ -24,33 +23,177 @@ abstract class GenerateFeatureVectors extends Job implements ShouldQueue */ const POINT_PADDING = 112; + /** + * Get the bounding box of an annotation + */ + public function getAnnotationBoundingBox( + array $points, + Shape $shape, + // This results in the 224x224 expected by ExtractFeatures.py. + int $pointPadding = 112, + int $boxPadding = 0 + ): array + { + $box = match ($shape->id) { + Shape::pointId() => $this->getPointBoundingBox($points, $pointPadding), + Shape::circleId() => $this->getCircleBoundingBox($points), + // An ellipse will not be handled correctly by this but I didn't bother + // because this shape is almost never used anyway. + default => $this->getPolygonBoundingBox($points), + }; + + if ($boxPadding > 0) { + $box = [ + $box[0] - $boxPadding, + $box[1] - $boxPadding, + $box[2] + $boxPadding, + $box[3] + $boxPadding, + ]; + } + + return $this->makeBoxIntegers([ + $box[0], // left + $box[1], // top + $box[2] - $box[0], // width + $box[3] - $box[1], // height + ]); + } + + /** + * Modify a bounding box so it adheres to the aspect ratio given by width and height. + */ + public function ensureBoxAspectRatio(array $box, int $aspectWidth, int $aspectHeight): array + { + [$left, $top, $width, $height] = $box; + + // Ensure the minimum width so the annotation patch is not "zoomed in". + if ($width < $aspectWidth) { + $left -= ($aspectWidth - $width) / 2.0; + $width = $aspectWidth; + } + + // Ensure the minimum height so the annotation patch is not "zoomed in". + if ($height < $aspectHeight) { + $top -= ($aspectHeight - $height) / 2.0; + $height = $aspectHeight; + } + + $widthRatio = $width / $aspectWidth; + $heightRatio = $height / $aspectHeight; + + // Increase the size of the patch so its aspect ratio is the same than the + // ratio of the given dimensions. + if ($widthRatio > $heightRatio) { + $newHeight = round($aspectHeight * $widthRatio); + $top -= round(($newHeight - $height) / 2); + $height = $newHeight; + } else { + $newWidth = round($aspectWidth * $heightRatio); + $left -= round(($newWidth - $width) / 2); + $width = $newWidth; + } + + return $this->makeBoxIntegers([$left, $top, $width, $height]); + } + + /** + * Adjust the position and size of the box so it is contained in a box with the given + * dimensions. + */ + public function makeBoxContained(array $box, ?int $maxWidth, ?int $maxHeight) + { + [$left, $top, $width, $height] = $box; + + if (!is_null($maxWidth)) { + $left = min($maxWidth - $width, $left); + // Adjust dimensions of rect if it is larger than the image. + $width = min($maxWidth, $width); + } + + if (!is_null($maxHeight)) { + $top = min($maxHeight - $height, $top); + // Adjust dimensions of rect if it is larger than the image. + $height = min($maxHeight, $height); + } + + // Order of min max is importans so the point gets no negative coordinates. + $left = max(0, $left); + $top = max(0, $top); + + return [$left, $top, $width, $height]; + } + + /** + * Get the bounding box of a point annotation. + */ + protected function getPointBoundingBox(array $points, int $padding): array + { + return [ + $points[0] - $padding, + $points[1] - $padding, + $points[0] + $padding, + $points[1] + $padding, + ]; + } + + /** + * Get the bounding box of a circle annotation. + */ + protected function getCircleBoundingBox(array $points): array + { + return [ + $points[0] - $points[2], + $points[1] - $points[2], + $points[0] + $points[2], + $points[1] + $points[2], + ]; + } + + /** + * Get the bounding box of an annotation that is no point, circle or whole frame. + */ + protected function getPolygonBoundingBox(array $points): array + { + $minX = INF; + $minY = INF; + $maxX = -INF; + $maxY = -INF; + + for ($i = 0; $i < count($points); $i += 2) { + $minX = min($minX, $points[$i]); + $minY = min($minY, $points[$i + 1]); + $maxX = max($maxX, $points[$i]); + $maxY = max($maxY, $points[$i + 1]); + } + + return [$minX, $minY, $maxX, $maxY]; + } + + /** + * Round and cast box values to int. + */ + protected function makeBoxIntegers(array $box): array + { + return array_map(fn ($v) => intval(round($v)), $box); + } + /** * Generate the input for the python script. * - * @param \Illuminate\Support\Collection $annotations + * @param array $files VolumeFile instances of the files to which the annotations + * belong. + * @param array $paths Paths of locally cached files. + * @param \Illuminate\Support\Collection $annotations Annotations grouped by their + * file ID (e.g. image_id). */ - protected function generateInput(array $images, array $paths, Collection $annotations): array + protected function generateInput(array $files, array $paths, Collection $annotations): array { - $annotations = $annotations->groupBy('image_id'); $input = []; - foreach ($images as $index => $image) { + foreach ($files as $index => $file) { $path = $paths[$index]; - $imageAnnotations = $annotations[$image->id]; - $boxes = []; - foreach ($imageAnnotations as $a) { - $box = $this->getAnnotationBoundingBox($a->points, $a->shape, self::POINT_PADDING); - $box = $this->makeBoxContained($box, $image->width, $image->height); - $zeroSize = $box[2] === 0 && $box[3] === 0; - - if (!$zeroSize) { - // Convert width and height to "right" and "bottom" coordinates. - $box[2] = $box[0] + $box[2]; - $box[3] = $box[1] + $box[3]; - - $boxes[$a->id] = $box; - } - } + $fileAnnotations = $annotations[$file->id]; + $boxes = $this->generateFileInput($file, $fileAnnotations); if (!empty($boxes)) { $input[$path] = $boxes; @@ -60,25 +203,43 @@ protected function generateInput(array $images, array $paths, Collection $annota return $input; } + protected function generateFileInput(VolumeFile $file, Collection $annotations): array + { + $boxes = []; + foreach ($annotations as $a) { + $points = $a->points; + if (($a instanceof VideoAnnotation) && !empty($points)) { + $points = $points[0]; + } + $box = $this->getAnnotationBoundingBox($points, $a->shape, self::POINT_PADDING); + $box = $this->makeBoxContained($box, $file->width, $file->height); + $zeroSize = $box[2] === 0 && $box[3] === 0; + + if (!$zeroSize) { + // Convert width and height to "right" and "bottom" coordinates. + $box[2] = $box[0] + $box[2]; + $box[3] = $box[1] + $box[3]; + + $boxes[$a->id] = $box; + } + } + + return $boxes; + } + /** * Run the Python command. * * @param string $command */ - protected function python(array $input, string $outputPath) + protected function python(string $inputPath, string $outputPath) { $python = config('largo.python'); $script = config('largo.extract_features_script'); - $inputPath = tempnam(sys_get_temp_dir(), 'largo_feature_vector_input'); - File::put($inputPath, json_encode($input)); - try { - $result = Process::forever() - ->env(['TORCH_HOME' => config('largo.torch_hub_path')]) - ->run("{$python} -u {$script} {$inputPath} {$outputPath}") - ->throw(); - } finally { - File::delete($inputPath); - } + $result = Process::forever() + ->env(['TORCH_HOME' => config('largo.torch_hub_path')]) + ->run("{$python} -u {$script} {$inputPath} {$outputPath}") + ->throw(); } /** diff --git a/src/Jobs/GenerateImageAnnotationPatch.php b/src/Jobs/GenerateImageAnnotationPatch.php index 2c456330..0aa47558 100644 --- a/src/Jobs/GenerateImageAnnotationPatch.php +++ b/src/Jobs/GenerateImageAnnotationPatch.php @@ -2,11 +2,14 @@ namespace Biigle\Modules\Largo\Jobs; +use Biigle\Modules\Largo\ImageAnnotationLabelFeatureVector; use Biigle\Shape; use Biigle\VolumeFile; use Exception; use FileCache; -use Storage; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Storage; use VipsImage; class GenerateImageAnnotationPatch extends GenerateAnnotationPatch @@ -28,6 +31,24 @@ public function handleFile(VolumeFile $file, $path) $buffer = $this->getAnnotationPatch($image, $this->annotation->getPoints(), $this->annotation->getShape()); Storage::disk($this->targetDisk)->put($targetPath, $buffer); + + $this->generateFeatureVector($file, $path); + } + + /** + * Get a query for the feature vectors associated with the annotation of this job. + */ + protected function getFeatureVectorQuery(): Builder + { + return ImageAnnotationLabelFeatureVector::where('annotation_id', $this->annotation->id); + } + + /** + * Create a new feature vector model for the annotation of this job. + */ + protected function createFeatureVector(array $attributes): void + { + ImageAnnotationLabelFeatureVector::create($attributes); } /** diff --git a/src/Jobs/GenerateVideoAnnotationPatch.php b/src/Jobs/GenerateVideoAnnotationPatch.php index df0ec368..353aac17 100644 --- a/src/Jobs/GenerateVideoAnnotationPatch.php +++ b/src/Jobs/GenerateVideoAnnotationPatch.php @@ -2,11 +2,14 @@ namespace Biigle\Modules\Largo\Jobs; +use Biigle\Modules\Largo\VideoAnnotationLabelFeatureVector; use Biigle\VolumeFile; use FFMpeg\Coordinate\TimeCode; use FFMpeg\FFMpeg; use FFMpeg\Media\Video; -use Storage; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Storage; use VipsImage; class GenerateVideoAnnotationPatch extends GenerateAnnotationPatch @@ -31,6 +34,32 @@ public function handleFile(VolumeFile $file, $path) $frame = $this->getVideoFrame($video, $this->annotation->frames[0]); $buffer = $this->getAnnotationPatch($frame, $points, $this->annotation->shape); Storage::disk($this->targetDisk)->put($targetPath, $buffer); + + + $framePath = tempnam(sys_get_temp_dir(), 'largo_video_frame').'.png'; + + try { + $frame->writeToFile($framePath); + $this->generateFeatureVector($file, $framePath); + } finally { + File::delete($framePath); + } + } + + /** + * Get a query for the feature vectors associated with the annotation of this job. + */ + protected function getFeatureVectorQuery(): Builder + { + return VideoAnnotationLabelFeatureVector::where('annotation_id', $this->annotation->id); + } + + /** + * Create a new feature vector model for the annotation of this job. + */ + protected function createFeatureVector(array $attributes): void + { + VideoAnnotationLabelFeatureVector::create($attributes); } /** @@ -51,7 +80,7 @@ protected function getVideo($path) * @param Video $video * @param float $time * - * @return \Jcupitt\Vips\Image + * @return \Jcupitt\Vips\Video */ protected function getVideoFrame(Video $video, $time) { diff --git a/src/Traits/ComputesAnnotationBox.php b/src/Traits/ComputesAnnotationBox.php deleted file mode 100644 index a2954ec8..00000000 --- a/src/Traits/ComputesAnnotationBox.php +++ /dev/null @@ -1,162 +0,0 @@ -id) { - Shape::pointId() => $this->getPointBoundingBox($points, $pointPadding), - Shape::circleId() => $this->getCircleBoundingBox($points), - // An ellipse will not be handled correctly by this but I didn't bother - // because this shape is almost never used anyway. - default => $this->getPolygonBoundingBox($points), - }; - - if ($boxPadding > 0) { - $box = [ - $box[0] - $boxPadding, - $box[1] - $boxPadding, - $box[2] + $boxPadding, - $box[3] + $boxPadding, - ]; - } - - return $this->makeBoxIntegers([ - $box[0], // left - $box[1], // top - $box[2] - $box[0], // width - $box[3] - $box[1], // height - ]); - } - - /** - * Modify a bounding box so it adheres to the aspect ratio given by width and height. - */ - public function ensureBoxAspectRatio(array $box, int $aspectWidth, int $aspectHeight): array - { - [$left, $top, $width, $height] = $box; - - // Ensure the minimum width so the annotation patch is not "zoomed in". - if ($width < $aspectWidth) { - $left -= ($aspectWidth - $width) / 2.0; - $width = $aspectWidth; - } - - // Ensure the minimum height so the annotation patch is not "zoomed in". - if ($height < $aspectHeight) { - $top -= ($aspectHeight - $height) / 2.0; - $height = $aspectHeight; - } - - $widthRatio = $width / $aspectWidth; - $heightRatio = $height / $aspectHeight; - - // Increase the size of the patch so its aspect ratio is the same than the - // ratio of the given dimensions. - if ($widthRatio > $heightRatio) { - $newHeight = round($aspectHeight * $widthRatio); - $top -= round(($newHeight - $height) / 2); - $height = $newHeight; - } else { - $newWidth = round($aspectWidth * $heightRatio); - $left -= round(($newWidth - $width) / 2); - $width = $newWidth; - } - - return $this->makeBoxIntegers([$left, $top, $width, $height]); - } - - /** - * Adjust the position and size of the box so it is contained in a box with the given - * dimensions. - */ - public function makeBoxContained(array $box, ?int $maxWidth, ?int $maxHeight) - { - [$left, $top, $width, $height] = $box; - - if (!is_null($maxWidth)) { - $left = min($maxWidth - $width, $left); - // Adjust dimensions of rect if it is larger than the image. - $width = min($maxWidth, $width); - } - - if (!is_null($maxHeight)) { - $top = min($maxHeight - $height, $top); - // Adjust dimensions of rect if it is larger than the image. - $height = min($maxHeight, $height); - } - - // Order of min max is importans so the point gets no negative coordinates. - $left = max(0, $left); - $top = max(0, $top); - - return [$left, $top, $width, $height]; - } - - /** - * Get the bounding box of a point annotation. - */ - protected function getPointBoundingBox(array $points, int $padding): array - { - return [ - $points[0] - $padding, - $points[1] - $padding, - $points[0] + $padding, - $points[1] + $padding, - ]; - } - - /** - * Get the bounding box of a circle annotation. - */ - protected function getCircleBoundingBox(array $points): array - { - return [ - $points[0] - $points[2], - $points[1] - $points[2], - $points[0] + $points[2], - $points[1] + $points[2], - ]; - } - - /** - * Get the bounding box of an annotation that is no point, circle or whole frame. - */ - protected function getPolygonBoundingBox(array $points): array - { - $minX = INF; - $minY = INF; - $maxX = -INF; - $maxY = -INF; - - for ($i = 0; $i < count($points); $i += 2) { - $minX = min($minX, $points[$i]); - $minY = min($minY, $points[$i + 1]); - $maxX = max($maxX, $points[$i]); - $maxY = max($maxY, $points[$i + 1]); - } - - return [$minX, $minY, $maxX, $maxY]; - } - - /** - * Round and cast box values to int. - */ - protected function makeBoxIntegers(array $box): array - { - return array_map(fn ($v) => intval(round($v)), $box); - } -} diff --git a/tests/Jobs/GenerateImageAnnotationPatchTest.php b/tests/Jobs/GenerateImageAnnotationPatchTest.php index fd9af7a0..80213a66 100644 --- a/tests/Jobs/GenerateImageAnnotationPatchTest.php +++ b/tests/Jobs/GenerateImageAnnotationPatchTest.php @@ -3,8 +3,10 @@ namespace Biigle\Tests\Modules\Largo\Jobs; use Biigle\FileCache\Exceptions\FileLockedException; +use Biigle\Modules\Largo\ImageAnnotationLabelFeatureVector; use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; use Biigle\Shape; +use Biigle\Tests\ImageAnnotationLabelTest; use Biigle\Tests\ImageAnnotationTest; use Bus; use Exception; @@ -290,6 +292,79 @@ public function testFileLockedError() Bus::assertDispatched(GenerateImageAnnotationPatch::class); } + public function testGenerateFeatureVectorNew() + { + Storage::fake('test'); + $image = $this->getImageMock(); + $image->shouldReceive('crop')->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + $annotation = ImageAnnotationTest::create([ + 'points' => [200, 200], + 'shape_id' => Shape::pointId(), + ]); + $annotationLabel = ImageAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $job = new GenerateImageAnnotationPatchStub($annotation); + $job->mock = $image; + $job->output = [[$annotation->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($annotation->image, 'abc'); + + $input = $job->input; + $this->assertCount(1, $input); + $filename = array_keys($input)[0]; + $this->assertArrayHasKey($annotation->id, $input[$filename]); + $box = $input[$filename][$annotation->id]; + $this->assertEquals([88, 88, 312, 312], $box); + + $vectors = ImageAnnotationLabelFeatureVector::where('annotation_id', $annotation->id)->get(); + $this->assertCount(1, $vectors); + $this->assertEquals($annotationLabel->id, $vectors[0]->id); + $this->assertEquals($annotationLabel->label_id, $vectors[0]->label_id); + $this->assertEquals($annotationLabel->label->label_tree_id, $vectors[0]->label_tree_id); + $this->assertEquals($annotation->image->volume_id, $vectors[0]->volume_id); + $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); + } + + public function testGenerateFeatureVectorUpdate() + { + Storage::fake('test'); + $image = $this->getImageMock(); + $image->shouldReceive('crop')->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + $annotation = ImageAnnotationTest::create([ + 'points' => [200, 200], + 'shape_id' => Shape::pointId(), + ]); + $annotationLabel = ImageAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $iafv = ImageAnnotationLabelFeatureVector::factory()->create([ + 'id' => $annotationLabel->id, + 'annotation_id' => $annotation->id, + 'vector' => range(0, 383), + ]); + + $annotationLabel2 = ImageAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $iafv2 = ImageAnnotationLabelFeatureVector::factory()->create([ + 'id' => $annotationLabel2->id, + 'annotation_id' => $annotation->id, + 'vector' => range(0, 383), + ]); + + $job = new GenerateImageAnnotationPatchStub($annotation); + $job->mock = $image; + $job->output = [[$annotation->id, '"'.json_encode(range(1, 384)).'"']]; + $job->handleFile($annotation->image, 'abc'); + + $count = ImageAnnotationLabelFeatureVector::count(); + $this->assertEquals(2, $count); + $this->assertEquals(range(1, 384), $iafv->fresh()->vector->toArray()); + $this->assertEquals(range(1, 384), $iafv2->fresh()->vector->toArray()); + } + protected function getImageMock($times = 1) { $image = Mockery::mock(); @@ -305,8 +380,20 @@ protected function getImageMock($times = 1) class GenerateImageAnnotationPatchStub extends GenerateImageAnnotationPatch { + public $input; + public $outputPath; + public $output = []; + public function getVipsImage($path) { return $this->mock; } + + protected function python(string $inputPath, string $outputPath) + { + $this->input = json_decode(File::get($inputPath), true); + $this->outputPath = $outputPath; + $csv = implode("\n", array_map(fn ($row) => implode(',', $row), $this->output)); + File::put($outputPath, $csv); + } } diff --git a/tests/Jobs/GenerateVideoAnnotationPatchesTest.php b/tests/Jobs/GenerateVideoAnnotationPatchTest.php similarity index 73% rename from tests/Jobs/GenerateVideoAnnotationPatchesTest.php rename to tests/Jobs/GenerateVideoAnnotationPatchTest.php index 3dd1c9cd..3776f39a 100644 --- a/tests/Jobs/GenerateVideoAnnotationPatchesTest.php +++ b/tests/Jobs/GenerateVideoAnnotationPatchTest.php @@ -4,7 +4,9 @@ use Biigle\FileCache\Exceptions\FileLockedException; use Biigle\Modules\Largo\Jobs\GenerateVideoAnnotationPatch; +use Biigle\Modules\Largo\VideoAnnotationLabelFeatureVector; use Biigle\Shape; +use Biigle\Tests\VideoAnnotationLabelTest; use Biigle\Tests\VideoAnnotationTest; use Biigle\VideoAnnotation; use Bus; @@ -17,7 +19,7 @@ use Storage; use TestCase; -class GenerateVideoAnnotationPatchesTest extends TestCase +class GenerateVideoAnnotationPatchTest extends TestCase { public function setUp(): void { @@ -311,6 +313,81 @@ public function testFileLockedError() Bus::assertDispatched(GenerateVideoAnnotationPatch::class); } + public function testGenerateFeatureVectorNew() + { + Storage::fake('test'); + $video = $this->getFrameMock(); + $video->shouldReceive('crop')->andReturn($video); + $video->shouldReceive('writeToBuffer')->andReturn('abc123'); + $annotation = VideoAnnotationTest::create([ + 'points' => [[200, 200]], + 'frames' => [1], + 'shape_id' => Shape::pointId(), + ]); + $annotationLabel = VideoAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $job = new GenerateVideoAnnotationPatchStub($annotation); + $job->mock = $video; + $job->output = [[$annotation->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($annotation->video, 'abc'); + + $input = $job->input; + $this->assertCount(1, $input); + $filename = array_keys($input)[0]; + $this->assertArrayHasKey($annotation->id, $input[$filename]); + $box = $input[$filename][$annotation->id]; + $this->assertEquals([88, 88, 312, 312], $box); + + $vectors = VideoAnnotationLabelFeatureVector::where('annotation_id', $annotation->id)->get(); + $this->assertCount(1, $vectors); + $this->assertEquals($annotationLabel->id, $vectors[0]->id); + $this->assertEquals($annotationLabel->label_id, $vectors[0]->label_id); + $this->assertEquals($annotationLabel->label->label_tree_id, $vectors[0]->label_tree_id); + $this->assertEquals($annotation->video->volume_id, $vectors[0]->volume_id); + $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); + } + + public function testGenerateFeatureVectorUpdate() + { + Storage::fake('test'); + $video = $this->getFrameMock(); + $video->shouldReceive('crop')->andReturn($video); + $video->shouldReceive('writeToBuffer')->andReturn('abc123'); + $annotation = VideoAnnotationTest::create([ + 'points' => [[200, 200]], + 'frames' => [1], + 'shape_id' => Shape::pointId(), + ]); + $annotationLabel = VideoAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $iafv = VideoAnnotationLabelFeatureVector::factory()->create([ + 'id' => $annotationLabel->id, + 'annotation_id' => $annotation->id, + 'vector' => range(0, 383), + ]); + + $annotationLabel2 = VideoAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $iafv2 = VideoAnnotationLabelFeatureVector::factory()->create([ + 'id' => $annotationLabel2->id, + 'annotation_id' => $annotation->id, + 'vector' => range(0, 383), + ]); + + $job = new GenerateVideoAnnotationPatchStub($annotation); + $job->mock = $video; + $job->output = [[$annotation->id, '"'.json_encode(range(1, 384)).'"']]; + $job->handleFile($annotation->video, 'abc'); + + $count = VideoAnnotationLabelFeatureVector::count(); + $this->assertEquals(2, $count); + $this->assertEquals(range(1, 384), $iafv->fresh()->vector->toArray()); + $this->assertEquals(range(1, 384), $iafv2->fresh()->vector->toArray()); + } + protected function getFrameMock($times = 1) { $video = Mockery::mock(); @@ -319,6 +396,7 @@ protected function getFrameMock($times = 1) $video->shouldReceive('resize') ->times($times) ->andReturn($video); + $video->shouldReceive('writeToFile'); return $video; } @@ -327,6 +405,9 @@ protected function getFrameMock($times = 1) class GenerateVideoAnnotationPatchStub extends GenerateVideoAnnotationPatch { public $times = []; + public $input; + public $outputPath; + public $output = []; public function getVideo($path) { @@ -339,4 +420,12 @@ public function getVideoFrame(Video $video, $time) return $this->mock; } + + protected function python(string $inputPath, string $outputPath) + { + $this->input = json_decode(File::get($inputPath), true); + $this->outputPath = $outputPath; + $csv = implode("\n", array_map(fn ($row) => implode(',', $row), $this->output)); + File::put($outputPath, $csv); + } } From c1df26f050bde46941ac348e20eebef86f988725 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Thu, 4 Jan 2024 15:30:16 +0100 Subject: [PATCH 08/37] Generate new feature vectors for all annotation labels instead of first --- src/Jobs/GenerateAnnotationPatch.php | 6 ++-- .../Jobs/GenerateImageAnnotationPatchTest.php | 32 ++++++++++++++++++ .../Jobs/GenerateVideoAnnotationPatchTest.php | 33 +++++++++++++++++++ 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/Jobs/GenerateAnnotationPatch.php b/src/Jobs/GenerateAnnotationPatch.php index 281434c1..e594b9ad 100644 --- a/src/Jobs/GenerateAnnotationPatch.php +++ b/src/Jobs/GenerateAnnotationPatch.php @@ -207,13 +207,13 @@ protected function generateFeatureVector(VolumeFile $file, string $path): void if ($shouldUpdate) { $this->getFeatureVectorQuery()->update(['vector' => $output[1]]); } else { - $annotationLabel = $this->annotation + $annotationLabels = $this->annotation ->labels() ->orderBy('id', 'asc') ->with('label') - ->first(); + ->get(); - if (!is_null($annotationLabel)) { + foreach ($annotationLabels as $annotationLabel) { $this->createFeatureVector([ 'id' => $annotationLabel->id, 'annotation_id' => $this->annotation->id, diff --git a/tests/Jobs/GenerateImageAnnotationPatchTest.php b/tests/Jobs/GenerateImageAnnotationPatchTest.php index 80213a66..e7f78a19 100644 --- a/tests/Jobs/GenerateImageAnnotationPatchTest.php +++ b/tests/Jobs/GenerateImageAnnotationPatchTest.php @@ -326,6 +326,38 @@ public function testGenerateFeatureVectorNew() $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); } + public function testGenerateFeatureVectorManyLabels() + { + Storage::fake('test'); + $image = $this->getImageMock(); + $image->shouldReceive('crop')->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + $annotation = ImageAnnotationTest::create([ + 'points' => [200, 200], + 'shape_id' => Shape::pointId(), + ]); + $annotationLabel1 = ImageAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $annotationLabel2 = ImageAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $job = new GenerateImageAnnotationPatchStub($annotation); + $job->mock = $image; + $job->output = [[$annotation->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($annotation->image, 'abc'); + + $vectors = ImageAnnotationLabelFeatureVector::where('annotation_id', $annotation->id)->get(); + $this->assertCount(2, $vectors); + $this->assertEquals($annotationLabel1->id, $vectors[0]->id); + $this->assertEquals($annotationLabel1->label_id, $vectors[0]->label_id); + $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); + + $this->assertEquals($annotationLabel2->id, $vectors[1]->id); + $this->assertEquals($annotationLabel2->label_id, $vectors[1]->label_id); + $this->assertEquals(range(0, 383), $vectors[1]->vector->toArray()); + } + public function testGenerateFeatureVectorUpdate() { Storage::fake('test'); diff --git a/tests/Jobs/GenerateVideoAnnotationPatchTest.php b/tests/Jobs/GenerateVideoAnnotationPatchTest.php index 3776f39a..47f6c0ee 100644 --- a/tests/Jobs/GenerateVideoAnnotationPatchTest.php +++ b/tests/Jobs/GenerateVideoAnnotationPatchTest.php @@ -348,6 +348,39 @@ public function testGenerateFeatureVectorNew() $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); } + public function testGenerateFeatureVectorManyLabels() + { + Storage::fake('test'); + $video = $this->getFrameMock(); + $video->shouldReceive('crop')->andReturn($video); + $video->shouldReceive('writeToBuffer')->andReturn('abc123'); + $annotation = VideoAnnotationTest::create([ + 'points' => [[200, 200]], + 'frames' => [1], + 'shape_id' => Shape::pointId(), + ]); + $annotationLabel1 = VideoAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $annotationLabel2 = VideoAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $job = new GenerateVideoAnnotationPatchStub($annotation); + $job->mock = $video; + $job->output = [[$annotation->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($annotation->video, 'abc'); + + $vectors = VideoAnnotationLabelFeatureVector::where('annotation_id', $annotation->id)->get(); + $this->assertCount(2, $vectors); + $this->assertEquals($annotationLabel1->id, $vectors[0]->id); + $this->assertEquals($annotationLabel1->label_id, $vectors[0]->label_id); + $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); + + $this->assertEquals($annotationLabel2->id, $vectors[1]->id); + $this->assertEquals($annotationLabel2->label_id, $vectors[1]->label_id); + $this->assertEquals(range(0, 383), $vectors[1]->vector->toArray()); + } + public function testGenerateFeatureVectorUpdate() { Storage::fake('test'); From bc57c1ef04e6421f5d2aa96c61a43783923fcae4 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Thu, 4 Jan 2024 16:07:51 +0100 Subject: [PATCH 09/37] Implement annotation label observers to copy feature vectors --- src/Jobs/CopyAnnotationFeatureVector.php | 59 +++++++++++++++++++ src/Jobs/CopyImageAnnotationFeatureVector.php | 28 +++++++++ src/Jobs/CopyVideoAnnotationFeatureVector.php | 28 +++++++++ src/LargoServiceProvider.php | 6 ++ .../ImageAnnotationLabelObserver.php | 17 ++++++ .../VideoAnnotationLabelObserver.php | 17 ++++++ .../CopyImageAnnotationFeatureVectorTest.php | 31 ++++++++++ .../CopyVideoAnnotationFeatureVectorTest.php | 31 ++++++++++ .../ImageAnnotationLabelObserverTest.php | 17 ++++++ .../VideoAnnotationLabelObserverTest.php | 17 ++++++ 10 files changed, 251 insertions(+) create mode 100644 src/Jobs/CopyAnnotationFeatureVector.php create mode 100644 src/Jobs/CopyImageAnnotationFeatureVector.php create mode 100644 src/Jobs/CopyVideoAnnotationFeatureVector.php create mode 100644 src/Observers/ImageAnnotationLabelObserver.php create mode 100644 src/Observers/VideoAnnotationLabelObserver.php create mode 100644 tests/Jobs/CopyImageAnnotationFeatureVectorTest.php create mode 100644 tests/Jobs/CopyVideoAnnotationFeatureVectorTest.php create mode 100644 tests/Observers/ImageAnnotationLabelObserverTest.php create mode 100644 tests/Observers/VideoAnnotationLabelObserverTest.php diff --git a/src/Jobs/CopyAnnotationFeatureVector.php b/src/Jobs/CopyAnnotationFeatureVector.php new file mode 100644 index 00000000..90f3a3bf --- /dev/null +++ b/src/Jobs/CopyAnnotationFeatureVector.php @@ -0,0 +1,59 @@ +getFeatureVectorQuery()->first(); + $this->updateOrCreateFeatureVector([ + 'id' => $this->annotationLabel->id, + 'annotation_id' => $this->annotationLabel->annotation_id, + 'label_id' => $this->annotationLabel->label_id, + 'label_tree_id' => $this->annotationLabel->label->label_tree_id, + 'volume_id' => $vector->volume_id, + 'vector' => $vector->vector, + ]); + } + + /** + * Get a query for the feature vectors associated with the annotation of this job. + */ + abstract protected function getFeatureVectorQuery(): Builder; + + /** + * Create a new feature vector model for the annotation of this job. + */ + abstract protected function updateOrCreateFeatureVector(array $attributes): void; +} diff --git a/src/Jobs/CopyImageAnnotationFeatureVector.php b/src/Jobs/CopyImageAnnotationFeatureVector.php new file mode 100644 index 00000000..2a5c0c31 --- /dev/null +++ b/src/Jobs/CopyImageAnnotationFeatureVector.php @@ -0,0 +1,28 @@ +annotationLabel->annotation_id); + } + + + /** + * {@inheritdoc} + */ + protected function updateOrCreateFeatureVector(array $attributes): void + { + $idArray = ['id' => $attributes['id']]; + unset($attributes['id']); + ImageAnnotationLabelFeatureVector::updateOrCreate($idArray, $attributes); + } +} diff --git a/src/Jobs/CopyVideoAnnotationFeatureVector.php b/src/Jobs/CopyVideoAnnotationFeatureVector.php new file mode 100644 index 00000000..21c9964b --- /dev/null +++ b/src/Jobs/CopyVideoAnnotationFeatureVector.php @@ -0,0 +1,28 @@ +annotationLabel->annotation_id); + } + + + /** + * {@inheritdoc} + */ + protected function updateOrCreateFeatureVector(array $attributes): void + { + $idArray = ['id' => $attributes['id']]; + unset($attributes['id']); + VideoAnnotationLabelFeatureVector::updateOrCreate($idArray, $attributes); + } +} diff --git a/src/LargoServiceProvider.php b/src/LargoServiceProvider.php index 320e8eb3..ba79ba71 100644 --- a/src/LargoServiceProvider.php +++ b/src/LargoServiceProvider.php @@ -5,12 +5,16 @@ use Biigle\Events\ImagesDeleted; use Biigle\Events\VideosDeleted; use Biigle\ImageAnnotation; +use Biigle\ImageAnnotationLabel; use Biigle\Modules\Largo\Listeners\ImagesCleanupListener; use Biigle\Modules\Largo\Listeners\VideosCleanupListener; use Biigle\Modules\Largo\Observers\ImageAnnotationObserver; +use Biigle\Modules\Largo\Observers\ImageAnnotationLabelObserver; use Biigle\Modules\Largo\Observers\VideoAnnotationObserver; +use Biigle\Modules\Largo\Observers\VideoAnnotationLabelObserver; use Biigle\Services\Modules; use Biigle\VideoAnnotation; +use Biigle\VideoAnnotationLabel; use Event; use Illuminate\Routing\Router; use Illuminate\Support\ServiceProvider; @@ -46,7 +50,9 @@ public function boot(Modules $modules, Router $router) }); ImageAnnotation::observe(new ImageAnnotationObserver); + ImageAnnotationLabel::observe(new ImageAnnotationLabelObserver); VideoAnnotation::observe(new VideoAnnotationObserver); + VideoAnnotationLabel::observe(new VideoAnnotationLabelObserver); Event::listen(ImagesDeleted::class, ImagesCleanupListener::class); Event::listen(VideosDeleted::class, VideosCleanupListener::class); diff --git a/src/Observers/ImageAnnotationLabelObserver.php b/src/Observers/ImageAnnotationLabelObserver.php new file mode 100644 index 00000000..fb38fd9d --- /dev/null +++ b/src/Observers/ImageAnnotationLabelObserver.php @@ -0,0 +1,17 @@ +create(); + $annotationLabel = ImageAnnotationLabel::factory()->create([ + 'annotation_id' => $vector->annotation_id, + ]); + + (new CopyImageAnnotationFeatureVector($annotationLabel))->handle(); + + $vectors = ImageAnnotationLabelFeatureVector::where('annotation_id', $vector->annotation_id) + ->orderBy('id', 'asc')->get(); + $this->assertCount(2, $vectors); + $this->assertEquals($annotationLabel->id, $vectors[1]->id); + $this->assertEquals($annotationLabel->annotation_id, $vectors[1]->annotation_id); + $this->assertEquals($annotationLabel->label_id, $vectors[1]->label_id); + $this->assertEquals($annotationLabel->label->label_tree_id, $vectors[1]->label_tree_id); + $this->assertEquals($vector->volume_id, $vectors[1]->volume_id); + $this->assertEquals($vector->vector, $vectors[1]->vector); + } +} diff --git a/tests/Jobs/CopyVideoAnnotationFeatureVectorTest.php b/tests/Jobs/CopyVideoAnnotationFeatureVectorTest.php new file mode 100644 index 00000000..bc35a521 --- /dev/null +++ b/tests/Jobs/CopyVideoAnnotationFeatureVectorTest.php @@ -0,0 +1,31 @@ +create(); + $annotationLabel = VideoAnnotationLabel::factory()->create([ + 'annotation_id' => $vector->annotation_id, + ]); + + (new CopyVideoAnnotationFeatureVector($annotationLabel))->handle(); + + $vectors = VideoAnnotationLabelFeatureVector::where('annotation_id', $vector->annotation_id) + ->orderBy('id', 'asc')->get(); + $this->assertCount(2, $vectors); + $this->assertEquals($annotationLabel->id, $vectors[1]->id); + $this->assertEquals($annotationLabel->annotation_id, $vectors[1]->annotation_id); + $this->assertEquals($annotationLabel->label_id, $vectors[1]->label_id); + $this->assertEquals($annotationLabel->label->label_tree_id, $vectors[1]->label_tree_id); + $this->assertEquals($vector->volume_id, $vectors[1]->volume_id); + $this->assertEquals($vector->vector, $vectors[1]->vector); + } +} diff --git a/tests/Observers/ImageAnnotationLabelObserverTest.php b/tests/Observers/ImageAnnotationLabelObserverTest.php new file mode 100644 index 00000000..7d08454e --- /dev/null +++ b/tests/Observers/ImageAnnotationLabelObserverTest.php @@ -0,0 +1,17 @@ + Date: Thu, 4 Jan 2024 16:14:07 +0100 Subject: [PATCH 10/37] Update/fix feature vector migration --- ...reate_annotation_feature_vectors_tables.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php b/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php index c43c176d..0740b94e 100644 --- a/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php +++ b/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php @@ -68,6 +68,12 @@ public function up() // For Largo queries. $table->index(['label_id', 'volume_id']); + + // For create/update queries. + $table->index('annotation_id'); + + // Ensure consistency and speed up updateOrCreate queries. + $table->primary('id'); }); Schema::create('video_annotation_label_feature_vectors', function (Blueprint $table) { @@ -100,6 +106,12 @@ public function up() // For Largo queries. $table->index(['label_id', 'volume_id']); + + // For create/update queries. + $table->index('annotation_id'); + + // Ensure consistency and speed up updateOrCreate queries. + $table->primary('id'); }); } @@ -110,9 +122,7 @@ public function up() */ public function down() { - Schema::connection('pgvector') - ->dropIfExists('image_annotation_label_feature_vectors'); - Schema::connection('pgvector') - ->dropIfExists('video_annotation_label_feature_vectors'); + Schema::dropIfExists('image_annotation_label_feature_vectors'); + Schema::dropIfExists('video_annotation_label_feature_vectors'); } }; From 271d0dd715356278a172b62bb3924b3af68df5a2 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Thu, 4 Jan 2024 16:57:21 +0100 Subject: [PATCH 11/37] Copy feature vectors when a Largo job is applied --- src/Jobs/ApplyLargoSession.php | 28 +++++++++++++-- src/Jobs/CopyAnnotationFeatureVector.php | 18 +++++----- tests/Jobs/ApplyLargoSessionTest.php | 46 ++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/Jobs/ApplyLargoSession.php b/src/Jobs/ApplyLargoSession.php index b404bee4..3da07a6f 100644 --- a/src/Jobs/ApplyLargoSession.php +++ b/src/Jobs/ApplyLargoSession.php @@ -8,6 +8,8 @@ use Biigle\Label; use Biigle\Modules\Largo\Events\LargoSessionFailed; use Biigle\Modules\Largo\Events\LargoSessionSaved; +use Biigle\Modules\Largo\Jobs\CopyImageAnnotationFeatureVector; +use Biigle\Modules\Largo\Jobs\CopyVideoAnnotationFeatureVector; use Biigle\Modules\Largo\Jobs\RemoveImageAnnotationPatches; use Biigle\Modules\Largo\Jobs\RemoveVideoAnnotationPatches; use Biigle\User; @@ -144,8 +146,11 @@ protected function handleImageAnnotations() { [$dismissed, $changed] = $this->ignoreDeletedLabels($this->dismissedImageAnnotations, $this->changedImageAnnotations); - $this->applyDismissedLabels($this->user, $dismissed, $this->force, ImageAnnotationLabel::class); + // Change labels first, then dismiss to keep the opportunity to copy feature + // vectors. If labels are deleted first, the feature vectors will be immediately + // deleted, too, and nothing can be copied any more. $this->applyChangedLabels($this->user, $changed, ImageAnnotation::class, ImageAnnotationLabel::class); + $this->applyDismissedLabels($this->user, $dismissed, $this->force, ImageAnnotationLabel::class); $this->deleteDanglingAnnotations($dismissed, $changed, ImageAnnotation::class); } @@ -156,8 +161,11 @@ protected function handleVideoAnnotations() { [$dismissed, $changed] = $this->ignoreDeletedLabels($this->dismissedVideoAnnotations, $this->changedVideoAnnotations); - $this->applyDismissedLabels($this->user, $dismissed, $this->force, VideoAnnotationLabel::class); + // Change labels first, then dismiss to keep the opportunity to copy feature + // vectors. If labels are deleted first, the feature vectors will be immediately + // deleted, too, and nothing can be copied any more. $this->applyChangedLabels($this->user, $changed, VideoAnnotation::class, VideoAnnotationLabel::class); + $this->applyDismissedLabels($this->user, $dismissed, $this->force, VideoAnnotationLabel::class); $this->deleteDanglingAnnotations($dismissed, $changed, VideoAnnotation::class); } @@ -286,6 +294,10 @@ protected function applyChangedLabels($user, $changed, $annotationModel, $labelM } } + // Store the ID so we can efficiently loop over the models later. This is only + // possible because all this happens inside a DB transaction (see above). + $startId = $labelModel::orderBy('id', 'desc')->select('id')->first()->id; + collect($newAnnotationLabels) ->when($labelModel === ImageAnnotationLabel::class, function ($labels) { return $labels->map(function ($item) { @@ -301,6 +313,18 @@ protected function applyChangedLabels($user, $changed, $annotationModel, $labelM ->each(function ($chunk) use ($labelModel) { $labelModel::insert($chunk->toArray()); }); + + $labelModel::where('id', '>', $startId) + ->eachById(function ($annotationLabel) use ($labelModel) { + // Execute the jobs synchronously because after this method the + // old annotation labels may be deleted and there will be nothing + // to copy any more. + if ($labelModel === ImageAnnotationLabel::class) { + (new CopyImageAnnotationFeatureVector($annotationLabel))->handle(); + } else { + (new CopyVideoAnnotationFeatureVector($annotationLabel))->handle(); + } + }); } /** diff --git a/src/Jobs/CopyAnnotationFeatureVector.php b/src/Jobs/CopyAnnotationFeatureVector.php index 90f3a3bf..5559a87b 100644 --- a/src/Jobs/CopyAnnotationFeatureVector.php +++ b/src/Jobs/CopyAnnotationFeatureVector.php @@ -37,14 +37,16 @@ public function __construct(public AnnotationLabel $annotationLabel) public function handle() { $vector = $this->getFeatureVectorQuery()->first(); - $this->updateOrCreateFeatureVector([ - 'id' => $this->annotationLabel->id, - 'annotation_id' => $this->annotationLabel->annotation_id, - 'label_id' => $this->annotationLabel->label_id, - 'label_tree_id' => $this->annotationLabel->label->label_tree_id, - 'volume_id' => $vector->volume_id, - 'vector' => $vector->vector, - ]); + if (!is_null($vector)) { + $this->updateOrCreateFeatureVector([ + 'id' => $this->annotationLabel->id, + 'annotation_id' => $this->annotationLabel->annotation_id, + 'label_id' => $this->annotationLabel->label_id, + 'label_tree_id' => $this->annotationLabel->label->label_tree_id, + 'volume_id' => $vector->volume_id, + 'vector' => $vector->vector, + ]); + } } /** diff --git a/tests/Jobs/ApplyLargoSessionTest.php b/tests/Jobs/ApplyLargoSessionTest.php index 7e855b34..1041e5f3 100644 --- a/tests/Jobs/ApplyLargoSessionTest.php +++ b/tests/Jobs/ApplyLargoSessionTest.php @@ -4,9 +4,11 @@ use Biigle\Modules\Largo\Events\LargoSessionFailed; use Biigle\Modules\Largo\Events\LargoSessionSaved; +use Biigle\Modules\Largo\ImageAnnotationLabelFeatureVector; use Biigle\Modules\Largo\Jobs\ApplyLargoSession; use Biigle\Modules\Largo\Jobs\RemoveImageAnnotationPatches; use Biigle\Modules\Largo\Jobs\RemoveVideoAnnotationPatches; +use Biigle\Modules\Largo\VideoAnnotationLabelFeatureVector; use Biigle\Tests\ImageAnnotationLabelTest; use Biigle\Tests\ImageAnnotationTest; use Biigle\Tests\ImageTest; @@ -626,6 +628,50 @@ public function testDispatchEventOnError() return true; }); } + + public function testChangeImageAnnotationCopyFeatureVector() + { + $user = UserTest::create(); + $al1 = ImageAnnotationLabelTest::create(['user_id' => $user->id]); + $vector1 = ImageAnnotationLabelFeatureVector::factory()->create([ + 'id' => $al1->id, + 'annotation_id' => $al1->annotation_id, + ]); + $l1 = LabelTest::create(); + + $dismissed = [$al1->label_id => [$al1->annotation_id]]; + $changed = [$l1->id => [$al1->annotation_id]]; + $job = new ApplyLargoSession('job_id', $user, $dismissed, $changed, [], [], false); + $job->handle(); + + $vectors = ImageAnnotationLabelFeatureVector::where('annotation_id', $al1->annotation_id)->get(); + $this->assertCount(1, $vectors); + $this->assertNotEquals($al1->id, $vectors[0]->id); + $this->assertEquals($l1->id, $vectors[0]->label_id); + $this->assertEquals($vector1->vector, $vectors[0]->vector); + } + + public function testChangeVideoAnnotationCopyFeatureVector() + { + $user = UserTest::create(); + $al1 = VideoAnnotationLabelTest::create(['user_id' => $user->id]); + $vector1 = VideoAnnotationLabelFeatureVector::factory()->create([ + 'id' => $al1->id, + 'annotation_id' => $al1->annotation_id, + ]); + $l1 = LabelTest::create(); + + $dismissed = [$al1->label_id => [$al1->annotation_id]]; + $changed = [$l1->id => [$al1->annotation_id]]; + $job = new ApplyLargoSession('job_id', $user, [], [], $dismissed, $changed, false); + $job->handle(); + + $vectors = VideoAnnotationLabelFeatureVector::where('annotation_id', $al1->annotation_id)->get(); + $this->assertCount(1, $vectors); + $this->assertNotEquals($al1->id, $vectors[0]->id); + $this->assertEquals($l1->id, $vectors[0]->label_id); + $this->assertEquals($vector1->vector, $vectors[0]->vector); + } } class ApplyLargoSessionStub extends ApplyLargoSession From f458f4b7fe6178eccfc63bd4bc455d370ffad7f9 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Fri, 5 Jan 2024 11:47:38 +0100 Subject: [PATCH 12/37] Handle feature vector generation of whole frame annotations --- src/Jobs/GenerateFeatureVectors.php | 15 ++++++--- .../Jobs/GenerateVideoAnnotationPatchTest.php | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/Jobs/GenerateFeatureVectors.php b/src/Jobs/GenerateFeatureVectors.php index 7d683886..07fa3ed1 100644 --- a/src/Jobs/GenerateFeatureVectors.php +++ b/src/Jobs/GenerateFeatureVectors.php @@ -207,12 +207,17 @@ protected function generateFileInput(VolumeFile $file, Collection $annotations): { $boxes = []; foreach ($annotations as $a) { - $points = $a->points; - if (($a instanceof VideoAnnotation) && !empty($points)) { - $points = $points[0]; + if ($a->shape_id === Shape::wholeFrameId()) { + $box = [0, 0, $file->width ?: 0, $file->height ?: 0]; + } else { + $points = $a->points; + if (($a instanceof VideoAnnotation) && !empty($points)) { + $points = $points[0]; + } + $box = $this->getAnnotationBoundingBox($points, $a->shape, self::POINT_PADDING); + $box = $this->makeBoxContained($box, $file->width, $file->height); } - $box = $this->getAnnotationBoundingBox($points, $a->shape, self::POINT_PADDING); - $box = $this->makeBoxContained($box, $file->width, $file->height); + $zeroSize = $box[2] === 0 && $box[3] === 0; if (!$zeroSize) { diff --git a/tests/Jobs/GenerateVideoAnnotationPatchTest.php b/tests/Jobs/GenerateVideoAnnotationPatchTest.php index 47f6c0ee..de132f26 100644 --- a/tests/Jobs/GenerateVideoAnnotationPatchTest.php +++ b/tests/Jobs/GenerateVideoAnnotationPatchTest.php @@ -8,6 +8,7 @@ use Biigle\Shape; use Biigle\Tests\VideoAnnotationLabelTest; use Biigle\Tests\VideoAnnotationTest; +use Biigle\Video as VideoModel; use Biigle\VideoAnnotation; use Bus; use Exception; @@ -421,6 +422,38 @@ public function testGenerateFeatureVectorUpdate() $this->assertEquals(range(1, 384), $iafv2->fresh()->vector->toArray()); } + public function testGenerateFeatureVectorWholeFrame() + { + Storage::fake('test'); + $videoMock = $this->getFrameMock(); + $videoMock->shouldReceive('crop')->andReturn($videoMock); + $videoMock->shouldReceive('writeToBuffer')->andReturn('abc123'); + + $video = VideoModel::factory()->create([ + 'attrs' => ['width' => 1000, 'height' => 750], + ]); + $annotation = VideoAnnotationTest::create([ + 'points' => [], + 'frames' => [1], + 'shape_id' => Shape::wholeFrameId(), + 'video_id' => $video->id, + ]); + $annotationLabel = VideoAnnotationLabelTest::create([ + 'annotation_id' => $annotation->id, + ]); + $job = new GenerateVideoAnnotationPatchStub($annotation); + $job->mock = $videoMock; + $job->output = [[$annotation->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($video, 'abc'); + + $input = $job->input; + $this->assertCount(1, $input); + $filename = array_keys($input)[0]; + $this->assertArrayHasKey($annotation->id, $input[$filename]); + $box = $input[$filename][$annotation->id]; + $this->assertEquals([0, 0, 1000, 750], $box); + } + protected function getFrameMock($times = 1) { $video = Mockery::mock(); From 5098614ada1c0d223578537f4ffd785a9a113767 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Tue, 9 Jan 2024 16:49:45 +0100 Subject: [PATCH 13/37] Add update schema action --- .github/workflows/update-schema.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/update-schema.yaml diff --git a/.github/workflows/update-schema.yaml b/.github/workflows/update-schema.yaml new file mode 100644 index 00000000..7110dea2 --- /dev/null +++ b/.github/workflows/update-schema.yaml @@ -0,0 +1,22 @@ +name: Update Schema + +on: + push: + branches: + - master + paths: + - 'src/Database/migrations/**' + +jobs: + update-schema: + + runs-on: ubuntu-latest + + steps: + - name: Trigger schema update + run: | + curl -X POST --fail \ + -H "Authorization: token ${{ secrets.BIIGLE_SCHEMA_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + --data '{"event_type": "build_application"}' \ + https://api.github.com/repos/biigle/schema/dispatches From 16b3328a8aa904e01503587f062789f410831850 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 10 Jan 2024 16:58:51 +0100 Subject: [PATCH 14/37] WIP Start implementing sorting tab with sort by ID and outlier --- src/public/assets/scripts/main.js | 2 +- src/public/assets/styles/main.css | 2 +- src/public/mix-manifest.json | 4 +- .../assets/js/components/sortingTab.vue | 118 ++++++++++++++++++ .../assets/js/mixins/largoContainer.vue | 34 +++++ src/resources/assets/sass/main.scss | 2 +- src/resources/views/show/content.blade.php | 16 ++- 7 files changed, 168 insertions(+), 10 deletions(-) create mode 100644 src/resources/assets/js/components/sortingTab.vue diff --git a/src/public/assets/scripts/main.js b/src/public/assets/scripts/main.js index b3209b3d..b7d164c5 100644 --- a/src/public/assets/scripts/main.js +++ b/src/public/assets/scripts/main.js @@ -1 +1 @@ -(()=>{"use strict";var e,t={732:()=>{var e="imageAnnotation",t="videoAnnotation";function n(e,t,n,i,s,o,a,l){var r,u="function"==typeof e?e.options:e;if(t&&(u.render=t,u.staticRenderFns=n,u._compiled=!0),i&&(u.functional=!0),o&&(u._scopeId="data-v-"+o),a?(r=function(e){(e=e||this.$vnode&&this.$vnode.ssrContext||this.parent&&this.parent.$vnode&&this.parent.$vnode.ssrContext)||"undefined"==typeof __VUE_SSR_CONTEXT__||(e=__VUE_SSR_CONTEXT__),s&&s.call(this,e),e&&e._registeredComponents&&e._registeredComponents.add(a)},u._ssrRegister=r):s&&(r=l?function(){s.call(this,(u.functional?this.parent:this).$root.$options.shadowRoot)}:s),r)if(u.functional){u._injectStyles=r;var d=u.render;u.render=function(e,t){return r.call(t),d(e,t)}}else{var h=u.beforeCreate;u.beforeCreate=h?[].concat(h,r):[r]}return{exports:e,options:u}}const i=n({computed:{id:function(){return this.image.id},uuid:function(){return this.image.uuid},type:function(){return this.image.type},patchPrefix:function(){return this.uuid[0]+this.uuid[1]+"/"+this.uuid[2]+this.uuid[3]+"/"+this.uuid},urlTemplate:function(){return biigle.$require("largo.patchUrlTemplate")}},methods:{getThumbnailUrl:function(){return this.type===t?this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id","v-".concat(this.id)):this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id",this.id)}},created:function(){this.type===e?this.showAnnotationRoute=biigle.$require("largo.showImageAnnotationRoute"):this.showAnnotationRoute=biigle.$require("largo.showVideoAnnotationRoute")}},undefined,undefined,!1,null,null,null).exports;const s=n({mixins:[i],props:{_id:{type:String,required:!0},_uuid:{type:String,required:!0},label:{type:Object,required:!0},emptySrc:{type:String,required:!0},_urlTemplate:{type:String,required:!0}},data:function(){return{url:""}},computed:{title:function(){return"Example annotation for label "+this.label.name},src:function(){return this.url||this.emptySrc},image:function(){return{id:this._id,uuid:this._uuid,type:e}},urlTemplate:function(){return this._urlTemplate}},methods:{showEmptyImage:function(){this.url=""}},created:function(){this.url=this.getThumbnailUrl()}},undefined,undefined,!1,null,null,null).exports,o=Vue.resource("api/v1/volumes{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/volumes{/id}/video-annotations/filter/label{/label_id}"},queryExampleAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/examples{/label_id}"}});var a=biigle.$require("echo"),l=biigle.$require("events"),r=biigle.$require("messages").handleErrorResponse,u=biigle.$require("volumes.components.imageGrid"),d=biigle.$require("volumes.components.imageGridImage"),h=biigle.$require("annotations.components.labelsTabPlugins"),c=biigle.$require("labelTrees.components.labelTrees"),m=biigle.$require("core.mixins.loader"),f=biigle.$require("messages"),g=biigle.$require("core.components.powerToggle"),p=biigle.$require("annotations.components.settingsTabPlugins"),b=biigle.$require("core.components.sidebar"),v=biigle.$require("core.components.sidebarTab");const w=n({mixins:[m],components:{annotationPatch:s},props:{label:{default:null},volumeId:{type:Number,required:!0},count:{type:Number,default:3}},data:function(){return{exampleLabel:null,exampleAnnotations:[],cache:{},shown:!0}},computed:{isShown:function(){return this.shown&&null!==this.label},hasExamples:function(){return this.exampleLabel&&this.exampleAnnotations&&Object.keys(this.exampleAnnotations).length>0}},methods:{parseResponse:function(e){return e.data},setExampleAnnotations:function(e){(!e[0].hasOwnProperty("annotations")||Object.keys(e[0].annotations).length0},dismissedImageAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(t){return t.type===e})))},dismissedVideoAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(e){return e.type===t})))},changedImageAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(t){return t.type===e})))},changedVideoAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(e){return e.type===t})))},toDeleteCount:function(){return this.dismissedAnnotations.length-this.annotationsWithNewLabel.length},saveButtonClass:function(){return this.forceChange?"btn-danger":"btn-success"}},methods:{getAnnotations:function(e){var t=this;this.annotationsCache.hasOwnProperty(e.id)||(Vue.set(this.annotationsCache,e.id,[]),this.startLoading(),this.queryAnnotations(e).then((function(n){return t.gotAnnotations(e,n)}),r).finally(this.finishLoading))},gotAnnotations:function(n,i){var s=i[0].data,o=i[1].data,a=[];s&&(a=a.concat(this.initAnnotations(n,s,e))),o&&(a=a.concat(this.initAnnotations(n,o,t))),a=a.sort((function(e,t){return t.id-e.id})),Vue.set(this.annotationsCache,n.id,a)},initAnnotations:function(e,t,n){return Object.keys(t).map((function(i){return{id:i,uuid:t[i],label_id:e.id,dismissed:!1,newLabel:null,type:n}}))},handleSelectedLabel:function(e){this.selectedLabel=e,this.isInDismissStep&&this.getAnnotations(e)},handleDeselectedLabel:function(){this.selectedLabel=null},handleSelectedImageDismiss:function(e,t){e.dismissed?(e.dismissed=!1,e.newLabel=null):(e.dismissed=!0,t.shiftKey&&this.lastSelectedImage?this.dismissAllImagesBetween(e,this.lastSelectedImage):this.lastSelectedImage=e)},goToRelabel:function(){this.step=1,this.lastSelectedImage=null},goToDismiss:function(){this.step=0,this.lastSelectedImage=null,this.selectedLabel&&this.getAnnotations(this.selectedLabel)},handleSelectedImageRelabel:function(e,t){e.newLabel?this.selectedLabel&&e.newLabel.id!==this.selectedLabel.id?e.newLabel=this.selectedLabel:e.newLabel=null:this.selectedLabel&&(e.newLabel=this.selectedLabel,t.shiftKey&&this.lastSelectedImage?this.relabelAllImagesBetween(e,this.lastSelectedImage):this.lastSelectedImage=e)},save:function(){var e=this;if(!this.loading){if(this.toDeleteCount>0){for(var t;null!==t&&parseInt(t,10)!==this.toDeleteCount;)t=prompt("This might delete ".concat(this.toDeleteCount," annotation(s). Please enter the number to continue."));if(null===t)return}this.startLoading(),this.performSave({dismissed_image_annotations:this.dismissedImageAnnotationsToSave,changed_image_annotations:this.changedImageAnnotationsToSave,dismissed_video_annotations:this.dismissedVideoAnnotationsToSave,changed_video_annotations:this.changedVideoAnnotationsToSave,force:this.forceChange}).then((function(t){return e.waitForSessionId=t.body.id}),(function(t){e.finishLoading(),r(t)}))}},handleSessionSaved:function(e){if(e.id==this.waitForSessionId){for(var t in this.finishLoading(),f.success("Saved. You can now start a new re-evaluation session."),this.step=0,this.annotationsCache)this.annotationsCache.hasOwnProperty(t)&&delete this.annotationsCache[t];this.handleSelectedLabel(this.selectedLabel)}},handleSessionFailed:function(e){e.id==this.waitForSessionId&&(this.finishLoading(),f.danger("There was an unexpected error."))},performOnAllImagesBetween:function(e,t,n){var i=this.allAnnotations.indexOf(e),s=this.allAnnotations.indexOf(t);if(s=0;n--)t.hasOwnProperty(e[n].label_id)?t[e[n].label_id].push(e[n].id):t[e[n].label_id]=[e[n].id];return t},packChangedToSave:function(e){for(var t={},n=e.length-1;n>=0;n--)t.hasOwnProperty(e[n].newLabel.id)?t[e[n].newLabel.id].push(e[n].id):t[e[n].newLabel.id]=[e[n].id];return t},initializeEcho:function(){a.getInstance().private("user-".concat(this.user.id)).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionSaved",this.handleSessionSaved).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionFailed",this.handleSessionFailed)}},watch:{annotations:function(e){l.$emit("annotations-count",e.length)},dismissedAnnotations:function(e){l.$emit("dismissed-annotations-count",e.length)},step:function(e){l.$emit("step",e)},selectedLabel:function(){this.isInDismissStep&&this.$refs.dismissGrid.setOffset(0)}},created:function(){var e=this;this.user=biigle.$require("largo.user"),window.addEventListener("beforeunload",(function(t){if(e.hasDismissedAnnotations)return t.preventDefault(),t.returnValue="","This page is asking you to confirm that you want to leave - data you have entered may not be saved."})),this.initializeEcho()}},undefined,undefined,!1,null,null,null).exports;const q=n({mixins:[T],components:{catalogImageGrid:y},data:function(){return{labelTrees:[]}},methods:{queryAnnotations:function(e){var t=S.queryImageAnnotations({id:e.id}),n=S.queryVideoAnnotations({id:e.id});return Vue.Promise.all([t,n])}},created:function(){var e=biigle.$require("annotationCatalog.labelTree");this.labelTrees=[e]}},undefined,undefined,!1,null,null,null).exports;const $=n({mixins:[T],data:function(){return{volumeId:null,labelTrees:[],mediaType:""}},methods:{queryAnnotations:function(e){var t,n;return"image"===this.mediaType?(t=o.queryImageAnnotations({id:this.volumeId,label_id:e.id}),n=Vue.Promise.resolve([])):(t=Vue.Promise.resolve([]),n=o.queryVideoAnnotations({id:this.volumeId,label_id:e.id})),Vue.Promise.all([t,n])},performSave:function(e){return o.save({id:this.volumeId},e)}},created:function(){this.volumeId=biigle.$require("largo.volumeId"),this.labelTrees=biigle.$require("largo.labelTrees"),this.mediaType=biigle.$require("largo.mediaType")}},undefined,undefined,!1,null,null,null).exports;const O=n({data:function(){return{step:0,count:0,dismissedCount:0}},computed:{shownCount:function(){return this.isInDismissStep?this.count:this.dismissedCount},isInDismissStep:function(){return 0===this.step},isInRelabelStep:function(){return 1===this.step}},methods:{updateStep:function(e){this.step=e},updateCount:function(e){this.count=e},updateDismissedCount:function(e){this.dismissedCount=e}},created:function(){l.$on("annotations-count",this.updateCount),l.$on("dismissed-annotations-count",this.updateDismissedCount),l.$on("step",this.updateStep)}},undefined,undefined,!1,null,null,null).exports,E=Vue.resource("api/v1/projects{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/projects{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/projects{/id}/video-annotations/filter/label{/label_id}"}});const k=n({mixins:[T],data:function(){return{projectId:null,labelTrees:[]}},methods:{queryAnnotations:function(e){var t=E.queryImageAnnotations({id:this.projectId,label_id:e.id}),n=E.queryVideoAnnotations({id:this.projectId,label_id:e.id});return Vue.Promise.all([t,n])},performSave:function(e){return E.save({id:this.projectId},e)}},created:function(){this.projectId=biigle.$require("largo.projectId"),this.labelTrees=biigle.$require("largo.labelTrees")}},undefined,undefined,!1,null,null,null).exports;biigle.$mount("annotation-catalog-container",q),biigle.$mount("largo-container",$),biigle.$mount("largo-title",O),biigle.$mount("project-largo-container",k)},401:()=>{}},n={};function i(e){var s=n[e];if(void 0!==s)return s.exports;var o=n[e]={exports:{}};return t[e](o,o.exports,i),o.exports}i.m=t,e=[],i.O=(t,n,s,o)=>{if(!n){var a=1/0;for(d=0;d=o)&&Object.keys(i.O).every((e=>i.O[e](n[r])))?n.splice(r--,1):(l=!1,o0&&e[d-1][2]>o;d--)e[d]=e[d-1];e[d]=[n,s,o]},i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),(()=>{var e={355:0,392:0};i.O.j=t=>0===e[t];var t=(t,n)=>{var s,o,[a,l,r]=n,u=0;if(a.some((t=>0!==e[t]))){for(s in l)i.o(l,s)&&(i.m[s]=l[s]);if(r)var d=r(i)}for(t&&t(n);ui(732)));var s=i.O(void 0,[392],(()=>i(401)));s=i.O(s)})(); \ No newline at end of file +(()=>{"use strict";var t,e={505:()=>{var t="imageAnnotation",e="videoAnnotation";function n(t,e,n,i,s,o,a,r){var l,u="function"==typeof t?t.options:t;if(e&&(u.render=e,u.staticRenderFns=n,u._compiled=!0),i&&(u.functional=!0),o&&(u._scopeId="data-v-"+o),a?(l=function(t){(t=t||this.$vnode&&this.$vnode.ssrContext||this.parent&&this.parent.$vnode&&this.parent.$vnode.ssrContext)||"undefined"==typeof __VUE_SSR_CONTEXT__||(t=__VUE_SSR_CONTEXT__),s&&s.call(this,t),t&&t._registeredComponents&&t._registeredComponents.add(a)},u._ssrRegister=l):s&&(l=r?function(){s.call(this,(u.functional?this.parent:this).$root.$options.shadowRoot)}:s),l)if(u.functional){u._injectStyles=l;var d=u.render;u.render=function(t,e){return l.call(e),d(t,e)}}else{var c=u.beforeCreate;u.beforeCreate=c?[].concat(c,l):[l]}return{exports:t,options:u}}const i=n({computed:{id:function(){return this.image.id},uuid:function(){return this.image.uuid},type:function(){return this.image.type},patchPrefix:function(){return this.uuid[0]+this.uuid[1]+"/"+this.uuid[2]+this.uuid[3]+"/"+this.uuid},urlTemplate:function(){return biigle.$require("largo.patchUrlTemplate")}},methods:{getThumbnailUrl:function(){return this.type===e?this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id","v-".concat(this.id)):this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id",this.id)}},created:function(){this.type===t?this.showAnnotationRoute=biigle.$require("largo.showImageAnnotationRoute"):this.showAnnotationRoute=biigle.$require("largo.showVideoAnnotationRoute")}},undefined,undefined,!1,null,null,null).exports;const s=n({mixins:[i],props:{_id:{type:String,required:!0},_uuid:{type:String,required:!0},label:{type:Object,required:!0},emptySrc:{type:String,required:!0},_urlTemplate:{type:String,required:!0}},data:function(){return{url:""}},computed:{title:function(){return"Example annotation for label "+this.label.name},src:function(){return this.url||this.emptySrc},image:function(){return{id:this._id,uuid:this._uuid,type:t}},urlTemplate:function(){return this._urlTemplate}},methods:{showEmptyImage:function(){this.url=""}},created:function(){this.url=this.getThumbnailUrl()}},undefined,undefined,!1,null,null,null).exports,o=Vue.resource("api/v1/volumes{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/volumes{/id}/video-annotations/filter/label{/label_id}"},queryExampleAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/examples{/label_id}"}});var a=biigle.$require("echo"),r=biigle.$require("events"),l=biigle.$require("messages").handleErrorResponse,u=biigle.$require("volumes.components.imageGrid"),d=biigle.$require("volumes.components.imageGridImage"),c=biigle.$require("annotations.components.labelsTabPlugins"),h=biigle.$require("labelTrees.components.labelTrees"),m=biigle.$require("core.mixins.loader"),g=biigle.$require("messages"),f=biigle.$require("core.components.powerToggle"),p=biigle.$require("annotations.components.settingsTabPlugins"),b=biigle.$require("core.components.sidebar"),v=biigle.$require("core.components.sidebarTab");const w=n({mixins:[m],components:{annotationPatch:s},props:{label:{default:null},volumeId:{type:Number,required:!0},count:{type:Number,default:3}},data:function(){return{exampleLabel:null,exampleAnnotations:[],cache:{},shown:!0}},computed:{isShown:function(){return this.shown&&null!==this.label},hasExamples:function(){return this.exampleLabel&&this.exampleAnnotations&&Object.keys(this.exampleAnnotations).length>0}},methods:{parseResponse:function(t){return t.data},setExampleAnnotations:function(t){(!t[0].hasOwnProperty("annotations")||Object.keys(t[0].annotations).length0},dismissedImageAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(e){return e.type===t})))},dismissedVideoAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(t){return t.type===e})))},changedImageAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(e){return e.type===t})))},changedVideoAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(t){return t.type===e})))},toDeleteCount:function(){return this.dismissedAnnotations.length-this.annotationsWithNewLabel.length},saveButtonClass:function(){return this.forceChange?"btn-danger":"btn-success"}},methods:{getAnnotations:function(t){var e=this;this.annotationsCache.hasOwnProperty(t.id)||(Vue.set(this.annotationsCache,t.id,[]),this.startLoading(),this.queryAnnotations(t).then((function(n){return e.gotAnnotations(t,n)}),l).finally(this.finishLoading))},gotAnnotations:function(n,i){var s=i[0].data,o=i[1].data,a=[];s&&(a=a.concat(this.initAnnotations(n,s,t))),o&&(a=a.concat(this.initAnnotations(n,o,e))),a=a.sort((function(t,e){return e.id-t.id})),Vue.set(this.annotationsCache,n.id,a)},initAnnotations:function(t,e,n){return Object.keys(e).map((function(i){return{id:i,uuid:e[i],label_id:t.id,dismissed:!1,newLabel:null,type:n}}))},handleSelectedLabel:function(t){this.selectedLabel=t,this.isInDismissStep&&this.getAnnotations(t)},handleDeselectedLabel:function(){this.selectedLabel=null},handleSelectedImageDismiss:function(t,e){t.dismissed?(t.dismissed=!1,t.newLabel=null):(t.dismissed=!0,e.shiftKey&&this.lastSelectedImage?this.dismissAllImagesBetween(t,this.lastSelectedImage):this.lastSelectedImage=t)},goToRelabel:function(){this.step=1,this.lastSelectedImage=null},goToDismiss:function(){this.step=0,this.lastSelectedImage=null,this.selectedLabel&&this.getAnnotations(this.selectedLabel)},handleSelectedImageRelabel:function(t,e){t.newLabel?this.selectedLabel&&t.newLabel.id!==this.selectedLabel.id?t.newLabel=this.selectedLabel:t.newLabel=null:this.selectedLabel&&(t.newLabel=this.selectedLabel,e.shiftKey&&this.lastSelectedImage?this.relabelAllImagesBetween(t,this.lastSelectedImage):this.lastSelectedImage=t)},save:function(){var t=this;if(!this.loading){if(this.toDeleteCount>0){for(var e;null!==e&&parseInt(e,10)!==this.toDeleteCount;)e=prompt("This might delete ".concat(this.toDeleteCount," annotation(s). Please enter the number to continue."));if(null===e)return}this.startLoading(),this.performSave({dismissed_image_annotations:this.dismissedImageAnnotationsToSave,changed_image_annotations:this.changedImageAnnotationsToSave,dismissed_video_annotations:this.dismissedVideoAnnotationsToSave,changed_video_annotations:this.changedVideoAnnotationsToSave,force:this.forceChange}).then((function(e){return t.waitForSessionId=e.body.id}),(function(e){t.finishLoading(),l(e)}))}},handleSessionSaved:function(t){if(t.id==this.waitForSessionId){for(var e in this.finishLoading(),g.success("Saved. You can now start a new re-evaluation session."),this.step=0,this.annotationsCache)this.annotationsCache.hasOwnProperty(e)&&delete this.annotationsCache[e];this.handleSelectedLabel(this.selectedLabel)}},handleSessionFailed:function(t){t.id==this.waitForSessionId&&(this.finishLoading(),g.danger("There was an unexpected error."))},performOnAllImagesBetween:function(t,e,n){var i=this.allAnnotations.indexOf(t),s=this.allAnnotations.indexOf(e);if(s=0;n--)e.hasOwnProperty(t[n].label_id)?e[t[n].label_id].push(t[n].id):e[t[n].label_id]=[t[n].id];return e},packChangedToSave:function(t){for(var e={},n=t.length-1;n>=0;n--)e.hasOwnProperty(t[n].newLabel.id)?e[t[n].newLabel.id].push(t[n].id):e[t[n].newLabel.id]=[t[n].id];return e},initializeEcho:function(){a.getInstance().private("user-".concat(this.user.id)).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionSaved",this.handleSessionSaved).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionFailed",this.handleSessionFailed)},updateSortDirection:function(t){this.sortingDirection=t},updateSortKey:function(t){this.sortingKey=t}},watch:{annotations:function(t){r.$emit("annotations-count",t.length)},dismissedAnnotations:function(t){r.$emit("dismissed-annotations-count",t.length)},step:function(t){r.$emit("step",t)},selectedLabel:function(){this.isInDismissStep&&this.$refs.dismissGrid.setOffset(0)}},created:function(){var t=this;this.user=biigle.$require("largo.user"),window.addEventListener("beforeunload",(function(e){if(t.hasDismissedAnnotations)return e.preventDefault(),e.returnValue="","This page is asking you to confirm that you want to leave - data you have entered may not be saved."})),this.initializeEcho()}},undefined,undefined,!1,null,null,null).exports;const E=n({mixins:[D],components:{catalogImageGrid:_},data:function(){return{labelTrees:[]}},methods:{queryAnnotations:function(t){var e=S.queryImageAnnotations({id:t.id}),n=S.queryVideoAnnotations({id:t.id});return Vue.Promise.all([e,n])}},created:function(){var t=biigle.$require("annotationCatalog.labelTree");this.labelTrees=[t]}},undefined,undefined,!1,null,null,null).exports;const R=n({mixins:[D],data:function(){return{volumeId:null,labelTrees:[],mediaType:""}},methods:{queryAnnotations:function(t){var e,n;return"image"===this.mediaType?(e=o.queryImageAnnotations({id:this.volumeId,label_id:t.id}),n=Vue.Promise.resolve([])):(e=Vue.Promise.resolve([]),n=o.queryVideoAnnotations({id:this.volumeId,label_id:t.id})),Vue.Promise.all([e,n])},performSave:function(t){return o.save({id:this.volumeId},t)}},created:function(){this.volumeId=biigle.$require("largo.volumeId"),this.labelTrees=biigle.$require("largo.labelTrees"),this.mediaType=biigle.$require("largo.mediaType")}},undefined,undefined,!1,null,null,null).exports;const V=n({data:function(){return{step:0,count:0,dismissedCount:0}},computed:{shownCount:function(){return this.isInDismissStep?this.count:this.dismissedCount},isInDismissStep:function(){return 0===this.step},isInRelabelStep:function(){return 1===this.step}},methods:{updateStep:function(t){this.step=t},updateCount:function(t){this.count=t},updateDismissedCount:function(t){this.dismissedCount=t}},created:function(){r.$on("annotations-count",this.updateCount),r.$on("dismissed-annotations-count",this.updateDismissedCount),r.$on("step",this.updateStep)}},undefined,undefined,!1,null,null,null).exports,P=Vue.resource("api/v1/projects{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/projects{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/projects{/id}/video-annotations/filter/label{/label_id}"}});const j=n({mixins:[D],data:function(){return{projectId:null,labelTrees:[]}},methods:{queryAnnotations:function(t){var e=P.queryImageAnnotations({id:this.projectId,label_id:t.id}),n=P.queryVideoAnnotations({id:this.projectId,label_id:t.id});return Vue.Promise.all([e,n])},performSave:function(t){return P.save({id:this.projectId},t)}},created:function(){this.projectId=biigle.$require("largo.projectId"),this.labelTrees=biigle.$require("largo.labelTrees")}},undefined,undefined,!1,null,null,null).exports;biigle.$mount("annotation-catalog-container",E),biigle.$mount("largo-container",R),biigle.$mount("largo-title",V),biigle.$mount("project-largo-container",j)},401:()=>{}},n={};function i(t){var s=n[t];if(void 0!==s)return s.exports;var o=n[t]={exports:{}};return e[t](o,o.exports,i),o.exports}i.m=e,t=[],i.O=(e,n,s,o)=>{if(!n){var a=1/0;for(d=0;d=o)&&Object.keys(i.O).every((t=>i.O[t](n[l])))?n.splice(l--,1):(r=!1,o0&&t[d-1][2]>o;d--)t[d]=t[d-1];t[d]=[n,s,o]},i.o=(t,e)=>Object.prototype.hasOwnProperty.call(t,e),(()=>{var t={355:0,392:0};i.O.j=e=>0===t[e];var e=(e,n)=>{var s,o,[a,r,l]=n,u=0;if(a.some((e=>0!==t[e]))){for(s in r)i.o(r,s)&&(i.m[s]=r[s]);if(l)var d=l(i)}for(e&&e(n);ui(505)));var s=i.O(void 0,[392],(()=>i(401)));s=i.O(s)})(); \ No newline at end of file diff --git a/src/public/assets/styles/main.css b/src/public/assets/styles/main.css index 232cfe9b..d6499f1c 100644 --- a/src/public/assets/styles/main.css +++ b/src/public/assets/styles/main.css @@ -1 +1 @@ -.navbar-text.navbar-largo-breadcrumbs,.navbar-text.navbar-largo-breadcrumbs a{color:#ccc}.largo-images__alerts{align-items:center;display:flex;height:100%;justify-content:center;left:0;pointer-events:none;position:absolute;top:0;transition:background-color .25s ease;width:100%;z-index:2}.largo-images__alerts.block{background-color:rgba(0,0,0,.5);pointer-events:auto}.largo-tab{display:flex!important;flex-direction:column}.largo-tab .largo-tab__button{padding-bottom:10px}.largo-tab .largo-tab__label-trees{flex:1;overflow:auto}.image-grid__image--catalog>a{width:100%}.image-grid__image--relabel .new-label{bottom:0;left:0;overflow-x:hidden;padding:.5em;pointer-events:none;position:absolute;text-align:center;text-overflow:ellipsis;white-space:nowrap;width:100%}.image-grid__image--relabel .new-label__name{padding-right:16px}.image-grid__image--relabel .new-label__color{border:2px solid #eee;border-radius:50%;display:inline-block;height:1em;vertical-align:middle;width:1em}.largo-example-annotations .largo-example-annotations__images{margin-right:-1em;overflow-x:auto;white-space:nowrap}.largo-example-annotations .largo-example-annotation{image-orientation:none;height:125px;width:auto}.largo-example-annotations .largo-example-annotation:not(:last-child){margin-right:.5em}.largo-example-annotations .alert{margin-bottom:0} +.navbar-text.navbar-largo-breadcrumbs,.navbar-text.navbar-largo-breadcrumbs a{color:#ccc}.largo-images__alerts{align-items:center;display:flex;height:100%;justify-content:center;left:0;pointer-events:none;position:absolute;top:0;transition:background-color .25s ease;width:100%;z-index:2}.largo-images__alerts.block{background-color:rgba(0,0,0,.5);pointer-events:auto}.sidebar__tab--open.largo-tab{display:flex!important;flex-direction:column}.sidebar__tab--open.largo-tab .largo-tab__button{padding-bottom:10px}.sidebar__tab--open.largo-tab .largo-tab__label-trees{flex:1;overflow:auto}.image-grid__image--catalog>a{width:100%}.image-grid__image--relabel .new-label{bottom:0;left:0;overflow-x:hidden;padding:.5em;pointer-events:none;position:absolute;text-align:center;text-overflow:ellipsis;white-space:nowrap;width:100%}.image-grid__image--relabel .new-label__name{padding-right:16px}.image-grid__image--relabel .new-label__color{border:2px solid #eee;border-radius:50%;display:inline-block;height:1em;vertical-align:middle;width:1em}.largo-example-annotations .largo-example-annotations__images{margin-right:-1em;overflow-x:auto;white-space:nowrap}.largo-example-annotations .largo-example-annotation{image-orientation:none;height:125px;width:auto}.largo-example-annotations .largo-example-annotation:not(:last-child){margin-right:.5em}.largo-example-annotations .alert{margin-bottom:0} diff --git a/src/public/mix-manifest.json b/src/public/mix-manifest.json index 1b90a3e9..fa90f02b 100644 --- a/src/public/mix-manifest.json +++ b/src/public/mix-manifest.json @@ -1,4 +1,4 @@ { - "/assets/scripts/main.js": "/assets/scripts/main.js?id=7dc83b19fd9d27eb012fa7ccdf3ffa24", - "/assets/styles/main.css": "/assets/styles/main.css?id=4be2d80b899c739e0bed700e0ca8362c" + "/assets/scripts/main.js": "/assets/scripts/main.js?id=6d54228ed93adb52483f223570005c62", + "/assets/styles/main.css": "/assets/styles/main.css?id=4bcd521568b8ea929efd8e3f72f7d338" } diff --git a/src/resources/assets/js/components/sortingTab.vue b/src/resources/assets/js/components/sortingTab.vue new file mode 100644 index 00000000..fe1ded35 --- /dev/null +++ b/src/resources/assets/js/components/sortingTab.vue @@ -0,0 +1,118 @@ + + + + diff --git a/src/resources/assets/js/mixins/largoContainer.vue b/src/resources/assets/js/mixins/largoContainer.vue index 62b820be..91420019 100644 --- a/src/resources/assets/js/mixins/largoContainer.vue +++ b/src/resources/assets/js/mixins/largoContainer.vue @@ -1,6 +1,7 @@