diff --git a/.github/workflows/update-schema.yaml b/.github/workflows/update-schema.yaml new file mode 100644 index 00000000..7110dea2 --- /dev/null +++ b/.github/workflows/update-schema.yaml @@ -0,0 +1,22 @@ +name: Update Schema + +on: + push: + branches: + - master + paths: + - 'src/Database/migrations/**' + +jobs: + update-schema: + + runs-on: ubuntu-latest + + steps: + - name: Trigger schema update + run: | + curl -X POST --fail \ + -H "Authorization: token ${{ secrets.BIIGLE_SCHEMA_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + --data '{"event_type": "build_application"}' \ + https://api.github.com/repos/biigle/schema/dispatches diff --git a/README.md b/README.md index bf07fe49..0c88eba6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This is the BIIGLE module to review image annotations in a regular grid. This module is already included in [`biigle/biigle`](https://github.com/biigle/biigle). 1. Run `composer require biigle/largo`. +2. Install the Python dependencies with `pip install -r requirements.txt`. 2. Add `Biigle\Modules\Largo\LargoServiceProvider::class` to the `providers` array in `config/app.php`. 3. Run `php artisan vendor:publish --tag=public` to publish the public assets of this module. 4. Configure a storage disk for the Largo annotation patches and set the `LARGO_PATCH_STORAGE_DISK` variable to the name of this storage disk in the `.env` file. The content of the storage disk should be publicly accessible. Example for a local disk: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7d20fe0e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +Pillow==10.2.0 +torch==2.1.* +torchvision==0.16.* +xformers==0.0.18 diff --git a/src/Console/Commands/GenerateMissing.php b/src/Console/Commands/GenerateMissing.php index 30083bc7..12da1a03 100644 --- a/src/Console/Commands/GenerateMissing.php +++ b/src/Console/Commands/GenerateMissing.php @@ -2,13 +2,18 @@ namespace Biigle\Modules\Largo\Console\Commands; +use Biigle\Annotation; +use Biigle\Image; use Biigle\ImageAnnotation; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; -use Biigle\Modules\Largo\Jobs\GenerateVideoAnnotationPatch; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedFile; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedImage; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedVideo; use Biigle\VideoAnnotation; +use Biigle\VolumeFile; use Carbon\Carbon; -use File; use Illuminate\Console\Command; +use Illuminate\Contracts\Filesystem\Filesystem; +use Illuminate\Database\Eloquent\Builder; use Storage; class GenerateMissing extends Command @@ -18,39 +23,38 @@ class GenerateMissing extends Command * * @var string */ - protected $signature = 'largo:generate-missing {--dry-run} {--volume=} {--no-image-annotations} {--no-video-annotations} {--queue=} {--newer-than=} - {--older-than=}'; + protected $signature = 'largo:generate-missing + {--dry-run : Do not submit processing jobs to the queue} + {--volume= : Check only this volume} + {--skip-images : Do not check image annotations} + {--skip-videos : Do not check video annotations} + {--skip-vectors : Do not check feature vectors} + {--skip-patches : Do not check annotation patches} + {--queue= : Submit processing jobs to this queue} + {--newer-than= : Only check annotations newer than this date} + {--older-than= : Only check annotations older than this date}'; /** * The console command description. * * @var string */ - protected $description = 'Generate missing patches for annotations.'; + protected $description = 'Generate missing data for annotations.'; /** - * Largo patch storage file format. - * - * @var string + * Queue to push process jobs to. */ - protected $format; + protected string $queue; /** - * Number of annotations missing patches. - * - * @var int + * Whether to skip checking for missing patches. */ - protected $count; + protected bool $skipPatches; /** - * Create a new command instance. + * Whether to skip checking for missing feature vectors. */ - public function __construct() - { - parent::__construct(); - $this->format = config('largo.patch_format'); - $this->count = 0.0; - } + protected bool $skipVectors; /** * Execute the command. @@ -59,31 +63,31 @@ public function __construct() */ public function handle() { - $pushToQueue = !$this->option('dry-run'); - $storage = Storage::disk(config('largo.patch_storage_disk')); - $queue = $this->option('queue') ?: config('largo.generate_annotation_patch_queue'); + $this->queue = $this->option('queue') ?: config('largo.generate_annotation_patch_queue'); + $this->skipPatches = $this->option('skip-patches'); + $this->skipVectors = $this->option('skip-vectors'); - if (!$this->option('no-image-annotations')) { - $this->handleImageAnnotations($storage, $pushToQueue, $queue); + if (!$this->option('skip-images')) { + $this->handleImageAnnotations(); } - $this->count = 0; - - if (!$this->option('no-video-annotations')) { - $this->handleVideoAnnotations($storage, $pushToQueue, $queue); + if (!$this->option('skip-videos')) { + $this->handleVideoAnnotations(); } } /** * Check image annnotation patches - * - * @param \Illuminate\Filesystem\FilesystemAdapter $storage - * @param bool $pushToQueue - * @param string $queue */ - protected function handleImageAnnotations($storage, $pushToQueue, $queue) + protected function handleImageAnnotations(): void { $annotations = ImageAnnotation::join('images', 'images.id', '=', 'image_annotations.image_id') + // Order by image ID first because we want to submit the annotations in + // batches for each image. + ->orderBy('image_annotations.image_id') + // Order by annotation ID second to ensure a deterministic order for lazy(). + ->orderBy('image_annotations.id') + ->select('image_annotations.id', 'image_annotations.image_id') ->when($this->option('volume'), function ($query) { $query->where('images.volume_id', $this->option('volume')); }) @@ -93,75 +97,97 @@ protected function handleImageAnnotations($storage, $pushToQueue, $queue) ->when($this->option('older-than'), function ($query) { $query->where('image_annotations.created_at', '<', new Carbon($this->option('older-than'))); }) - ->select('image_annotations.id', 'images.uuid as uuid'); - - $total = $annotations->count(); - $progress = $this->output->createProgressBar($total); - $this->info("Checking {$total} image annotations..."); - - $handleAnnotation = function ($annotation) use ($progress, $pushToQueue, $storage, $queue) { - $prefix = fragment_uuid_path($annotation->uuid); - if (!$storage->exists("{$prefix}/{$annotation->id}.{$this->format}")) { - $this->count++; - if ($pushToQueue) { - GenerateImageAnnotationPatch::dispatch($annotation) - ->onQueue($queue); - } - } - $progress->advance(); - }; + ->when(!$this->skipVectors, function ($query) { + $query->leftJoin('image_annotation_label_feature_vectors', 'image_annotation_label_feature_vectors.annotation_id', '=', 'image_annotations.id') + ->addSelect('image_annotation_label_feature_vectors.id as vector_id'); + }); - $annotations->eachById($handleAnnotation, 10000, 'image_annotations.id', 'id'); - - $progress->finish(); - - if($total === 0) { - $this->info("\n"); - return; - } - - $percent = round($this->count / $total * 100, 2); - $this->info("\nFound {$this->count} image annotations with missing patches ({$percent} %)."); - if ($pushToQueue) { - $this->info("Pushed {$this->count} jobs to queue {$queue}."); - } + $this->line("Image annotations"); + $this->handleAnnotations($annotations); } /** * Check video annnotation patches - * - * @param \Illuminate\Filesystem\FilesystemAdapter $storage - * @param bool $pushToQueue - * @param string $queue */ - protected function handleVideoAnnotations($storage, $pushToQueue, $queue) + protected function handleVideoAnnotations(): void { $annotations = VideoAnnotation::join('videos', 'videos.id', '=', 'video_annotations.video_id') + // Order by video ID first because we want to submit the annotations in + // batches for each video. + ->orderBy('video_annotations.video_id') + // Order by annotation ID second to ensure a deterministic order for lazy(). + ->orderBy('video_annotations.id') + ->select('video_annotations.id', 'video_annotations.video_id') ->when($this->option('volume'), function ($query) { $query->where('videos.volume_id', $this->option('volume')); }) ->when($this->option('newer-than'), function ($query) { $query->where('video_annotations.created_at', '>', new Carbon($this->option('newer-than'))); }) - ->select('video_annotations.id', 'videos.uuid as uuid'); + ->when($this->option('older-than'), function ($query) { + $query->where('video_annotations.created_at', '<', new Carbon($this->option('older-than'))); + }) + ->when(!$this->skipVectors, function ($query) { + $query->leftJoin('video_annotation_label_feature_vectors', 'video_annotation_label_feature_vectors.annotation_id', '=', 'video_annotations.id') + ->addSelect('video_annotation_label_feature_vectors.id as vector_id'); + }); + + $this->line("Video annotations"); + $this->handleAnnotations($annotations); + } + + protected function handleAnnotations(Builder $annotations): void + { + $pushToQueue = !$this->option('dry-run'); + $storage = Storage::disk(config('largo.patch_storage_disk')); + $count = 0; + $jobCount = 0; $total = $annotations->count(); $progress = $this->output->createProgressBar($total); - $this->info("Checking {$total} video annotations..."); - - $handleAnnotation = function ($annotation) use ($progress, $pushToQueue, $storage, $queue) { - $prefix = fragment_uuid_path($annotation->uuid); - if (!$storage->exists("{$prefix}/v-{$annotation->id}.{$this->format}")) { - $this->count++; - if ($pushToQueue) { - GenerateVideoAnnotationPatch::dispatch($annotation) - ->onQueue($queue); + $this->info("Checking {$total} annotations..."); + + $currentFile = null; + $currentAnnotationBatch = []; + + // lazy() is crucial as we can't load all annotations at once! + foreach ($annotations->with('file')->lazy() as $annotation) { + $progress->advance(); + + if ($this->skipPatches) { + $needsPatch = false; + } else { + $needsPatch = !$storage->exists( + ProcessAnnotatedFile::getTargetPath($annotation) + ); + } + + $needsVector = !$this->skipVectors && is_null($annotation->vector_id); + + if (!$needsPatch && !$needsVector) { + continue; + } + + $count++; + + if (!$currentFile || $currentFile->id !== $annotation->file->id) { + if (!empty($currentAnnotationBatch) && $pushToQueue) { + $jobCount++; + $this->dispatcheProcessJob($currentFile, $currentAnnotationBatch); } + + $currentFile = $annotation->file; + $currentAnnotationBatch = []; } - $progress->advance(); - }; - $annotations->eachById($handleAnnotation, 10000, 'video_annotations.id', 'id'); + $currentAnnotationBatch[] = $annotation->id; + } + + // Push final job. + if (!empty($currentAnnotationBatch) && $pushToQueue) { + $jobCount++; + $this->dispatcheProcessJob($currentFile, $currentAnnotationBatch); + } $progress->finish(); @@ -170,10 +196,29 @@ protected function handleVideoAnnotations($storage, $pushToQueue, $queue) return; } - $percent = round($this->count / $total * 100, 2); - $this->info("\nFound {$this->count} video annotations with missing patches ({$percent} %)."); + $percent = round($count / $total * 100, 2); + $this->info("\nFound {$count} annotations with missing patches ({$percent} %)."); if ($pushToQueue) { - $this->info("Pushed {$this->count} jobs to queue {$queue}."); + $this->info("Pushed {$jobCount} jobs to queue {$this->queue}."); + } + } + + protected function dispatcheProcessJob(VolumeFile $file, array $ids) + { + if ($file instanceof Image) { + ProcessAnnotatedImage::dispatch($file, + only: $ids, + skipPatches: $this->skipPatches, + skipFeatureVectors: $this->skipVectors + ) + ->onQueue($this->queue); + } else { + ProcessAnnotatedVideo::dispatch($file, + only: $ids, + skipPatches: $this->skipPatches, + skipFeatureVectors: $this->skipVectors + ) + ->onQueue($this->queue); } } } diff --git a/src/Console/Commands/InitializeFeatureVectors.php b/src/Console/Commands/InitializeFeatureVectors.php new file mode 100644 index 00000000..4da2c3d1 --- /dev/null +++ b/src/Console/Commands/InitializeFeatureVectors.php @@ -0,0 +1,100 @@ +argument('volume')); + if ($volume->isImageVolume()) { + $this->processImages($volume); + } else { + $this->processVideos($volume); + } + } + + protected function processImages(Volume $volume) + { + $chunkSize = intval($this->option('chunk-size')); + $loopChunkSize = max($chunkSize, 10000); + + $query = ImageAnnotation::join('images', 'images.id', '=', 'image_annotations.image_id') + ->where('images.volume_id', $volume->id); + + $count = $query->count(); + $this->info("Processing {$count} image annotations."); + $p = $this->output->createProgressBar($count); + + $query->select('image_annotations.id') + ->chunkById($loopChunkSize, function ($chunk) use ($p, $chunkSize) { + $chunk->chunk($chunkSize)->each(function ($c) use ($p) { + $job = new InitializeFeatureVectorChunk($c->pluck('id')->all(), []); + if (!$this->option('dry-run')) { + Queue::pushOn($this->option('queue'), $job); + } + $p->advance($c->count()); + }); + }); + + $p->finish(); + $this->line(''); + } + + protected function processVideos(Volume $volume) + { + $chunkSize = intval($this->option('chunk-size')); + $loopChunkSize = max($chunkSize, 10000); + + $query = VideoAnnotation::join('videos', 'videos.id', '=', 'video_annotations.video_id') + ->where('videos.volume_id', $volume->id); + + $count = $query->count(); + $this->info("Processing {$count} video annotations."); + $p = $this->output->createProgressBar($count); + + $query->select('video_annotations.id') + ->chunkById($loopChunkSize, function ($chunk) use ($p, $chunkSize) { + $chunk->chunk($chunkSize)->each(function ($c) use ($p) { + $job = new InitializeFeatureVectorChunk([], $c->pluck('id')->all()); + if (!$this->option('dry-run')) { + Queue::pushOn($this->option('queue'), $job); + } + $p->advance($c->count()); + }); + }); + + $p->finish(); + $this->line(''); + } +} diff --git a/src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php b/src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php new file mode 100644 index 00000000..e32ae067 --- /dev/null +++ b/src/Database/Factories/ImageAnnotationLabelFeatureVectorFactory.php @@ -0,0 +1,38 @@ + ImageAnnotationLabel::factory(), + 'annotation_id' => ImageAnnotation::factory(), + 'label_id' => Label::factory(), + 'label_tree_id' => LabelTree::factory(), + 'volume_id' => Volume::factory(), + 'vector' => range(0, 383), + ]; + } +} diff --git a/src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php b/src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php new file mode 100644 index 00000000..4e80e588 --- /dev/null +++ b/src/Database/Factories/VideoAnnotationLabelFeatureVectorFactory.php @@ -0,0 +1,38 @@ + VideoAnnotationLabel::factory(), + 'annotation_id' => VideoAnnotation::factory(), + 'label_id' => Label::factory(), + 'label_tree_id' => LabelTree::factory(), + 'volume_id' => Volume::factory(), + 'vector' => range(0, 383), + ]; + } +} diff --git a/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php b/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php new file mode 100644 index 00000000..0740b94e --- /dev/null +++ b/src/Database/migrations/2023_12_15_153400_create_annotation_feature_vectors_tables.php @@ -0,0 +1,128 @@ +unsignedBigInteger('id'); + $table->foreign('id') + ->references('id') + ->on('image_annotation_labels') + ->onDelete('cascade'); + + $table->foreignId('annotation_id') + ->constrained(table: 'image_annotations') + ->onDelete('cascade'); + + $table->foreignId('label_id') + ->constrained() + ->onDelete('restrict'); + + // This is added to be used by LabelBOT in the future. + // We still have to think of an efficient indexing strategy so indexes + // are added later. + $table->foreignId('label_tree_id') + ->constrained() + ->onDelete('restrict'); + + $table->foreignId('volume_id') + ->constrained() + ->onDelete('cascade'); + + $table->vector('vector', 384); + + // For Largo queries. + $table->index(['label_id', 'volume_id']); + + // For create/update queries. + $table->index('annotation_id'); + + // Ensure consistency and speed up updateOrCreate queries. + $table->primary('id'); + }); + + Schema::create('video_annotation_label_feature_vectors', function (Blueprint $table) { + $table->unsignedBigInteger('id'); + $table->foreign('id') + ->references('id') + ->on('video_annotation_labels') + ->onDelete('cascade'); + + $table->foreignId('annotation_id') + ->constrained(table: 'video_annotations') + ->onDelete('cascade'); + + $table->foreignId('label_id') + ->constrained() + ->onDelete('restrict'); + + // This is added to be used by LabelBOT in the future. + // We still have to think of an efficient indexing strategy so indexes + // are added later. + $table->foreignId('label_tree_id') + ->constrained() + ->onDelete('restrict'); + + $table->foreignId('volume_id') + ->constrained() + ->onDelete('cascade'); + + $table->vector('vector', 384); + + // For Largo queries. + $table->index(['label_id', 'volume_id']); + + // For create/update queries. + $table->index('annotation_id'); + + // Ensure consistency and speed up updateOrCreate queries. + $table->primary('id'); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('image_annotation_label_feature_vectors'); + Schema::dropIfExists('video_annotation_label_feature_vectors'); + } +}; diff --git a/src/Http/Controllers/Api/Projects/SortAnnotationsByOutliersController.php b/src/Http/Controllers/Api/Projects/SortAnnotationsByOutliersController.php new file mode 100644 index 00000000..f748e665 --- /dev/null +++ b/src/Http/Controllers/Api/Projects/SortAnnotationsByOutliersController.php @@ -0,0 +1,74 @@ +authorize('access', $project); + + // This was too complicated with the query builder. Since there is no rist of SQL + // injection here, we just use raw SQL. + $sql = << ( + SELECT AVG("temp2"."vector") FROM ( + SELECT "vector" FROM "image_annotation_label_feature_vectors" + WHERE "label_id" = :lid AND "volume_id" IN ( + SELECT "volume_id" FROM "project_volume" WHERE "project_id" = :pid + ) + UNION ALL + SELECT "vector" FROM "video_annotation_label_feature_vectors" + WHERE "label_id" = :lid AND "volume_id" IN ( + SELECT "volume_id" FROM "project_volume" WHERE "project_id" = :pid + ) + ) AS temp2 + ) DESC, id DESC + SQL; + + $ids = DB::select($sql, ['pid' => $pid, 'lid' => $lid]); + + // Filtering unique IDs is not required here because the UNIQUE in the query + // takes care of that. + return array_map(fn ($v) => $v->id, $ids); + } +} diff --git a/src/Http/Controllers/Api/Volumes/SortAnnotationsByOutliersController.php b/src/Http/Controllers/Api/Volumes/SortAnnotationsByOutliersController.php new file mode 100644 index 00000000..495f1372 --- /dev/null +++ b/src/Http/Controllers/Api/Volumes/SortAnnotationsByOutliersController.php @@ -0,0 +1,49 @@ +authorize('access', $volume); + + if ($volume->isVideoVolume()) { + $query = VideoAnnotationLabelFeatureVector::where('volume_id', $vid) + ->where('label_id', $lid) + ->orderByRaw('vector <=> (SELECT AVG(vector) FROM video_annotation_label_feature_vectors WHERE volume_id = ? AND label_id = ?) DESC, annotation_id DESC', [$vid, $lid]); + } else { + $query = ImageAnnotationLabelFeatureVector::where('volume_id', $vid) + ->where('label_id', $lid) + ->orderByRaw('vector <=> (SELECT AVG(vector) FROM image_annotation_label_feature_vectors WHERE volume_id = ? AND label_id = ?) DESC, annotation_id DESC', [$vid, $lid]); + } + + return $query->pluck('annotation_id') + // Use distinct/unique *after* fetchig from the DB because otherwise the + // vector would need to be selected, too (order by expressions must appear in + // the select list). But we don't want to get the huge vectors. + ->unique() + ->values(); + } +} diff --git a/src/Http/routes.php b/src/Http/routes.php index 08b1e293..ad39aa3d 100644 --- a/src/Http/routes.php +++ b/src/Http/routes.php @@ -25,39 +25,47 @@ 'prefix' => 'api/v1', 'middleware' => ['api', 'auth:web,api'], ], function ($router) { - $router->post('volumes/{id}/largo', [ - 'uses' => 'Volumes\LargoController@save', + $router->get('labels/{id}/image-annotations', [ + 'uses' => 'Labels\ImageAnnotationsController@index', ]); - $router->get('volumes/{id}/image-annotations/filter/label/{id2}', [ - 'uses' => 'Volumes\FilterImageAnnotationsByLabelController@index', + $router->get('labels/{id}/video-annotations', [ + 'uses' => 'Labels\VideoAnnotationsController@index', ]); - $router->get('volumes/{id}/video-annotations/filter/label/{id2}', [ - 'uses' => 'Volumes\FilterVideoAnnotationsByLabelController@index', + $router->get('projects/{id}/annotations/sort/outliers/{id2}', [ + 'uses' => 'Projects\SortAnnotationsByOutliersController@index', ]); - $router->get('volumes/{id}/image-annotations/examples/{id2}', [ - 'uses' => 'Volumes\ImageAnnotationExamplesController@index', + $router->get('projects/{id}/image-annotations/filter/label/{id2}', [ + 'uses' => 'Projects\FilterImageAnnotationsByLabelController@index', ]); $router->post('projects/{id}/largo', [ 'uses' => 'Projects\LargoController@save', ]); - $router->get('projects/{id}/image-annotations/filter/label/{id2}', [ - 'uses' => 'Projects\FilterImageAnnotationsByLabelController@index', - ]); - $router->get('projects/{id}/video-annotations/filter/label/{id2}', [ 'uses' => 'Projects\FilterVideoAnnotationsByLabelController@index', ]); - $router->get('labels/{id}/image-annotations', [ - 'uses' => 'Labels\ImageAnnotationsController@index', + $router->get('volumes/{id}/annotations/sort/outliers/{id2}', [ + 'uses' => 'Volumes\SortAnnotationsByOutliersController@index', ]); - $router->get('labels/{id}/video-annotations', [ - 'uses' => 'Labels\VideoAnnotationsController@index', + $router->get('volumes/{id}/image-annotations/examples/{id2}', [ + 'uses' => 'Volumes\ImageAnnotationExamplesController@index', + ]); + + $router->get('volumes/{id}/image-annotations/filter/label/{id2}', [ + 'uses' => 'Volumes\FilterImageAnnotationsByLabelController@index', + ]); + + $router->post('volumes/{id}/largo', [ + 'uses' => 'Volumes\LargoController@save', + ]); + + $router->get('volumes/{id}/video-annotations/filter/label/{id2}', [ + 'uses' => 'Volumes\FilterVideoAnnotationsByLabelController@index', ]); }); diff --git a/src/ImageAnnotationLabelFeatureVector.php b/src/ImageAnnotationLabelFeatureVector.php new file mode 100644 index 00000000..a673d08a --- /dev/null +++ b/src/ImageAnnotationLabelFeatureVector.php @@ -0,0 +1,53 @@ + Vector::class, + ]; + + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'id', + 'annotation_id', + 'label_id', + 'label_tree_id', + 'volume_id', + 'vector', + ]; + + /** + * Create a new factory instance for the model. + * + * @return \Illuminate\Database\Eloquent\Factories\Factory + */ + protected static function newFactory() + { + return ImageAnnotationLabelFeatureVectorFactory::new(); + } +} diff --git a/src/Jobs/ApplyLargoSession.php b/src/Jobs/ApplyLargoSession.php index b404bee4..3da07a6f 100644 --- a/src/Jobs/ApplyLargoSession.php +++ b/src/Jobs/ApplyLargoSession.php @@ -8,6 +8,8 @@ use Biigle\Label; use Biigle\Modules\Largo\Events\LargoSessionFailed; use Biigle\Modules\Largo\Events\LargoSessionSaved; +use Biigle\Modules\Largo\Jobs\CopyImageAnnotationFeatureVector; +use Biigle\Modules\Largo\Jobs\CopyVideoAnnotationFeatureVector; use Biigle\Modules\Largo\Jobs\RemoveImageAnnotationPatches; use Biigle\Modules\Largo\Jobs\RemoveVideoAnnotationPatches; use Biigle\User; @@ -144,8 +146,11 @@ protected function handleImageAnnotations() { [$dismissed, $changed] = $this->ignoreDeletedLabels($this->dismissedImageAnnotations, $this->changedImageAnnotations); - $this->applyDismissedLabels($this->user, $dismissed, $this->force, ImageAnnotationLabel::class); + // Change labels first, then dismiss to keep the opportunity to copy feature + // vectors. If labels are deleted first, the feature vectors will be immediately + // deleted, too, and nothing can be copied any more. $this->applyChangedLabels($this->user, $changed, ImageAnnotation::class, ImageAnnotationLabel::class); + $this->applyDismissedLabels($this->user, $dismissed, $this->force, ImageAnnotationLabel::class); $this->deleteDanglingAnnotations($dismissed, $changed, ImageAnnotation::class); } @@ -156,8 +161,11 @@ protected function handleVideoAnnotations() { [$dismissed, $changed] = $this->ignoreDeletedLabels($this->dismissedVideoAnnotations, $this->changedVideoAnnotations); - $this->applyDismissedLabels($this->user, $dismissed, $this->force, VideoAnnotationLabel::class); + // Change labels first, then dismiss to keep the opportunity to copy feature + // vectors. If labels are deleted first, the feature vectors will be immediately + // deleted, too, and nothing can be copied any more. $this->applyChangedLabels($this->user, $changed, VideoAnnotation::class, VideoAnnotationLabel::class); + $this->applyDismissedLabels($this->user, $dismissed, $this->force, VideoAnnotationLabel::class); $this->deleteDanglingAnnotations($dismissed, $changed, VideoAnnotation::class); } @@ -286,6 +294,10 @@ protected function applyChangedLabels($user, $changed, $annotationModel, $labelM } } + // Store the ID so we can efficiently loop over the models later. This is only + // possible because all this happens inside a DB transaction (see above). + $startId = $labelModel::orderBy('id', 'desc')->select('id')->first()->id; + collect($newAnnotationLabels) ->when($labelModel === ImageAnnotationLabel::class, function ($labels) { return $labels->map(function ($item) { @@ -301,6 +313,18 @@ protected function applyChangedLabels($user, $changed, $annotationModel, $labelM ->each(function ($chunk) use ($labelModel) { $labelModel::insert($chunk->toArray()); }); + + $labelModel::where('id', '>', $startId) + ->eachById(function ($annotationLabel) use ($labelModel) { + // Execute the jobs synchronously because after this method the + // old annotation labels may be deleted and there will be nothing + // to copy any more. + if ($labelModel === ImageAnnotationLabel::class) { + (new CopyImageAnnotationFeatureVector($annotationLabel))->handle(); + } else { + (new CopyVideoAnnotationFeatureVector($annotationLabel))->handle(); + } + }); } /** diff --git a/src/Jobs/CopyAnnotationFeatureVector.php b/src/Jobs/CopyAnnotationFeatureVector.php new file mode 100644 index 00000000..5559a87b --- /dev/null +++ b/src/Jobs/CopyAnnotationFeatureVector.php @@ -0,0 +1,61 @@ +getFeatureVectorQuery()->first(); + if (!is_null($vector)) { + $this->updateOrCreateFeatureVector([ + 'id' => $this->annotationLabel->id, + 'annotation_id' => $this->annotationLabel->annotation_id, + 'label_id' => $this->annotationLabel->label_id, + 'label_tree_id' => $this->annotationLabel->label->label_tree_id, + 'volume_id' => $vector->volume_id, + 'vector' => $vector->vector, + ]); + } + } + + /** + * Get a query for the feature vectors associated with the annotation of this job. + */ + abstract protected function getFeatureVectorQuery(): Builder; + + /** + * Create a new feature vector model for the annotation of this job. + */ + abstract protected function updateOrCreateFeatureVector(array $attributes): void; +} diff --git a/src/Jobs/CopyImageAnnotationFeatureVector.php b/src/Jobs/CopyImageAnnotationFeatureVector.php new file mode 100644 index 00000000..2a5c0c31 --- /dev/null +++ b/src/Jobs/CopyImageAnnotationFeatureVector.php @@ -0,0 +1,28 @@ +annotationLabel->annotation_id); + } + + + /** + * {@inheritdoc} + */ + protected function updateOrCreateFeatureVector(array $attributes): void + { + $idArray = ['id' => $attributes['id']]; + unset($attributes['id']); + ImageAnnotationLabelFeatureVector::updateOrCreate($idArray, $attributes); + } +} diff --git a/src/Jobs/CopyVideoAnnotationFeatureVector.php b/src/Jobs/CopyVideoAnnotationFeatureVector.php new file mode 100644 index 00000000..21c9964b --- /dev/null +++ b/src/Jobs/CopyVideoAnnotationFeatureVector.php @@ -0,0 +1,28 @@ +annotationLabel->annotation_id); + } + + + /** + * {@inheritdoc} + */ + protected function updateOrCreateFeatureVector(array $attributes): void + { + $idArray = ['id' => $attributes['id']]; + unset($attributes['id']); + VideoAnnotationLabelFeatureVector::updateOrCreate($idArray, $attributes); + } +} diff --git a/src/Jobs/GenerateAnnotationPatch.php b/src/Jobs/GenerateAnnotationPatch.php deleted file mode 100644 index eac153b3..00000000 --- a/src/Jobs/GenerateAnnotationPatch.php +++ /dev/null @@ -1,299 +0,0 @@ -annotation = $annotation; - $this->targetDisk = $targetDisk !== null - ? $targetDisk - : config('largo.patch_storage_disk'); - } - - /** - * Execute the job. - * - * @return void - */ - public function handle() - { - try { - FileCache::get($this->annotation->getFile(), [$this, 'handleFile'], true); - } catch (FileLockedException $e) { - // Retry this job without increasing the attempts if the file is currently - // written by another worker. This worker can process other jobs in the - // meantime. - // See: https://github.com/laravel/ideas/issues/735 - static::dispatch($this->annotation, $this->targetDisk) - ->onConnection($this->connection) - ->onQueue($this->queue) - ->delay(60); - } catch (Exception $e) { - if ($this->shouldRetryAfterException($e)) { - // Exponential backoff for retry after 10 and then 20 minutes. - $this->release($this->attempts() * 600); - } else { - $class = get_class($this->annotation); - Log::warning("Could not generate annotation patch for {$class} {$this->annotation->id}: {$e->getMessage()}", ['exception' => $e]); - } - } - } - - /** - * Handle a single file. - * - * @param VolumeFile $file - * @param string $path Path to the cached file. - */ - abstract public function handleFile(VolumeFile $file, $path); - - /** - * Assemble the target path for an annotation patch. - * - * @param Annotation $annotation - * - * @return string - */ - protected function getTargetPath(Annotation $annotation): string - { - $prefix = fragment_uuid_path($annotation->getFile()->uuid); - $format = config('largo.patch_format'); - - if ($annotation instanceof VideoAnnotation) { - // Add "v-" to make absolutely sure that no collisions (same UUID, same ID) - // occur because patches are stored on the same disk. - return "{$prefix}/v-{$annotation->id}.{$format}"; - } - - // This is the old patch storage scheme, so we don't add "i-" for backwards - // compatibility. - return "{$prefix}/{$annotation->id}.{$format}"; - } - - /** - * Determine if this job should retry instead of fail after an exception - * - * @param Exception $e - * - * @return bool - */ - protected function shouldRetryAfterException(Exception $e) - { - $message = $e->getMessage(); - return $this->attempts() < $this->tries && ( - // The remote source might be available again after a while. - Str::contains($message, 'The source resource could not be established') || - // This error presumably occurs due to worker concurrency. - Str::contains($message, 'Impossible to create the root directory') - ); - } - - /** - * Calculate the bounding rectangle of the patch to extract. - * - * @param array $points - * @param Shape $Shape - * @param int $thumbWidth - * @param int $thumbHeight - * - * @return array Containing width, height, top and left - */ - protected function getPatchRect(array $points, Shape $shape, $thumbWidth, $thumbHeight) - { - $padding = config('largo.patch_padding'); - - switch ($shape->id) { - case Shape::pointId(): - $pointPadding = config('largo.point_padding'); - $left = $points[0] - $pointPadding; - $right = $points[0] + $pointPadding; - $top = $points[1] - $pointPadding; - $bottom = $points[1] + $pointPadding; - break; - - case Shape::circleId(): - $left = $points[0] - $points[2]; - $right = $points[0] + $points[2]; - $top = $points[1] - $points[2]; - $bottom = $points[1] + $points[2]; - break; - - default: - $left = INF; - $right = -INF; - $top = INF; - $bottom = -INF; - foreach ($points as $index => $value) { - if ($index % 2 === 0) { - $left = min($left, $value); - $right = max($right, $value); - } else { - $top = min($top, $value); - $bottom = max($bottom, $value); - } - } - } - - $left -= $padding; - $right += $padding; - $top -= $padding; - $bottom += $padding; - - $width = $right - $left; - $height = $bottom - $top; - - // Ensure the minimum width so the annotation patch is not "zoomed in". - if ($width < $thumbWidth) { - $delta = ($thumbWidth - $width) / 2.0; - $left -= $delta; - $right += $delta; - $width = $thumbWidth; - } - - // Ensure the minimum height so the annotation patch is not "zoomed in". - if ($height < $thumbHeight) { - $delta = ($thumbHeight - $height) / 2.0; - $top -= $delta; - $bottom += $delta; - $height = $thumbHeight; - } - - $widthRatio = $width / $thumbWidth; - $heightRatio = $height / $thumbHeight; - - // increase the size of the patch so its aspect ratio is the same than the - // ratio of the thumbnail dimensions - if ($widthRatio > $heightRatio) { - $newHeight = round($thumbHeight * $widthRatio); - $top -= round(($newHeight - $height) / 2); - $height = $newHeight; - } else { - $newWidth = round($thumbWidth * $heightRatio); - $left -= round(($newWidth - $width) / 2); - $width = $newWidth; - } - - return [ - 'width' => intval(round($width)), - 'height' => intval(round($height)), - 'left' => intval(round($left)), - 'top' => intval(round($top)), - ]; - } - - /** - * Adjust the position and size of the patch rectangle so it is contained in the - * image. - * - * @param array $rect - * @param Image $image - * - * @return array - */ - protected function makeRectContained($rect, $image) - { - // Order of min max is importans so the point gets no negative coordinates. - $rect['left'] = min($image->width - $rect['width'], $rect['left']); - $rect['left'] = max(0, $rect['left']); - $rect['top'] = min($image->height - $rect['height'], $rect['top']); - $rect['top'] = max(0, $rect['top']); - - // Adjust dimensions of rect if it is larger than the image. - $rect['width'] = min($image->width, $rect['width']); - $rect['height'] = min($image->height, $rect['height']); - - return $rect; - } - - /** - * Get the annotation patch as buffer. - * - * @param Image $image - * @param array $points - * @param Shape $shape - * - * @return string - */ - protected function getAnnotationPatch($image, $points, $shape) - { - $thumbWidth = config('thumbnails.width'); - $thumbHeight = config('thumbnails.height'); - - if ($shape->id === Shape::wholeFrameId()) { - $image = $image->resize(floatval($thumbWidth) / $image->width); - } else { - $rect = $this->getPatchRect($points, $shape, $thumbWidth, $thumbHeight); - $rect = $this->makeRectContained($rect, $image); - - $image = $image->crop( - $rect['left'], - $rect['top'], - $rect['width'], - $rect['height'] - ) - ->resize(floatval($thumbWidth) / $rect['width']); - } - - return $image->writeToBuffer('.'.config('largo.patch_format'), [ - 'Q' => 85, - 'strip' => true, - ]); - } -} diff --git a/src/Jobs/GenerateFeatureVectors.php b/src/Jobs/GenerateFeatureVectors.php new file mode 100644 index 00000000..8b0da453 --- /dev/null +++ b/src/Jobs/GenerateFeatureVectors.php @@ -0,0 +1,262 @@ +id) { + Shape::pointId() => $this->getPointBoundingBox($points, $pointPadding), + Shape::circleId() => $this->getCircleBoundingBox($points), + // An ellipse will not be handled correctly by this but I didn't bother + // because this shape is almost never used anyway. + default => $this->getPolygonBoundingBox($points), + }; + + if ($boxPadding > 0) { + $box = [ + $box[0] - $boxPadding, + $box[1] - $boxPadding, + $box[2] + $boxPadding, + $box[3] + $boxPadding, + ]; + } + + return $this->makeBoxIntegers([ + $box[0], // left + $box[1], // top + $box[2] - $box[0], // width + $box[3] - $box[1], // height + ]); + } + + /** + * Modify a bounding box so it adheres to the aspect ratio given by width and height. + */ + public function ensureBoxAspectRatio(array $box, int $aspectWidth, int $aspectHeight): array + { + [$left, $top, $width, $height] = $box; + + // Ensure the minimum width so the annotation patch is not "zoomed in". + if ($width < $aspectWidth) { + $left -= ($aspectWidth - $width) / 2.0; + $width = $aspectWidth; + } + + // Ensure the minimum height so the annotation patch is not "zoomed in". + if ($height < $aspectHeight) { + $top -= ($aspectHeight - $height) / 2.0; + $height = $aspectHeight; + } + + $widthRatio = $width / $aspectWidth; + $heightRatio = $height / $aspectHeight; + + // Increase the size of the patch so its aspect ratio is the same than the + // ratio of the given dimensions. + if ($widthRatio > $heightRatio) { + $newHeight = round($aspectHeight * $widthRatio); + $top -= round(($newHeight - $height) / 2); + $height = $newHeight; + } else { + $newWidth = round($aspectWidth * $heightRatio); + $left -= round(($newWidth - $width) / 2); + $width = $newWidth; + } + + return $this->makeBoxIntegers([$left, $top, $width, $height]); + } + + /** + * Adjust the position and size of the box so it is contained in a box with the given + * dimensions. + */ + public function makeBoxContained(array $box, ?int $maxWidth, ?int $maxHeight) + { + [$left, $top, $width, $height] = $box; + + if (!is_null($maxWidth)) { + $left = min($maxWidth - $width, $left); + // Adjust dimensions of rect if it is larger than the image. + $width = min($maxWidth, $width); + } + + if (!is_null($maxHeight)) { + $top = min($maxHeight - $height, $top); + // Adjust dimensions of rect if it is larger than the image. + $height = min($maxHeight, $height); + } + + // Order of min max is importans so the point gets no negative coordinates. + $left = max(0, $left); + $top = max(0, $top); + + return [$left, $top, $width, $height]; + } + + /** + * Get the bounding box of a point annotation. + */ + protected function getPointBoundingBox(array $points, int $padding): array + { + return [ + $points[0] - $padding, + $points[1] - $padding, + $points[0] + $padding, + $points[1] + $padding, + ]; + } + + /** + * Get the bounding box of a circle annotation. + */ + protected function getCircleBoundingBox(array $points): array + { + return [ + $points[0] - $points[2], + $points[1] - $points[2], + $points[0] + $points[2], + $points[1] + $points[2], + ]; + } + + /** + * Get the bounding box of an annotation that is no point, circle or whole frame. + */ + protected function getPolygonBoundingBox(array $points): array + { + $minX = INF; + $minY = INF; + $maxX = -INF; + $maxY = -INF; + + for ($i = 0; $i < count($points); $i += 2) { + $minX = min($minX, $points[$i]); + $minY = min($minY, $points[$i + 1]); + $maxX = max($maxX, $points[$i]); + $maxY = max($maxY, $points[$i + 1]); + } + + return [$minX, $minY, $maxX, $maxY]; + } + + /** + * Round and cast box values to int. + */ + protected function makeBoxIntegers(array $box): array + { + return array_map(fn ($v) => intval(round($v)), $box); + } + + /** + * Generate the input for the python script. + * + * @param array $files VolumeFile instances of the files to which the annotations + * belong. + * @param array $paths Paths of locally cached files. + * @param \Illuminate\Support\Collection $annotations Annotations grouped by their + * file ID (e.g. image_id). + */ + protected function generateInput(array $files, array $paths, Collection $annotations): array + { + $input = []; + + foreach ($files as $index => $file) { + $path = $paths[$index]; + $fileAnnotations = $annotations[$file->id]; + $boxes = $this->generateFileInput($file, $fileAnnotations); + + if (!empty($boxes)) { + $input[$path] = $boxes; + } + } + + return $input; + } + + protected function generateFileInput(VolumeFile $file, Collection $annotations): array + { + $boxes = []; + foreach ($annotations as $a) { + if ($a->shape_id === Shape::wholeFrameId()) { + $box = [0, 0, $file->width ?: 0, $file->height ?: 0]; + } else { + $points = $a->getPoints(); + if (($a instanceof VideoAnnotation) && !empty($points)) { + $points = $points[0]; + } + $box = $this->getAnnotationBoundingBox($points, $a->getShape(), self::POINT_PADDING); + $box = $this->makeBoxContained($box, $file->width, $file->height); + } + + $zeroSize = $box[2] === 0 && $box[3] === 0; + + if (!$zeroSize) { + // Convert width and height to "right" and "bottom" coordinates. + $box[2] = $box[0] + $box[2]; + $box[3] = $box[1] + $box[3]; + + $boxes[$a->id] = $box; + } + } + + return $boxes; + } + + /** + * Run the Python command. + * + * @param string $command + */ + protected function python(string $inputPath, string $outputPath) + { + $python = config('largo.python'); + $script = config('largo.extract_features_script'); + $result = Process::forever() + ->env(['TORCH_HOME' => config('largo.torch_hub_path')]) + ->run("{$python} -u {$script} {$inputPath} {$outputPath}") + ->throw(); + } + + /** + * Generator to read the output CSV row by row. + */ + protected function readOutputCsv(string $path): \Generator + { + $file = new SplFileObject($path); + while (!$file->eof()) { + $csv = $file->fgetcsv(); + if (count($csv) === 2) { + yield $csv; + } + } + } +} diff --git a/src/Jobs/GenerateImageAnnotationPatch.php b/src/Jobs/GenerateImageAnnotationPatch.php deleted file mode 100644 index 2c456330..00000000 --- a/src/Jobs/GenerateImageAnnotationPatch.php +++ /dev/null @@ -1,44 +0,0 @@ -getTargetPath($this->annotation); - $image = $this->getVipsImage($path); - - $buffer = $this->getAnnotationPatch($image, $this->annotation->getPoints(), $this->annotation->getShape()); - - Storage::disk($this->targetDisk)->put($targetPath, $buffer); - } - - /** - * Get the vips image instance. - * - * @param string $path - * - * @return \Jcupitt\Vips\Image - */ - protected function getVipsImage($path) - { - return VipsImage::newFromFile($path, ['access' => 'sequential']); - } -} diff --git a/src/Jobs/GenerateVideoAnnotationPatch.php b/src/Jobs/GenerateVideoAnnotationPatch.php deleted file mode 100644 index df0ec368..00000000 --- a/src/Jobs/GenerateVideoAnnotationPatch.php +++ /dev/null @@ -1,62 +0,0 @@ -annotation->frames) === 0) { - // Expect the unexpected. - return; - } - - $points = $this->annotation->points[0] ?? null; - $targetPath = $this->getTargetPath($this->annotation); - - $video = $this->getVideo($path); - $frame = $this->getVideoFrame($video, $this->annotation->frames[0]); - $buffer = $this->getAnnotationPatch($frame, $points, $this->annotation->shape); - Storage::disk($this->targetDisk)->put($targetPath, $buffer); - } - - /** - * Get the FFMpeg video instance. - * - * @param string $path - * - * @return Video - */ - protected function getVideo($path) - { - return FFMpeg::create()->open($path); - } - - /** - * Get a video frame from a specific time as VipsImage object. - * - * @param Video $video - * @param float $time - * - * @return \Jcupitt\Vips\Image - */ - protected function getVideoFrame(Video $video, $time) - { - $buffer = $video->frame(TimeCode::fromSeconds($time))->save(null, false, true); - - return VipsImage::newFromBuffer($buffer); - } -} diff --git a/src/Jobs/InitializeFeatureVectorChunk.php b/src/Jobs/InitializeFeatureVectorChunk.php new file mode 100644 index 00000000..1e749934 --- /dev/null +++ b/src/Jobs/InitializeFeatureVectorChunk.php @@ -0,0 +1,188 @@ +imageAnnotationIds) + ->distinct() + ->pluck('annotation_id') + ->toArray(); + + $ids = array_diff($this->imageAnnotationIds, $skipIds); + $models = ImageAnnotation::whereIn('id', $ids) + ->with('file', 'labels.label', 'shape') + ->get() + ->keyBy('id'); + + // Chunk to avoid maximum insert parameter limit. + $this->getInsertData($models)->chunk(10000)->each(fn ($chunk) => + ImageAnnotationLabelFeatureVector::insert($chunk->toArray()) + ); + + $skipIds = VideoAnnotationLabelFeatureVector::whereIn('annotation_id', $this->videoAnnotationIds) + ->distinct() + ->pluck('annotation_id') + ->toArray(); + + $ids = array_diff($this->videoAnnotationIds, $skipIds); + $models = VideoAnnotation::whereIn('id', $ids) + ->with('file', 'labels.label') + ->get() + ->keyBy('id'); + + // Chunk to avoid maximum insert parameter limit. + $this->getInsertData($models)->chunk(10000)->each(fn ($chunk) => + VideoAnnotationLabelFeatureVector::insert($chunk->toArray()) + ); + } + + /** + * Get the array to insert new feature vector models into the DB. + */ + public function getInsertData(Collection $models): Collection + { + $insert = collect([]); + + $outputPath = tempnam(sys_get_temp_dir(), 'largo_feature_vector_output'); + + try { + foreach ($this->getFeatureVectors($models, $outputPath) as $row) { + $annotation = $models->get($row[0]); + $vector = $row[1]; + $i = $annotation->labels + ->map(fn ($annotationLabel) => [ + 'id' => $annotationLabel->id, + 'annotation_id' => $annotation->id, + 'label_id' => $annotationLabel->label_id, + 'label_tree_id' => $annotationLabel->label->label_tree_id, + 'volume_id' => $annotation->file->volume_id, + 'vector' => $vector, + ]); + $insert = $insert->concat($i); + } + } finally { + File::delete($outputPath); + } + + return $insert; + } + + /** + * Generate feature vectors from the thumbnails of many annotations. + * + * @param Collection $models Annotation models + * @param string $outputPath Path to stroe the CSV file with feature vectors + */ + public function getFeatureVectors(Collection $models, string $outputPath): \Generator + { + if ($models->isEmpty()) { + return (fn () => yield from [])(); + } + + $disk = Storage::disk(config('largo.patch_storage_disk')); + $rect = [0, 0, config('thumbnails.width'), config('thumbnails.height')]; + $paths = []; + + try { + $input = []; + foreach ($models as $a) { + $srcPath = ProcessAnnotatedFile::getTargetPath($a); + $tmpPath = tempnam(sys_get_temp_dir(), ''); + + $thumbnail = $disk->get($srcPath); + if (is_null($thumbnail)) { + continue; + } + + File::put($tmpPath, $thumbnail); + $paths[] = $tmpPath; + + // Compute the annotation outlines in "thumbnail space". + $padding = config('largo.patch_padding'); + $pointPadding = config('largo.point_padding'); + $thumbWidth = config('thumbnails.width'); + $thumbHeight = config('thumbnails.height'); + + if ($a->shape_id === Shape::wholeFrameId()) { + $input[$tmpPath] = [$a->id => [0, 0, $thumbWidth, $thumbHeight]]; + continue; + } + + if ($a instanceof VideoAnnotation) { + $points = $a->points[0]; + } else { + $points = $a->points; + } + + // First determine the box of the thumbnail to get the position and scale + // factor. + $box = $this->getAnnotationBoundingBox($points, $a->shape, $pointPadding, $padding); + $box = $this->ensureBoxAspectRatio($box, $thumbWidth, $thumbHeight); + $box = $this->makeBoxContained($box, $a->file->width, $a->file->height); + $x = $box[0]; + $y = $box[1]; + $scale = floatval($thumbWidth) / $box[2]; + + // The get the box of the annotation. + $box = $this->getAnnotationBoundingBox($points, $a->shape, $pointPadding, $padding); + $box = $this->makeBoxContained($box, $a->file->width, $a->file->height); + // Make coordinates relative to thumbnail box. + $box[0] -= $x; + $box[1] -= $y; + // Than scale coordinates to "thumbnail space". + $box = array_map(fn ($v) => $v * $scale, $box); + $box = $this->makeBoxIntegers($box); + + $zeroSize = $box[2] === 0 && $box[3] === 0; + + if ($zeroSize) { + continue; + } + // Convert width and height to "right" and "bottom" coordinates. + $box[2] = $box[0] + $box[2]; + $box[3] = $box[1] + $box[3]; + + $input[$tmpPath] = [$a->id => $box]; + } + + if (empty($input)) { + return (fn () => yield from [])(); + } + + $inputPath = tempnam(sys_get_temp_dir(), 'largo_feature_vector_input'); + + File::put($inputPath, json_encode($input)); + $paths[] = $inputPath; + + $this->python($inputPath, $outputPath); + } finally { + File::delete($paths); + } + + return $this->readOutputCsv($outputPath); + } +} diff --git a/src/Jobs/ProcessAnnotatedFile.php b/src/Jobs/ProcessAnnotatedFile.php new file mode 100644 index 00000000..200fefd0 --- /dev/null +++ b/src/Jobs/ProcessAnnotatedFile.php @@ -0,0 +1,229 @@ +targetDisk = $targetDisk ?: config('largo.patch_storage_disk'); + } + + /** + * Assemble the target path for an annotation patch. + * + * @param Annotation $annotation + * + * @return string + */ + public static function getTargetPath(Annotation $annotation): string + { + $prefix = fragment_uuid_path($annotation->getFile()->uuid); + $format = config('largo.patch_format'); + + return match($annotation::class) { + // Add "v-" to make absolutely sure that no collisions (same UUID, same ID) + // occur because patches are stored on the same disk. + VideoAnnotation::class => "{$prefix}/v-{$annotation->id}.{$format}", + // This is the old patch storage scheme, so we don't add "i-" for backwards + // compatibility. + default => "{$prefix}/{$annotation->id}.{$format}", + }; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + try { + FileCache::get($this->file, [$this, 'handleFile'], true); + } catch (FileLockedException $e) { + // Retry this job without increasing the attempts if the file is currently + // written by another worker. This worker can process other jobs in the + // meantime. + // See: https://github.com/laravel/ideas/issues/735 + static::dispatch( + $this->file, + $this->only, + $this->skipPatches, + $this->skipFeatureVectors, + $this->targetDisk + ) + ->onConnection($this->connection) + ->onQueue($this->queue) + ->delay(60); + } catch (Exception $e) { + if ($this->shouldRetryAfterException($e)) { + // Exponential backoff for retry after 10 and then 20 minutes. + $this->release($this->attempts() * 600); + } else { + $class = get_class($this->file); + Log::warning("Could not process annotated {$class} {$this->file->id}: {$e->getMessage()}", ['exception' => $e]); + } + } + } + + /** + * Handle a single file. + * + * @param VolumeFile $file + * @param string $path Path to the cached file. + */ + abstract public function handleFile(VolumeFile $file, $path); + + /** + * Determine if this job should retry instead of fail after an exception + * + * @param Exception $e + * + * @return bool + */ + protected function shouldRetryAfterException(Exception $e) + { + $message = $e->getMessage(); + return $this->attempts() < $this->tries && ( + // The remote source might be available again after a while. + Str::contains($message, 'The source resource could not be established') || + // This error presumably occurs due to worker concurrency. + Str::contains($message, 'Impossible to create the root directory') + ); + } + + /** + * Get the annotation patch as buffer. + * + * @param Image $image + * @param array $points + * @param Shape $shape + * + * @return string + */ + protected function getAnnotationPatch($image, $points, $shape) + { + $thumbWidth = config('thumbnails.width'); + $thumbHeight = config('thumbnails.height'); + + if ($shape->id === Shape::wholeFrameId()) { + $image = $image->resize(floatval($thumbWidth) / $image->width); + } else { + $padding = config('largo.patch_padding'); + $pointPadding = config('largo.point_padding'); + + $box = $this->getAnnotationBoundingBox($points, $shape, $pointPadding, $padding); + $box = $this->ensureBoxAspectRatio($box, $thumbWidth, $thumbHeight); + $box = $this->makeBoxContained($box, $image->width, $image->height); + + $image = $image->crop(...$box)->resize(floatval($thumbWidth) / $box[2]); + } + + return $image->writeToBuffer('.'.config('largo.patch_format'), [ + 'Q' => 85, + 'strip' => true, + ]); + } + + /** + * Generates feature vectors for the specified annotations belonging to the file of + * this job. This method either creates new feature vector models or updates the + * existing ones for the annotations. + * + * @param Collection $annotation + * @param array|string $filePath If a string, a file path to the local image to use for feature vector generation. If an array, a map of annotation IDs to a local image file path. + */ + protected function generateFeatureVectors(Collection $annotations, array|string $filePath): void + { + $boxes = $this->generateFileInput($this->file, $annotations); + + if (empty($boxes)) { + return; + } + + $inputPath = tempnam(sys_get_temp_dir(), 'largo_feature_vector_input'); + $outputPath = tempnam(sys_get_temp_dir(), 'largo_feature_vector_output'); + + try { + if (is_array($filePath)) { + $input = []; + foreach ($boxes as $id => $box) { + $path = $filePath[$id]; + if (array_key_exists($path, $input)) { + $input[$path][$id] = $box; + } else { + $input[$path] = [$id => $box]; + } + } + } else { + $input = [$filePath => $boxes]; + } + + File::put($inputPath, json_encode($input)); + $this->python($inputPath, $outputPath); + $output = $this->readOutputCsv($outputPath); + $this->updateOrCreateFeatureVectors($annotations, $output); + } finally { + File::delete($outputPath); + File::delete($inputPath); + } + } + + /** + * Create the feature vectors based on the Python script output. + */ + abstract protected function updateOrCreateFeatureVectors(Collection $annotations, \Generator $output): void; +} diff --git a/src/Jobs/ProcessAnnotatedImage.php b/src/Jobs/ProcessAnnotatedImage.php new file mode 100644 index 00000000..c2efc916 --- /dev/null +++ b/src/Jobs/ProcessAnnotatedImage.php @@ -0,0 +1,101 @@ +skipPatches) { + $image = $this->getVipsImage($path); + } else { + $image = null; + } + + $this->getAnnotationQuery($file) + ->when(!empty($this->only), fn ($q) => $q->whereIn('id', $this->only)) + ->chunkById(1000, function ($annotations) use ($image, $path) { + if (!$this->skipPatches) { + $annotations->each(function ($a) use ($image) { + $buffer = $this->getAnnotationPatch( + $image, + $a->getPoints(), + $a->getShape() + ); + $targetPath = self::getTargetPath($a); + Storage::disk($this->targetDisk)->put($targetPath, $buffer); + }); + } + + if (!$this->skipFeatureVectors) { + $this->generateFeatureVectors($annotations, $path); + } + }); + } + + /** + * Create the feature vectors based on the Python script output. + */ + protected function updateOrCreateFeatureVectors(Collection $annotations, \Generator $output): void + { + $annotations = $annotations->load('labels.label')->keyBy('id'); + foreach ($output as $row) { + $annotation = $annotations->get($row[0]); + + foreach ($annotation->labels as $al) { + ImageAnnotationLabelFeatureVector::updateOrCreate( + ['id' => $al->id], + [ + 'annotation_id' => $annotation->id, + 'label_id' => $al->label_id, + 'label_tree_id' => $al->label->label_tree_id, + 'volume_id' => $this->file->volume_id, + 'vector' => $row[1], + ] + ); + } + } + } + + /** + * Get the vips image instance. + * + * @param string $path + * + * @return \Jcupitt\Vips\Image + */ + protected function getVipsImage($path) + { + // Must not use sequential access because multiple patches could be extracted. + return VipsImage::newFromFile($path); + } + + /** + * Get the query builder for the annotations. + * + * This can be used to extend this class and process different models than image + * annotations. + */ + protected function getAnnotationQuery(VolumeFile $file): Builder + { + return ImageAnnotation::where('image_id', $file->id); + } +} diff --git a/src/Jobs/ProcessAnnotatedVideo.php b/src/Jobs/ProcessAnnotatedVideo.php new file mode 100644 index 00000000..837fb19b --- /dev/null +++ b/src/Jobs/ProcessAnnotatedVideo.php @@ -0,0 +1,123 @@ +getVideo($path); + + VideoAnnotation::where('video_id', $file->id) + ->when(!empty($this->only), fn ($q) => $q->whereIn('id', $this->only)) + ->chunkById(1000, fn ($a) => $this->processAnnotationChunk($a, $video)); + } + + /** + * Process a chunk of annotations of this job's file. + */ + protected function processAnnotationChunk(Collection $annotations, Video $video): void + { + $frameFiles = []; + + try { + foreach ($annotations as $a) { + $points = $a->points[0] ?? null; + $frame = $a->frames[0]; + $videoFrame = $this->getVideoFrame($video, $frame); + + if (!$this->skipPatches) { + $buffer = $this->getAnnotationPatch($videoFrame, $points, $a->shape); + $targetPath = self::getTargetPath($a); + Storage::disk($this->targetDisk)->put($targetPath, $buffer); + } + + if (!$this->skipFeatureVectors && !array_key_exists($frame, $frameFiles)) { + $framePath = tempnam(sys_get_temp_dir(), 'largo_video_frame').'.png'; + $videoFrame->writeToFile($framePath); + $frameFiles[$frame] = $framePath; + } + } + + if (!$this->skipFeatureVectors) { + $annotationFrames = $annotations->mapWithKeys( + fn ($a) => [$a->id => $frameFiles[$a->frames[0]]] + ) + ->toArray(); + + $this->generateFeatureVectors($annotations, $annotationFrames); + } + } finally { + File::delete(array_values($frameFiles)); + } + } + + /** + * Create the feature vectors based on the Python script output. + */ + protected function updateOrCreateFeatureVectors(Collection $annotations, \Generator $output): void + { + $annotations = $annotations->load('labels.label')->keyBy('id'); + foreach ($output as $row) { + $annotation = $annotations->get($row[0]); + + foreach ($annotation->labels as $al) { + VideoAnnotationLabelFeatureVector::updateOrCreate( + ['id' => $al->id], + [ + 'annotation_id' => $annotation->id, + 'label_id' => $al->label_id, + 'label_tree_id' => $al->label->label_tree_id, + 'volume_id' => $this->file->volume_id, + 'vector' => $row[1], + ] + ); + } + } + } + + /** + * Get the FFMpeg video instance. + * + * @param string $path + * + * @return Video + */ + protected function getVideo($path) + { + return FFMpeg::create()->open($path); + } + + /** + * Get a video frame from a specific time as VipsImage object. + * + * @param Video $video + * @param float $time + * + * @return \Jcupitt\Vips\Video + */ + protected function getVideoFrame(Video $video, $time) + { + $buffer = $video->frame(TimeCode::fromSeconds($time))->save(null, false, true); + + return VipsImage::newFromBuffer($buffer); + } +} diff --git a/src/LargoServiceProvider.php b/src/LargoServiceProvider.php index 5d86ace7..99975c44 100644 --- a/src/LargoServiceProvider.php +++ b/src/LargoServiceProvider.php @@ -2,9 +2,11 @@ namespace Biigle\Modules\Largo; +use Biigle\Events\AnnotationLabelAttached; use Biigle\Events\ImagesDeleted; use Biigle\Events\VideosDeleted; use Biigle\ImageAnnotation; +use Biigle\Modules\Largo\Listeners\AttachLabelListener; use Biigle\Modules\Largo\Listeners\ImagesCleanupListener; use Biigle\Modules\Largo\Listeners\VideosCleanupListener; use Biigle\Modules\Largo\Observers\ImageAnnotationObserver; @@ -28,6 +30,7 @@ class LargoServiceProvider extends ServiceProvider public function boot(Modules $modules, Router $router) { $this->loadViewsFrom(__DIR__.'/resources/views', 'largo'); + $this->loadMigrationsFrom(__DIR__.'/Database/migrations'); $this->publishes([ __DIR__.'/public/assets' => public_path('vendor/largo'), @@ -48,6 +51,7 @@ public function boot(Modules $modules, Router $router) VideoAnnotation::observe(new VideoAnnotationObserver); Event::listen(ImagesDeleted::class, ImagesCleanupListener::class); Event::listen(VideosDeleted::class, VideosCleanupListener::class); + Event::listen(AnnotationLabelAttached::class, AttachLabelListener::class); $modules->register('largo', [ 'viewMixins' => [ @@ -95,6 +99,11 @@ public function register() return new \Biigle\Modules\Largo\Console\Commands\MigratePatchStorage; }); $this->commands('command.largo.migrate-patch-storage'); + + $this->app->singleton('command.largo.initialize-feature-vectors', function ($app) { + return new \Biigle\Modules\Largo\Console\Commands\InitializeFeatureVectors; + }); + $this->commands('command.largo.initialize-feature-vectors'); } /** @@ -109,6 +118,7 @@ public function provides() 'command.largo.config', 'command.largo.generate-missing', 'command.largo.migrate-patch-storage', + 'command.largo.initialize-feature-vectors', ]; } } diff --git a/src/Listeners/AttachLabelListener.php b/src/Listeners/AttachLabelListener.php new file mode 100644 index 00000000..2128ad8b --- /dev/null +++ b/src/Listeners/AttachLabelListener.php @@ -0,0 +1,20 @@ +annotationLabel instanceof ImageAnnotationLabel) { + CopyImageAnnotationFeatureVector::dispatch($event->annotationLabel); + } else { + CopyVideoAnnotationFeatureVector::dispatch($event->annotationLabel); + } + } +} diff --git a/src/Observers/ImageAnnotationObserver.php b/src/Observers/ImageAnnotationObserver.php index c61d4687..7e0ccc4a 100644 --- a/src/Observers/ImageAnnotationObserver.php +++ b/src/Observers/ImageAnnotationObserver.php @@ -3,7 +3,7 @@ namespace Biigle\Modules\Largo\Observers; use Biigle\Annotation; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedImage; use Biigle\Modules\Largo\Jobs\RemoveImageAnnotationPatches; class ImageAnnotationObserver extends AnnotationObserver @@ -11,9 +11,9 @@ class ImageAnnotationObserver extends AnnotationObserver /** * {@inheritdoc} */ - protected function getSavedDispatch(Annotation $annotation) + protected function getSavedDispatch(Annotation $a) { - return GenerateImageAnnotationPatch::dispatch($annotation); + return ProcessAnnotatedImage::dispatch($a->image, only: [$a->id]); } /** diff --git a/src/Observers/VideoAnnotationObserver.php b/src/Observers/VideoAnnotationObserver.php index 8e37faae..d26d053a 100644 --- a/src/Observers/VideoAnnotationObserver.php +++ b/src/Observers/VideoAnnotationObserver.php @@ -3,7 +3,7 @@ namespace Biigle\Modules\Largo\Observers; use Biigle\Annotation; -use Biigle\Modules\Largo\Jobs\GenerateVideoAnnotationPatch; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedVideo; use Biigle\Modules\Largo\Jobs\RemoveVideoAnnotationPatches; class VideoAnnotationObserver extends AnnotationObserver @@ -11,9 +11,9 @@ class VideoAnnotationObserver extends AnnotationObserver /** * {@inheritdoc} */ - protected function getSavedDispatch(Annotation $annotation) + protected function getSavedDispatch(Annotation $a) { - return GenerateVideoAnnotationPatch::dispatch($annotation); + return ProcessAnnotatedVideo::dispatch($a->video, only: [$a->id]); } /** diff --git a/src/VideoAnnotationLabelFeatureVector.php b/src/VideoAnnotationLabelFeatureVector.php new file mode 100644 index 00000000..4509501d --- /dev/null +++ b/src/VideoAnnotationLabelFeatureVector.php @@ -0,0 +1,53 @@ + Vector::class, + ]; + + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'id', + 'annotation_id', + 'label_id', + 'label_tree_id', + 'volume_id', + 'vector', + ]; + + /** + * Create a new factory instance for the model. + * + * @return \Illuminate\Database\Eloquent\Factories\Factory + */ + protected static function newFactory() + { + return VideoAnnotationLabelFeatureVectorFactory::new(); + } +} diff --git a/src/config/largo.php b/src/config/largo.php index 35caed40..fea01eed 100644 --- a/src/config/largo.php +++ b/src/config/largo.php @@ -39,4 +39,19 @@ | Specifies which queue should be used for the job to save a Largo session. */ 'apply_session_queue' => env('LARGO_APPLY_SESSION_QUEUE', 'default'), + + /* + | Path to the extract features script. + */ + 'extract_features_script' => __DIR__.'/../resources/scripts/ExtractFeatures.py', + + /* + | Path to the directory to use as Torch Hub cache. + */ + 'torch_hub_path' => storage_path('largo_cache'), + + /* + | Path to the Python executable. + */ + 'python' => env('LARGO_PYTHON', '/usr/bin/python3'), ]; diff --git a/src/public/assets/scripts/main.js b/src/public/assets/scripts/main.js index b3209b3d..4a7c7e63 100644 --- a/src/public/assets/scripts/main.js +++ b/src/public/assets/scripts/main.js @@ -1 +1 @@ -(()=>{"use strict";var e,t={732:()=>{var e="imageAnnotation",t="videoAnnotation";function n(e,t,n,i,s,o,a,l){var r,u="function"==typeof e?e.options:e;if(t&&(u.render=t,u.staticRenderFns=n,u._compiled=!0),i&&(u.functional=!0),o&&(u._scopeId="data-v-"+o),a?(r=function(e){(e=e||this.$vnode&&this.$vnode.ssrContext||this.parent&&this.parent.$vnode&&this.parent.$vnode.ssrContext)||"undefined"==typeof __VUE_SSR_CONTEXT__||(e=__VUE_SSR_CONTEXT__),s&&s.call(this,e),e&&e._registeredComponents&&e._registeredComponents.add(a)},u._ssrRegister=r):s&&(r=l?function(){s.call(this,(u.functional?this.parent:this).$root.$options.shadowRoot)}:s),r)if(u.functional){u._injectStyles=r;var d=u.render;u.render=function(e,t){return r.call(t),d(e,t)}}else{var h=u.beforeCreate;u.beforeCreate=h?[].concat(h,r):[r]}return{exports:e,options:u}}const i=n({computed:{id:function(){return this.image.id},uuid:function(){return this.image.uuid},type:function(){return this.image.type},patchPrefix:function(){return this.uuid[0]+this.uuid[1]+"/"+this.uuid[2]+this.uuid[3]+"/"+this.uuid},urlTemplate:function(){return biigle.$require("largo.patchUrlTemplate")}},methods:{getThumbnailUrl:function(){return this.type===t?this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id","v-".concat(this.id)):this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id",this.id)}},created:function(){this.type===e?this.showAnnotationRoute=biigle.$require("largo.showImageAnnotationRoute"):this.showAnnotationRoute=biigle.$require("largo.showVideoAnnotationRoute")}},undefined,undefined,!1,null,null,null).exports;const s=n({mixins:[i],props:{_id:{type:String,required:!0},_uuid:{type:String,required:!0},label:{type:Object,required:!0},emptySrc:{type:String,required:!0},_urlTemplate:{type:String,required:!0}},data:function(){return{url:""}},computed:{title:function(){return"Example annotation for label "+this.label.name},src:function(){return this.url||this.emptySrc},image:function(){return{id:this._id,uuid:this._uuid,type:e}},urlTemplate:function(){return this._urlTemplate}},methods:{showEmptyImage:function(){this.url=""}},created:function(){this.url=this.getThumbnailUrl()}},undefined,undefined,!1,null,null,null).exports,o=Vue.resource("api/v1/volumes{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/volumes{/id}/video-annotations/filter/label{/label_id}"},queryExampleAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/examples{/label_id}"}});var a=biigle.$require("echo"),l=biigle.$require("events"),r=biigle.$require("messages").handleErrorResponse,u=biigle.$require("volumes.components.imageGrid"),d=biigle.$require("volumes.components.imageGridImage"),h=biigle.$require("annotations.components.labelsTabPlugins"),c=biigle.$require("labelTrees.components.labelTrees"),m=biigle.$require("core.mixins.loader"),f=biigle.$require("messages"),g=biigle.$require("core.components.powerToggle"),p=biigle.$require("annotations.components.settingsTabPlugins"),b=biigle.$require("core.components.sidebar"),v=biigle.$require("core.components.sidebarTab");const w=n({mixins:[m],components:{annotationPatch:s},props:{label:{default:null},volumeId:{type:Number,required:!0},count:{type:Number,default:3}},data:function(){return{exampleLabel:null,exampleAnnotations:[],cache:{},shown:!0}},computed:{isShown:function(){return this.shown&&null!==this.label},hasExamples:function(){return this.exampleLabel&&this.exampleAnnotations&&Object.keys(this.exampleAnnotations).length>0}},methods:{parseResponse:function(e){return e.data},setExampleAnnotations:function(e){(!e[0].hasOwnProperty("annotations")||Object.keys(e[0].annotations).length0},dismissedImageAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(t){return t.type===e})))},dismissedVideoAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(e){return e.type===t})))},changedImageAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(t){return t.type===e})))},changedVideoAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(e){return e.type===t})))},toDeleteCount:function(){return this.dismissedAnnotations.length-this.annotationsWithNewLabel.length},saveButtonClass:function(){return this.forceChange?"btn-danger":"btn-success"}},methods:{getAnnotations:function(e){var t=this;this.annotationsCache.hasOwnProperty(e.id)||(Vue.set(this.annotationsCache,e.id,[]),this.startLoading(),this.queryAnnotations(e).then((function(n){return t.gotAnnotations(e,n)}),r).finally(this.finishLoading))},gotAnnotations:function(n,i){var s=i[0].data,o=i[1].data,a=[];s&&(a=a.concat(this.initAnnotations(n,s,e))),o&&(a=a.concat(this.initAnnotations(n,o,t))),a=a.sort((function(e,t){return t.id-e.id})),Vue.set(this.annotationsCache,n.id,a)},initAnnotations:function(e,t,n){return Object.keys(t).map((function(i){return{id:i,uuid:t[i],label_id:e.id,dismissed:!1,newLabel:null,type:n}}))},handleSelectedLabel:function(e){this.selectedLabel=e,this.isInDismissStep&&this.getAnnotations(e)},handleDeselectedLabel:function(){this.selectedLabel=null},handleSelectedImageDismiss:function(e,t){e.dismissed?(e.dismissed=!1,e.newLabel=null):(e.dismissed=!0,t.shiftKey&&this.lastSelectedImage?this.dismissAllImagesBetween(e,this.lastSelectedImage):this.lastSelectedImage=e)},goToRelabel:function(){this.step=1,this.lastSelectedImage=null},goToDismiss:function(){this.step=0,this.lastSelectedImage=null,this.selectedLabel&&this.getAnnotations(this.selectedLabel)},handleSelectedImageRelabel:function(e,t){e.newLabel?this.selectedLabel&&e.newLabel.id!==this.selectedLabel.id?e.newLabel=this.selectedLabel:e.newLabel=null:this.selectedLabel&&(e.newLabel=this.selectedLabel,t.shiftKey&&this.lastSelectedImage?this.relabelAllImagesBetween(e,this.lastSelectedImage):this.lastSelectedImage=e)},save:function(){var e=this;if(!this.loading){if(this.toDeleteCount>0){for(var t;null!==t&&parseInt(t,10)!==this.toDeleteCount;)t=prompt("This might delete ".concat(this.toDeleteCount," annotation(s). Please enter the number to continue."));if(null===t)return}this.startLoading(),this.performSave({dismissed_image_annotations:this.dismissedImageAnnotationsToSave,changed_image_annotations:this.changedImageAnnotationsToSave,dismissed_video_annotations:this.dismissedVideoAnnotationsToSave,changed_video_annotations:this.changedVideoAnnotationsToSave,force:this.forceChange}).then((function(t){return e.waitForSessionId=t.body.id}),(function(t){e.finishLoading(),r(t)}))}},handleSessionSaved:function(e){if(e.id==this.waitForSessionId){for(var t in this.finishLoading(),f.success("Saved. You can now start a new re-evaluation session."),this.step=0,this.annotationsCache)this.annotationsCache.hasOwnProperty(t)&&delete this.annotationsCache[t];this.handleSelectedLabel(this.selectedLabel)}},handleSessionFailed:function(e){e.id==this.waitForSessionId&&(this.finishLoading(),f.danger("There was an unexpected error."))},performOnAllImagesBetween:function(e,t,n){var i=this.allAnnotations.indexOf(e),s=this.allAnnotations.indexOf(t);if(s=0;n--)t.hasOwnProperty(e[n].label_id)?t[e[n].label_id].push(e[n].id):t[e[n].label_id]=[e[n].id];return t},packChangedToSave:function(e){for(var t={},n=e.length-1;n>=0;n--)t.hasOwnProperty(e[n].newLabel.id)?t[e[n].newLabel.id].push(e[n].id):t[e[n].newLabel.id]=[e[n].id];return t},initializeEcho:function(){a.getInstance().private("user-".concat(this.user.id)).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionSaved",this.handleSessionSaved).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionFailed",this.handleSessionFailed)}},watch:{annotations:function(e){l.$emit("annotations-count",e.length)},dismissedAnnotations:function(e){l.$emit("dismissed-annotations-count",e.length)},step:function(e){l.$emit("step",e)},selectedLabel:function(){this.isInDismissStep&&this.$refs.dismissGrid.setOffset(0)}},created:function(){var e=this;this.user=biigle.$require("largo.user"),window.addEventListener("beforeunload",(function(t){if(e.hasDismissedAnnotations)return t.preventDefault(),t.returnValue="","This page is asking you to confirm that you want to leave - data you have entered may not be saved."})),this.initializeEcho()}},undefined,undefined,!1,null,null,null).exports;const q=n({mixins:[T],components:{catalogImageGrid:y},data:function(){return{labelTrees:[]}},methods:{queryAnnotations:function(e){var t=S.queryImageAnnotations({id:e.id}),n=S.queryVideoAnnotations({id:e.id});return Vue.Promise.all([t,n])}},created:function(){var e=biigle.$require("annotationCatalog.labelTree");this.labelTrees=[e]}},undefined,undefined,!1,null,null,null).exports;const $=n({mixins:[T],data:function(){return{volumeId:null,labelTrees:[],mediaType:""}},methods:{queryAnnotations:function(e){var t,n;return"image"===this.mediaType?(t=o.queryImageAnnotations({id:this.volumeId,label_id:e.id}),n=Vue.Promise.resolve([])):(t=Vue.Promise.resolve([]),n=o.queryVideoAnnotations({id:this.volumeId,label_id:e.id})),Vue.Promise.all([t,n])},performSave:function(e){return o.save({id:this.volumeId},e)}},created:function(){this.volumeId=biigle.$require("largo.volumeId"),this.labelTrees=biigle.$require("largo.labelTrees"),this.mediaType=biigle.$require("largo.mediaType")}},undefined,undefined,!1,null,null,null).exports;const O=n({data:function(){return{step:0,count:0,dismissedCount:0}},computed:{shownCount:function(){return this.isInDismissStep?this.count:this.dismissedCount},isInDismissStep:function(){return 0===this.step},isInRelabelStep:function(){return 1===this.step}},methods:{updateStep:function(e){this.step=e},updateCount:function(e){this.count=e},updateDismissedCount:function(e){this.dismissedCount=e}},created:function(){l.$on("annotations-count",this.updateCount),l.$on("dismissed-annotations-count",this.updateDismissedCount),l.$on("step",this.updateStep)}},undefined,undefined,!1,null,null,null).exports,E=Vue.resource("api/v1/projects{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/projects{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/projects{/id}/video-annotations/filter/label{/label_id}"}});const k=n({mixins:[T],data:function(){return{projectId:null,labelTrees:[]}},methods:{queryAnnotations:function(e){var t=E.queryImageAnnotations({id:this.projectId,label_id:e.id}),n=E.queryVideoAnnotations({id:this.projectId,label_id:e.id});return Vue.Promise.all([t,n])},performSave:function(e){return E.save({id:this.projectId},e)}},created:function(){this.projectId=biigle.$require("largo.projectId"),this.labelTrees=biigle.$require("largo.labelTrees")}},undefined,undefined,!1,null,null,null).exports;biigle.$mount("annotation-catalog-container",q),biigle.$mount("largo-container",$),biigle.$mount("largo-title",O),biigle.$mount("project-largo-container",k)},401:()=>{}},n={};function i(e){var s=n[e];if(void 0!==s)return s.exports;var o=n[e]={exports:{}};return t[e](o,o.exports,i),o.exports}i.m=t,e=[],i.O=(t,n,s,o)=>{if(!n){var a=1/0;for(d=0;d=o)&&Object.keys(i.O).every((e=>i.O[e](n[r])))?n.splice(r--,1):(l=!1,o0&&e[d-1][2]>o;d--)e[d]=e[d-1];e[d]=[n,s,o]},i.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),(()=>{var e={355:0,392:0};i.O.j=t=>0===e[t];var t=(t,n)=>{var s,o,[a,l,r]=n,u=0;if(a.some((t=>0!==e[t]))){for(s in l)i.o(l,s)&&(i.m[s]=l[s]);if(r)var d=r(i)}for(t&&t(n);ui(732)));var s=i.O(void 0,[392],(()=>i(401)));s=i.O(s)})(); \ No newline at end of file +(()=>{"use strict";var t,e={486:()=>{var t="imageAnnotation",e="videoAnnotation";function n(t,e,n,i,s,o,a,r){var l,u="function"==typeof t?t.options:t;if(e&&(u.render=e,u.staticRenderFns=n,u._compiled=!0),i&&(u.functional=!0),o&&(u._scopeId="data-v-"+o),a?(l=function(t){(t=t||this.$vnode&&this.$vnode.ssrContext||this.parent&&this.parent.$vnode&&this.parent.$vnode.ssrContext)||"undefined"==typeof __VUE_SSR_CONTEXT__||(t=__VUE_SSR_CONTEXT__),s&&s.call(this,t),t&&t._registeredComponents&&t._registeredComponents.add(a)},u._ssrRegister=l):s&&(l=r?function(){s.call(this,(u.functional?this.parent:this).$root.$options.shadowRoot)}:s),l)if(u.functional){u._injectStyles=l;var d=u.render;u.render=function(t,e){return l.call(e),d(t,e)}}else{var c=u.beforeCreate;u.beforeCreate=c?[].concat(c,l):[l]}return{exports:t,options:u}}const i=n({computed:{id:function(){return this.image.id},uuid:function(){return this.image.uuid},type:function(){return this.image.type},patchPrefix:function(){return this.uuid[0]+this.uuid[1]+"/"+this.uuid[2]+this.uuid[3]+"/"+this.uuid},urlTemplate:function(){return biigle.$require("largo.patchUrlTemplate")}},methods:{getThumbnailUrl:function(){return this.type===e?this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id","v-".concat(this.id)):this.urlTemplate.replace(":prefix",this.patchPrefix).replace(":id",this.id)}},created:function(){this.type===t?this.showAnnotationRoute=biigle.$require("largo.showImageAnnotationRoute"):this.showAnnotationRoute=biigle.$require("largo.showVideoAnnotationRoute")}},undefined,undefined,!1,null,null,null).exports;const s=n({mixins:[i],props:{_id:{type:String,required:!0},_uuid:{type:String,required:!0},label:{type:Object,required:!0},emptySrc:{type:String,required:!0},_urlTemplate:{type:String,required:!0}},data:function(){return{url:""}},computed:{title:function(){return"Example annotation for label "+this.label.name},src:function(){return this.url||this.emptySrc},image:function(){return{id:this._id,uuid:this._uuid,type:t}},urlTemplate:function(){return this._urlTemplate}},methods:{showEmptyImage:function(){this.url=""}},created:function(){this.url=this.getThumbnailUrl()}},undefined,undefined,!1,null,null,null).exports,o=Vue.resource("api/v1/volumes{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/volumes{/id}/video-annotations/filter/label{/label_id}"},queryExampleAnnotations:{method:"GET",url:"api/v1/volumes{/id}/image-annotations/examples{/label_id}"},sortAnnotationsByOutlier:{method:"GET",url:"api/v1/volumes{/id}/annotations/sort/outliers{/label_id}"}});var a=biigle.$require("echo"),r=biigle.$require("events"),l=biigle.$require("messages").handleErrorResponse,u=biigle.$require("volumes.components.imageGrid"),d=biigle.$require("volumes.components.imageGridImage"),c=biigle.$require("annotations.components.labelsTabPlugins"),h=biigle.$require("labelTrees.components.labelTrees"),m=biigle.$require("core.mixins.loader"),g=biigle.$require("messages"),f=biigle.$require("core.components.powerToggle"),p=biigle.$require("annotations.components.settingsTabPlugins"),b=biigle.$require("core.components.sidebar"),v=biigle.$require("core.components.sidebarTab");const y=n({mixins:[m],components:{annotationPatch:s},props:{label:{default:null},volumeId:{type:Number,required:!0},count:{type:Number,default:3}},data:function(){return{exampleLabel:null,exampleAnnotations:[],cache:{},shown:!0}},computed:{isShown:function(){return this.shown&&null!==this.label},hasExamples:function(){return this.exampleLabel&&this.exampleAnnotations&&Object.keys(this.exampleAnnotations).length>0}},methods:{parseResponse:function(t){return t.data},setExampleAnnotations:function(t){(!t[0].hasOwnProperty("annotations")||Object.keys(t[0].annotations).length0},dismissedImageAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(e){return e.type===t})))},dismissedVideoAnnotationsToSave:function(){return this.packDismissedToSave(this.dismissedAnnotations.filter((function(t){return t.type===e})))},changedImageAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(e){return e.type===t})))},changedVideoAnnotationsToSave:function(){return this.packChangedToSave(this.annotationsWithNewLabel.filter((function(t){return t.type===e})))},toDeleteCount:function(){return this.dismissedAnnotations.length-this.annotationsWithNewLabel.length},saveButtonClass:function(){return this.forceChange?"btn-danger":"btn-success"},sortingIsActive:function(){return this.isInDismissStep&&(this.sortingKey!==$||this.sortingDirection!==T)}},methods:{getAnnotations:function(t){var e,n,i,s,o=this;this.annotationsCache.hasOwnProperty(t.id)?i=Vue.Promise.resolve():(Vue.set(this.annotationsCache,t.id,[]),this.startLoading(),i=this.queryAnnotations(t).then((function(e){return o.gotAnnotations(t,e)}),l));var a=null===(e=this.sortingSequenceCache)||void 0===e||null===(n=e[t.id])||void 0===n?void 0:n[this.sortingKey];this.sortingIsActive&&!a?(this.loading||this.startLoading(),s=this.fetchSortingSequence(this.sortingKey,t.id).catch(l)):s=Vue.Promise.resolve(),Vue.Promise.all([i,s]).finally(this.finishLoading)},gotAnnotations:function(n,i){var s=i[0].data,o=i[1].data,a=[];s&&(a=a.concat(this.initAnnotations(n,s,t))),o&&(a=a.concat(this.initAnnotations(n,o,e))),a=a.sort((function(t,e){return e.id-t.id})),Vue.set(this.annotationsCache,n.id,a)},initAnnotations:function(t,e,n){return Object.keys(e).map((function(i){return{id:i,uuid:e[i],label_id:t.id,dismissed:!1,newLabel:null,type:n}}))},handleSelectedLabel:function(t){this.selectedLabel=t,this.isInDismissStep&&this.getAnnotations(t)},handleDeselectedLabel:function(){this.selectedLabel=null},handleSelectedImageDismiss:function(t,e){t.dismissed?(t.dismissed=!1,t.newLabel=null):(t.dismissed=!0,e.shiftKey&&this.lastSelectedImage?this.dismissAllImagesBetween(t,this.lastSelectedImage):this.lastSelectedImage=t)},goToRelabel:function(){this.step=1,this.lastSelectedImage=null},goToDismiss:function(){this.step=0,this.lastSelectedImage=null,this.selectedLabel&&this.getAnnotations(this.selectedLabel)},handleSelectedImageRelabel:function(t,e){t.newLabel?this.selectedLabel&&t.newLabel.id!==this.selectedLabel.id?t.newLabel=this.selectedLabel:t.newLabel=null:this.selectedLabel&&(t.newLabel=this.selectedLabel,e.shiftKey&&this.lastSelectedImage?this.relabelAllImagesBetween(t,this.lastSelectedImage):this.lastSelectedImage=t)},save:function(){var t=this;if(!this.loading){if(this.toDeleteCount>0){for(var e;null!==e&&parseInt(e,10)!==this.toDeleteCount;)e=prompt("This might delete ".concat(this.toDeleteCount," annotation(s). Please enter the number to continue."));if(null===e)return}this.startLoading(),this.performSave({dismissed_image_annotations:this.dismissedImageAnnotationsToSave,changed_image_annotations:this.changedImageAnnotationsToSave,dismissed_video_annotations:this.dismissedVideoAnnotationsToSave,changed_video_annotations:this.changedVideoAnnotationsToSave,force:this.forceChange}).then((function(e){return t.waitForSessionId=e.body.id}),(function(e){t.finishLoading(),l(e)}))}},handleSessionSaved:function(t){if(t.id==this.waitForSessionId){for(var e in this.finishLoading(),g.success("Saved. You can now start a new re-evaluation session."),this.step=0,this.annotationsCache)this.annotationsCache.hasOwnProperty(e)&&delete this.annotationsCache[e];for(var n in this.sortingSequenceCache)this.sortingSequenceCache.hasOwnProperty(n)&&delete this.sortingSequenceCache[n];this.handleSelectedLabel(this.selectedLabel)}},handleSessionFailed:function(t){t.id==this.waitForSessionId&&(this.finishLoading(),g.danger("There was an unexpected error."))},performOnAllImagesBetween:function(t,e,n){var i=this.allAnnotations.indexOf(t),s=this.allAnnotations.indexOf(e);if(s=0;n--)e.hasOwnProperty(t[n].label_id)?e[t[n].label_id].push(t[n].id):e[t[n].label_id]=[t[n].id];return e},packChangedToSave:function(t){for(var e={},n=t.length-1;n>=0;n--)e.hasOwnProperty(t[n].newLabel.id)?e[t[n].newLabel.id].push(t[n].id):e[t[n].newLabel.id]=[t[n].id];return e},initializeEcho:function(){a.getInstance().private("user-".concat(this.user.id)).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionSaved",this.handleSessionSaved).listen(".Biigle\\Modules\\Largo\\Events\\LargoSessionFailed",this.handleSessionFailed)},updateSortDirection:function(t){this.sortingDirection=t},fetchSortingSequence:function(t,e){var n=this;return(t===O?this.querySortByOutlier(e).then((function(t){return t.body})):Vue.Promise.resolve([])).then((function(i){return n.putSortingSequenceToCache(t,e,i)}))},putSortingSequenceToCache:function(t,e,n){this.sortingSequenceCache[e]||Vue.set(this.sortingSequenceCache,e,{}),this.sortingSequenceCache[e][t]=n},updateSortKey:function(t){var e,n,i,s=this,o=null===(e=this.selectedLabel)||void 0===e?void 0:e.id,a=null===(n=this.sortingSequenceCache)||void 0===n||null===(i=n[o])||void 0===i?void 0:i[t];o&&!a?(this.startLoading(),this.fetchSortingSequence(t,o).then((function(){return s.sortingKey=t})).catch(l).finally(this.finishLoading)):this.sortingKey=t}},watch:{annotations:function(t){r.$emit("annotations-count",t.length)},dismissedAnnotations:function(t){r.$emit("dismissed-annotations-count",t.length)},step:function(t){r.$emit("step",t)},selectedLabel:function(){this.isInDismissStep&&this.$refs.dismissGrid.setOffset(0)}},created:function(){var t=this;this.user=biigle.$require("largo.user"),window.addEventListener("beforeunload",(function(e){if(t.hasDismissedAnnotations)return e.preventDefault(),e.returnValue="","This page is asking you to confirm that you want to leave - data you have entered may not be saved."})),this.initializeEcho()}},undefined,undefined,!1,null,null,null).exports;const D=n({mixins:[E],components:{catalogImageGrid:S},data:function(){return{labelTrees:[]}},methods:{queryAnnotations:function(t){var e=_.queryImageAnnotations({id:t.id}),n=_.queryVideoAnnotations({id:t.id});return Vue.Promise.all([e,n])}},created:function(){var t=biigle.$require("annotationCatalog.labelTree");this.labelTrees=[t]}},undefined,undefined,!1,null,null,null).exports;const R=n({mixins:[E],data:function(){return{volumeId:null,labelTrees:[],mediaType:""}},methods:{queryAnnotations:function(t){var e,n;return"image"===this.mediaType?(e=o.queryImageAnnotations({id:this.volumeId,label_id:t.id}),n=Vue.Promise.resolve([])):(e=Vue.Promise.resolve([]),n=o.queryVideoAnnotations({id:this.volumeId,label_id:t.id})),Vue.Promise.all([e,n])},performSave:function(t){return o.save({id:this.volumeId},t)},querySortByOutlier:function(t){var e=this;return o.sortAnnotationsByOutlier({id:this.volumeId,label_id:t}).then((function(t){return"image"===e.mediaType?t.body=t.body.map((function(t){return"i"+t})):t.body=t.body.map((function(t){return"v"+t})),t}))}},created:function(){this.volumeId=biigle.$require("largo.volumeId"),this.labelTrees=biigle.$require("largo.labelTrees"),this.mediaType=biigle.$require("largo.mediaType")}},undefined,undefined,!1,null,null,null).exports;const V=n({data:function(){return{step:0,count:0,dismissedCount:0}},computed:{shownCount:function(){return this.isInDismissStep?this.count:this.dismissedCount},isInDismissStep:function(){return 0===this.step},isInRelabelStep:function(){return 1===this.step}},methods:{updateStep:function(t){this.step=t},updateCount:function(t){this.count=t},updateDismissedCount:function(t){this.dismissedCount=t}},created:function(){r.$on("annotations-count",this.updateCount),r.$on("dismissed-annotations-count",this.updateDismissedCount),r.$on("step",this.updateStep)}},undefined,undefined,!1,null,null,null).exports,P=Vue.resource("api/v1/projects{/id}/largo",{},{queryImageAnnotations:{method:"GET",url:"api/v1/projects{/id}/image-annotations/filter/label{/label_id}"},queryVideoAnnotations:{method:"GET",url:"api/v1/projects{/id}/video-annotations/filter/label{/label_id}"},sortAnnotationsByOutlier:{method:"GET",url:"api/v1/projects{/id}/annotations/sort/outliers{/label_id}"}});const B=n({mixins:[E],data:function(){return{projectId:null,labelTrees:[]}},methods:{queryAnnotations:function(t){var e=P.queryImageAnnotations({id:this.projectId,label_id:t.id}),n=P.queryVideoAnnotations({id:this.projectId,label_id:t.id});return Vue.Promise.all([e,n])},performSave:function(t){return P.save({id:this.projectId},t)},querySortByOutlier:function(t){return P.sortAnnotationsByOutlier({id:this.projectId,label_id:t})}},created:function(){this.projectId=biigle.$require("largo.projectId"),this.labelTrees=biigle.$require("largo.labelTrees")}},undefined,undefined,!1,null,null,null).exports;biigle.$mount("annotation-catalog-container",D),biigle.$mount("largo-container",R),biigle.$mount("largo-title",V),biigle.$mount("project-largo-container",B)},401:()=>{}},n={};function i(t){var s=n[t];if(void 0!==s)return s.exports;var o=n[t]={exports:{}};return e[t](o,o.exports,i),o.exports}i.m=e,t=[],i.O=(e,n,s,o)=>{if(!n){var a=1/0;for(d=0;d=o)&&Object.keys(i.O).every((t=>i.O[t](n[l])))?n.splice(l--,1):(r=!1,o0&&t[d-1][2]>o;d--)t[d]=t[d-1];t[d]=[n,s,o]},i.o=(t,e)=>Object.prototype.hasOwnProperty.call(t,e),(()=>{var t={355:0,392:0};i.O.j=e=>0===t[e];var e=(e,n)=>{var s,o,[a,r,l]=n,u=0;if(a.some((e=>0!==t[e]))){for(s in r)i.o(r,s)&&(i.m[s]=r[s]);if(l)var d=l(i)}for(e&&e(n);ui(486)));var s=i.O(void 0,[392],(()=>i(401)));s=i.O(s)})(); \ No newline at end of file diff --git a/src/public/assets/styles/main.css b/src/public/assets/styles/main.css index 232cfe9b..d6499f1c 100644 --- a/src/public/assets/styles/main.css +++ b/src/public/assets/styles/main.css @@ -1 +1 @@ -.navbar-text.navbar-largo-breadcrumbs,.navbar-text.navbar-largo-breadcrumbs a{color:#ccc}.largo-images__alerts{align-items:center;display:flex;height:100%;justify-content:center;left:0;pointer-events:none;position:absolute;top:0;transition:background-color .25s ease;width:100%;z-index:2}.largo-images__alerts.block{background-color:rgba(0,0,0,.5);pointer-events:auto}.largo-tab{display:flex!important;flex-direction:column}.largo-tab .largo-tab__button{padding-bottom:10px}.largo-tab .largo-tab__label-trees{flex:1;overflow:auto}.image-grid__image--catalog>a{width:100%}.image-grid__image--relabel .new-label{bottom:0;left:0;overflow-x:hidden;padding:.5em;pointer-events:none;position:absolute;text-align:center;text-overflow:ellipsis;white-space:nowrap;width:100%}.image-grid__image--relabel .new-label__name{padding-right:16px}.image-grid__image--relabel .new-label__color{border:2px solid #eee;border-radius:50%;display:inline-block;height:1em;vertical-align:middle;width:1em}.largo-example-annotations .largo-example-annotations__images{margin-right:-1em;overflow-x:auto;white-space:nowrap}.largo-example-annotations .largo-example-annotation{image-orientation:none;height:125px;width:auto}.largo-example-annotations .largo-example-annotation:not(:last-child){margin-right:.5em}.largo-example-annotations .alert{margin-bottom:0} +.navbar-text.navbar-largo-breadcrumbs,.navbar-text.navbar-largo-breadcrumbs a{color:#ccc}.largo-images__alerts{align-items:center;display:flex;height:100%;justify-content:center;left:0;pointer-events:none;position:absolute;top:0;transition:background-color .25s ease;width:100%;z-index:2}.largo-images__alerts.block{background-color:rgba(0,0,0,.5);pointer-events:auto}.sidebar__tab--open.largo-tab{display:flex!important;flex-direction:column}.sidebar__tab--open.largo-tab .largo-tab__button{padding-bottom:10px}.sidebar__tab--open.largo-tab .largo-tab__label-trees{flex:1;overflow:auto}.image-grid__image--catalog>a{width:100%}.image-grid__image--relabel .new-label{bottom:0;left:0;overflow-x:hidden;padding:.5em;pointer-events:none;position:absolute;text-align:center;text-overflow:ellipsis;white-space:nowrap;width:100%}.image-grid__image--relabel .new-label__name{padding-right:16px}.image-grid__image--relabel .new-label__color{border:2px solid #eee;border-radius:50%;display:inline-block;height:1em;vertical-align:middle;width:1em}.largo-example-annotations .largo-example-annotations__images{margin-right:-1em;overflow-x:auto;white-space:nowrap}.largo-example-annotations .largo-example-annotation{image-orientation:none;height:125px;width:auto}.largo-example-annotations .largo-example-annotation:not(:last-child){margin-right:.5em}.largo-example-annotations .alert{margin-bottom:0} diff --git a/src/public/mix-manifest.json b/src/public/mix-manifest.json index 1b90a3e9..10af68e1 100644 --- a/src/public/mix-manifest.json +++ b/src/public/mix-manifest.json @@ -1,4 +1,4 @@ { - "/assets/scripts/main.js": "/assets/scripts/main.js?id=7dc83b19fd9d27eb012fa7ccdf3ffa24", - "/assets/styles/main.css": "/assets/styles/main.css?id=4be2d80b899c739e0bed700e0ca8362c" + "/assets/scripts/main.js": "/assets/scripts/main.js?id=40a936c2f7f3e8aa294df21e41289d94", + "/assets/styles/main.css": "/assets/styles/main.css?id=4bcd521568b8ea929efd8e3f72f7d338" } diff --git a/src/resources/assets/js/api/projects.js b/src/resources/assets/js/api/projects.js index adee0478..5cbb196e 100644 --- a/src/resources/assets/js/api/projects.js +++ b/src/resources/assets/js/api/projects.js @@ -23,4 +23,8 @@ export default Vue.resource('api/v1/projects{/id}/largo', {}, { method: 'GET', url: 'api/v1/projects{/id}/video-annotations/filter/label{/label_id}', }, + sortAnnotationsByOutlier: { + method: 'GET', + url: 'api/v1/projects{/id}/annotations/sort/outliers{/label_id}', + }, }); diff --git a/src/resources/assets/js/api/volumes.js b/src/resources/assets/js/api/volumes.js index f95fb4da..62542e72 100644 --- a/src/resources/assets/js/api/volumes.js +++ b/src/resources/assets/js/api/volumes.js @@ -31,4 +31,8 @@ export default Vue.resource('api/v1/volumes{/id}/largo', {}, { method: 'GET', url: 'api/v1/volumes{/id}/image-annotations/examples{/label_id}', }, + sortAnnotationsByOutlier: { + method: 'GET', + url: 'api/v1/volumes{/id}/annotations/sort/outliers{/label_id}', + }, }); diff --git a/src/resources/assets/js/components/sortingTab.vue b/src/resources/assets/js/components/sortingTab.vue new file mode 100644 index 00000000..c28c3013 --- /dev/null +++ b/src/resources/assets/js/components/sortingTab.vue @@ -0,0 +1,118 @@ + + + + diff --git a/src/resources/assets/js/largoContainer.vue b/src/resources/assets/js/largoContainer.vue index d972ea2c..c5e8b3c0 100644 --- a/src/resources/assets/js/largoContainer.vue +++ b/src/resources/assets/js/largoContainer.vue @@ -31,6 +31,20 @@ export default { performSave(payload) { return VolumesApi.save({id: this.volumeId}, payload); }, + querySortByOutlier(labelId) { + return VolumesApi.sortAnnotationsByOutlier({id: this.volumeId, label_id: labelId}) + .then((response) => { + // The sorting expects annotation IDs prefixed with 'i' or 'v' so it + // can work with mixed image and video annotations. + if (this.mediaType === 'image') { + response.body = response.body.map(id => 'i' + id); + } else { + response.body = response.body.map(id => 'v' + id); + } + + return response; + }); + }, }, created() { this.volumeId = biigle.$require('largo.volumeId'); diff --git a/src/resources/assets/js/mixins/largoContainer.vue b/src/resources/assets/js/mixins/largoContainer.vue index 62b820be..6f418768 100644 --- a/src/resources/assets/js/mixins/largoContainer.vue +++ b/src/resources/assets/js/mixins/largoContainer.vue @@ -1,6 +1,7 @@