diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88c6d68..46fe5b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,7 @@ jobs: composer config repositories.module --json '{"type": "path", "url": "'${GITHUB_WORKSPACE}'", "options": {"symlink": false}}' composer require --no-ansi --no-interaction --no-scripts --ignore-platform-reqs ${GITHUB_REPOSITORY}:@dev sed -i "/Insert Biigle module service providers/i Biigle\\\\Modules\\\\${MODULE_NAME}\\\\${MODULE_NAME}ServiceProvider::class," config/app.php + sed -i "/Insert Biigle module service providers/i Biigle\\\\Modules\\\\Largo\\\\LargoServiceProvider::class," config/app.php mkdir -p tests/php/Modules ln -sf ../../../vendor/${GITHUB_REPOSITORY}/tests tests/php/Modules/${MODULE_NAME} working-directory: ../core diff --git a/requirements.txt b/requirements.txt index 9962a48..7695b69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -torch==2.0.* -torchvision==0.15.* +torch==2.1.* +torchvision==0.16.* mmcv==2.0.* # Update CUDA Version if necessary. # See: https://mmcv.readthedocs.io/en/latest/get_started/installation.html#install-with-pip @@ -8,5 +8,3 @@ mmdet==3.1.* albumentations scikit-learn scikit-image -Pillow==10.2.0 -xformers==0.0.18 diff --git a/src/Http/Controllers/Api/AnnotationCandidateController.php b/src/Http/Controllers/Api/AnnotationCandidateController.php index 4b1d095..2aea02d 100644 --- a/src/Http/Controllers/Api/AnnotationCandidateController.php +++ b/src/Http/Controllers/Api/AnnotationCandidateController.php @@ -3,11 +3,11 @@ namespace Biigle\Modules\Maia\Http\Controllers\Api; use Biigle\Http\Controllers\Api\Controller; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; use Biigle\Modules\Maia\AnnotationCandidateFeatureVector; use Biigle\Modules\Maia\Http\Requests\SubmitAnnotationCandidates; use Biigle\Modules\Maia\Http\Requests\UpdateAnnotationCandidate; use Biigle\Modules\Maia\Jobs\ConvertAnnotationCandidates; +use Biigle\Modules\Maia\Jobs\ProcessObjectDetectedImage; use Biigle\Modules\Maia\MaiaJob; use Illuminate\Http\Response; use Pgvector\Laravel\Distance; @@ -150,17 +150,21 @@ public function submit(SubmitAnnotationCandidates $request) */ public function update(UpdateAnnotationCandidate $request) { + $candidate = $request->candidate; if ($request->filled('points')) { - $request->candidate->points = $request->input('points'); - $disk = config('maia.annotation_candidate_storage_disk'); - GenerateImageAnnotationPatch::dispatch($request->candidate, $disk) + $candidate->points = $request->input('points'); + ProcessObjectDetectedImage::dispatch($candidate->image, + only: [$candidate->id], + maiaJob: $candidate->job, + targetDisk: config('maia.annotation_candidate_storage_disk') + ) ->onQueue(config('largo.generate_annotation_patch_queue')); } if ($request->has('label_id')) { - $request->candidate->label_id = $request->input('label_id'); + $candidate->label_id = $request->input('label_id'); } - $request->candidate->save(); + $candidate->save(); } } diff --git a/src/Http/Controllers/Api/TrainingProposalController.php b/src/Http/Controllers/Api/TrainingProposalController.php index 2b13fc1..2cda06e 100644 --- a/src/Http/Controllers/Api/TrainingProposalController.php +++ b/src/Http/Controllers/Api/TrainingProposalController.php @@ -3,10 +3,10 @@ namespace Biigle\Modules\Maia\Http\Controllers\Api; use Biigle\Http\Controllers\Api\Controller; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; use Biigle\Modules\Maia\Events\MaiaJobContinued; use Biigle\Modules\Maia\Http\Requests\ContinueMaiaJob; use Biigle\Modules\Maia\Http\Requests\UpdateTrainingProposal; +use Biigle\Modules\Maia\Jobs\ProcessNoveltyDetectedImage; use Biigle\Modules\Maia\MaiaJob; use Biigle\Modules\Maia\MaiaJobState as State; use Biigle\Modules\Maia\TrainingProposalFeatureVector; @@ -149,14 +149,19 @@ public function submit(ContinueMaiaJob $request) */ public function update(UpdateTrainingProposal $request) { + $proposal = $request->proposal; if ($request->filled('points')) { - $request->proposal->points = $request->input('points'); - $disk = config('maia.training_proposal_storage_disk'); - GenerateImageAnnotationPatch::dispatch($request->proposal, $disk) + $proposal->points = $request->input('points'); + ProcessNoveltyDetectedImage::dispatch($proposal->image, + only: [$proposal->id], + maiaJob: $proposal->job, + targetDisk: config('maia.training_proposal_storage_disk') + ) ->onQueue(config('largo.generate_annotation_patch_queue')); } - $request->proposal->selected = $request->input('selected', $request->proposal->selected); - $request->proposal->save(); + $proposal->selected = $request->input('selected', $proposal->selected); + + $proposal->save(); } } diff --git a/src/Jobs/ConvertAnnotationCandidates.php b/src/Jobs/ConvertAnnotationCandidates.php index 02f7ea9..a5d36b1 100644 --- a/src/Jobs/ConvertAnnotationCandidates.php +++ b/src/Jobs/ConvertAnnotationCandidates.php @@ -5,12 +5,13 @@ use Biigle\ImageAnnotation; use Biigle\ImageAnnotationLabel; use Biigle\Jobs\Job; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedImage; use Biigle\Modules\Maia\MaiaJob; use Biigle\User; use Carbon\Carbon; use DB; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Collection; class ConvertAnnotationCandidates extends Job { @@ -47,9 +48,9 @@ class ConvertAnnotationCandidates extends Job /** * Newly created annotations. * - * @var array + * @var Collection */ - protected $newAnnotations = []; + protected $newAnnotations; /** * Create a new isntance. @@ -62,6 +63,7 @@ public function __construct(MaiaJob $job, User $user) $this->queue = config('maia.convert_annotations_queue'); $this->job = $job; $this->user = $user; + $this->newAnnotations = collect([]); } /** @@ -79,10 +81,15 @@ public function handle() ->chunkById(10000, [$this, 'processChunk']); }); - foreach ($this->newAnnotations as $annotation) { - GenerateImageAnnotationPatch::dispatch($annotation) - ->onQueue(config('largo.generate_annotation_patch_queue')); - } + $this->newAnnotations + ->groupBy('image_id') + ->each(function ($group) { + $image = $group[0]->image; + $ids = $group->pluck('id')->all(); + ProcessAnnotatedImage::dispatch($image, only: $ids) + ->onQueue(config('largo.generate_annotation_patch_queue')); + }); + } finally { $this->job->convertingCandidates = false; $this->job->save(); diff --git a/src/Jobs/GenerateAnnotationCandidatePatches.php b/src/Jobs/GenerateAnnotationCandidatePatches.php index 74a7273..57904ae 100644 --- a/src/Jobs/GenerateAnnotationCandidatePatches.php +++ b/src/Jobs/GenerateAnnotationCandidatePatches.php @@ -2,23 +2,46 @@ namespace Biigle\Modules\Maia\Jobs; -class GenerateAnnotationCandidatePatches extends GenerateAnnotationPatches +use Biigle\Jobs\Job; +use Biigle\Modules\Maia\MaiaJob; +use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Queue\SerializesModels; + +class GenerateAnnotationCandidatePatches extends Job implements ShouldQueue { + use SerializesModels; + /** - * Get a query for the annotations that have been created by this job. + * Ignore this job if the MAIA job does not exist any more. * - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo + * @var bool */ - protected function getCreatedAnnotations() - { - return $this->job->annotationCandidates(); - } + protected $deleteWhenMissingModels = true; /** - * Get the storage disk to store the annotation patches to. + * Create a new isntance. */ - protected function getPatchStorageDisk() + public function __construct(public MaiaJob $maiaJob) + { + // + } + + public function handle(): void { - return config('maia.annotation_candidate_storage_disk'); + $this->maiaJob->volume->images() + ->whereExists(fn ($q) => + $q->select(\DB::raw(1)) + ->from('maia_annotation_candidates') + ->where('maia_annotation_candidates.job_id', $this->maiaJob->id) + ->whereColumn('maia_annotation_candidates.image_id', 'images.id') + ) + ->eachById(fn ($image) => + ProcessObjectDetectedImage::dispatch($image, $this->maiaJob, + // Feature vectors are generated in a separate job on the GPU. + skipFeatureVectors: true, + targetDisk: config('maia.annotation_candidate_storage_disk') + ) + ->onQueue(config('largo.generate_annotation_patch_queue')) + ); } } diff --git a/src/Jobs/GenerateAnnotationFeatureVectors.php b/src/Jobs/GenerateAnnotationFeatureVectors.php index bf9688b..140e4d6 100644 --- a/src/Jobs/GenerateAnnotationFeatureVectors.php +++ b/src/Jobs/GenerateAnnotationFeatureVectors.php @@ -4,17 +4,15 @@ use Biigle\Image; use Biigle\Jobs\Job; +use Biigle\Modules\Largo\Jobs\GenerateFeatureVectors; use Biigle\Modules\Maia\MaiaAnnotation; use Biigle\Modules\Maia\MaiaJob; use Biigle\Shape; use FileCache; -use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\File; -use Illuminate\Support\Facades\Process; -use SplFileObject; -abstract class GenerateAnnotationFeatureVectors extends Job implements ShouldQueue +abstract class GenerateAnnotationFeatureVectors extends GenerateFeatureVectors { use SerializesModels; @@ -25,15 +23,6 @@ abstract class GenerateAnnotationFeatureVectors extends Job implements ShouldQue */ const INSERT_CHUNK_SIZE = 1000; - /** - * The "radius" of the bounding box around a point annotation. - * - * This is half the patch size of 224 that is expected by DINO. - * - * @var int - */ - const POINT_PADDING = 112; - /** * Ignore this job if the MAIA job does not exist any more. * @@ -60,22 +49,25 @@ public function handle() $annotations = $this->getAnnotations(); $imageIds = $annotations->pluck('image_id')->unique(); $images = Image::whereIn('id', $imageIds) - ->with('volume') + ->with('volume') // Required to efficiently determine the full image URL. ->get() ->all(); + $inputPath = tempnam(sys_get_temp_dir(), 'maia_feature_vector_input'); $outputPath = tempnam(sys_get_temp_dir(), 'maia_feature_vector_output'); try { - FileCache::batch($images, function ($images, $paths) use ($annotations, $outputPath) { + FileCache::batch($images, function ($images, $paths) use ($annotations, $inputPath, $outputPath) { + $annotations = $annotations->groupBy('image_id'); $input = $this->generateInput($images, $paths, $annotations); if (!empty($input)) { - $this->python($input, $outputPath); + File::put($inputPath, json_encode($input)); + $this->python($inputPath, $outputPath); } }); $insert = []; - foreach ($this->readOuputCsv($outputPath) as $row) { + foreach ($this->readOutputCsv($outputPath) as $row) { $insert[] = [ 'id' => $row[0], 'job_id' => $this->job->id, @@ -92,6 +84,7 @@ public function handle() $this->insertFeatureVectorModelChunk($insert); } finally { File::delete($outputPath); + File::delete($inputPath); } } @@ -106,196 +99,4 @@ abstract protected function getAnnotations(); * Insert a chunk of new feature vector models. */ abstract protected function insertFeatureVectorModelChunk(array $chunk): void; - - /** - * Generate the input for the python script. - * - * @param \Illuminate\Support\Collection $annotations - */ - protected function generateInput(array $images, array $paths, $annotations): array - { - $annotations = $annotations->groupBy('image_id'); - $input = []; - - foreach ($images as $index => $image) { - $path = $paths[$index]; - $imageAnnotations = $annotations[$image->id]; - $boxes = []; - foreach ($imageAnnotations as $a) { - $box = $this->getBoundingBox($a, $image); - $allEqual = $box[0] === $box[1] && $box[0] === $box[2] && $box[0] === $box[3]; - if (!$allEqual) { - $boxes[$a->id] = $box; - } - } - - if (!empty($boxes)) { - $input[$path] = $boxes; - } - } - - return $input; - } - - protected function getBoundingBox(MaiaAnnotation $annotation, Image $image): array - { - $box = match ($annotation->shape_id) { - Shape::pointId() => $this->getPointBoundingBox($annotation), - Shape::circleId() => $this->getCircleBoundingBox($annotation), - // TODO: An ellipse will not be handled correctly by this but I didn't bother - // because this shape is almost never used anyway. - default => $this->getPolygonBoundingBox($annotation), - }; - - // Now move the box if it overflows the image boundaries. - - if ($box[2] <= 0) { - return [0, 0, 0, 0]; - } - - if ($box[3] <= 0) { - return [0, 0, 0, 0]; - } - - $delta = [0, 0, 0, 0]; - - if ($box[0] <= 0) { - $delta[0] = -$box[0]; - } - - if ($box[1] <= 0) { - $delta[1] = -$box[1]; - } - - if (!is_null($image->width)) { - if ($box[0] >= $image->width) { - return [0, 0, 0, 0]; - } - - if ($box[2] >= $image->width) { - $delta[2] = $box[2] - $image->width; - } - } - - if (!is_null($image->height)) { - if ($box[1] >= $image->height) { - return [0, 0, 0, 0]; - } - - if ($box[3] >= $image->height) { - $delta[3] = $box[3] - $image->height; - } - } - - // The case of both delta values being >0 is handled below. - if ($delta[0] > 0) { - $box[0] += $delta[0]; - $box[2] += $delta[0]; - } elseif ($delta[2] > 0) { - $box[0] -= $delta[2]; - $box[2] -= $delta[2]; - } - - // The case of both delta values being >0 is handled below. - if ($delta[1] > 0) { - $box[1] += $delta[1]; - $box[3] += $delta[1]; - } elseif ($delta[3] > 0) { - $box[1] -= $delta[3]; - $box[3] -= $delta[3]; - } - - // Moving the box could have make it overflow on the "other" side. - // Shrink the box in this case. - - $box[0] = max(0, $box[0]); - $box[1] = max(0, $box[1]); - - if (!is_null($image->width)) { - $box[2] = min($image->width, $box[2]); - } - - if (!is_null($image->height)) { - $box[3] = min($image->height, $box[3]); - } - - return $box; - } - - protected function getPointBoundingBox(MaiaAnnotation $annotation): array - { - $points = $annotation->points; - - return [ - $points[0] - self::POINT_PADDING, - $points[1] - self::POINT_PADDING, - $points[0] + self::POINT_PADDING, - $points[1] + self::POINT_PADDING, - ]; - } - - protected function getCircleBoundingBox(MaiaAnnotation $annotation): array - { - $points = $annotation->points; - - return [ - $points[0] - $points[2], - $points[1] - $points[2], - $points[0] + $points[2], - $points[1] + $points[2], - ]; - } - - protected function getPolygonBoundingBox(MaiaAnnotation $annotation): array - { - $points = $annotation->points; - $minX = INF; - $minY = INF; - $maxX = -INF; - $maxY = -INF; - - for ($i = 0; $i < count($points); $i += 2) { - $minX = min($minX, $points[$i]); - $minY = min($minY, $points[$i + 1]); - $maxX = max($maxX, $points[$i]); - $maxY = max($maxY, $points[$i + 1]); - } - - return [$minX, $minY, $maxX, $maxY]; - } - - /** - * Run the Python command. - * - * @param string $command - */ - protected function python(array $input, string $outputPath) - { - $python = config('maia.python'); - $script = config('maia.extract_features_script'); - $inputPath = tempnam(sys_get_temp_dir(), 'maia_feature_vector_input'); - File::put($inputPath, json_encode($input)); - try { - $result = Process::forever() - ->env(['TORCH_HOME' => config('maia.torch_hub_path')]) - ->run("{$python} -u {$script} {$inputPath} {$outputPath}") - ->throw(); - } finally { - File::delete($inputPath); - } - } - - /** - * Generator to read the output CSV row by row. - */ - protected function readOuputCsv(string $path): \Generator - { - $file = new SplFileObject($path); - while (!$file->eof()) { - $csv = $file->fgetcsv(); - if (count($csv) === 2) { - yield $csv; - } - } - } } diff --git a/src/Jobs/GenerateAnnotationPatches.php b/src/Jobs/GenerateAnnotationPatches.php deleted file mode 100644 index b7fcfcd..0000000 --- a/src/Jobs/GenerateAnnotationPatches.php +++ /dev/null @@ -1,65 +0,0 @@ -getPatchStorageDisk(); - $this->getCreatedAnnotations() - ->chunkById(self::JOB_CHUNK_SIZE, function ($chunk) use ($disk) { - foreach ($chunk as $annotation) { - GenerateImageAnnotationPatch::dispatch($annotation, $disk) - ->onQueue(config('largo.generate_annotation_patch_queue')); - } - }); - } - - /** - * Get a query for the annotations that have been created by this job. - * - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo - */ - abstract protected function getCreatedAnnotations(); - - /** - * Get the storage disk to store the annotation patches to. - */ - abstract protected function getPatchStorageDisk(); -} diff --git a/src/Jobs/GenerateTrainingProposalPatches.php b/src/Jobs/GenerateTrainingProposalPatches.php index 50e6959..6140ac0 100644 --- a/src/Jobs/GenerateTrainingProposalPatches.php +++ b/src/Jobs/GenerateTrainingProposalPatches.php @@ -2,23 +2,46 @@ namespace Biigle\Modules\Maia\Jobs; -class GenerateTrainingProposalPatches extends GenerateAnnotationPatches +use Biigle\Jobs\Job; +use Biigle\Modules\Maia\MaiaJob; +use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Queue\SerializesModels; + +class GenerateTrainingProposalPatches extends Job implements ShouldQueue { + use SerializesModels; + /** - * Get a query for the annotations that have been created by this job. + * Ignore this job if the MAIA job does not exist any more. * - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo + * @var bool */ - protected function getCreatedAnnotations() - { - return $this->job->trainingProposals(); - } + protected $deleteWhenMissingModels = true; /** - * Get the storage disk to store the annotation patches to. + * Create a new isntance. */ - protected function getPatchStorageDisk() + public function __construct(public MaiaJob $maiaJob) + { + // + } + + public function handle(): void { - return config('maia.training_proposal_storage_disk'); + $this->maiaJob->volume->images() + ->whereExists(fn ($q) => + $q->select(\DB::raw(1)) + ->from('maia_training_proposals') + ->where('maia_training_proposals.job_id', $this->maiaJob->id) + ->whereColumn('maia_training_proposals.image_id', 'images.id') + ) + ->eachById(fn ($image) => + ProcessNoveltyDetectedImage::dispatch($image, $this->maiaJob, + // Feature vectors are generated in a separate job on the GPU. + skipFeatureVectors: true, + targetDisk: config('maia.training_proposal_storage_disk') + ) + ->onQueue(config('largo.generate_annotation_patch_queue')) + ); } } diff --git a/src/Jobs/ProcessNoveltyDetectedImage.php b/src/Jobs/ProcessNoveltyDetectedImage.php new file mode 100644 index 0000000..e07972d --- /dev/null +++ b/src/Jobs/ProcessNoveltyDetectedImage.php @@ -0,0 +1,56 @@ +id) + ->where('job_id', $this->maiaJob->id); + } + + /** + * Create the feature vectors based on the Python script output. + */ + protected function updateOrCreateFeatureVectors(Collection $annotations, \Generator $output): void + { + $annotations = $annotations->keyBy('id'); + foreach ($output as $row) { + $annotation = $annotations->get($row[0]); + TrainingProposalFeatureVector::updateOrCreate( + ['id' => $annotation->id], + [ + 'job_id' => $annotation->job_id, + 'vector' => $row[1], + ] + ); + } + } +} diff --git a/src/Jobs/ProcessObjectDetectedImage.php b/src/Jobs/ProcessObjectDetectedImage.php new file mode 100644 index 0000000..9eecc7e --- /dev/null +++ b/src/Jobs/ProcessObjectDetectedImage.php @@ -0,0 +1,56 @@ +id) + ->where('job_id', $this->maiaJob->id); + } + + /** + * Create the feature vectors based on the Python script output. + */ + protected function updateOrCreateFeatureVectors(Collection $annotations, \Generator $output): void + { + $annotations = $annotations->keyBy('id'); + foreach ($output as $row) { + $annotation = $annotations->get($row[0]); + AnnotationCandidateFeatureVector::updateOrCreate( + ['id' => $annotation->id], + [ + 'job_id' => $annotation->job_id, + 'vector' => $row[1], + ] + ); + } + } +} diff --git a/src/config/maia.php b/src/config/maia.php index 9f1d069..3dafee8 100644 --- a/src/config/maia.php +++ b/src/config/maia.php @@ -135,14 +135,4 @@ | Enable to disallow submission of new jobs. */ 'maintenance_mode' => env('MAIA_MAINTENANCE_MODE', false), - - /* - | Path to the extract features script. - */ - 'extract_features_script' => __DIR__.'/../resources/scripts/ExtractFeatures.py', - - /* - | Path to the directory to use as Torch Hub cache. - */ - 'torch_hub_path' => storage_path('maia_cache'), ]; diff --git a/src/resources/scripts/ExtractFeatures.py b/src/resources/scripts/ExtractFeatures.py deleted file mode 100644 index 7a55540..0000000 --- a/src/resources/scripts/ExtractFeatures.py +++ /dev/null @@ -1,53 +0,0 @@ -from PIL import Image -import csv -import json -import numpy as np -import sys -import torch -import torchvision.transforms as T - -# input_json = { -# cached_filename: { -# annotation_model_id: [left, top, right, bottom], -# }, -# } - -with open(sys.argv[1], 'r') as f: - input_json = json.load(f) - -if torch.cuda.is_available(): - device = torch.device('cuda') - dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14').cuda() -else: - device = torch.device('cpu') - dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14') - -dinov2_vits14.to(device) - -transform = T.Compose([ - T.Resize((224, 224), interpolation=T.InterpolationMode.BICUBIC), - T.ToTensor(), - T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), -]) - -def normalize_image_mode(image): - if image.mode == 'RGBA' or image.mode == 'L' or image.mode == 'P': - image = image.convert('RGB') - - if image.mode =='I': - # I images (32 bit signed integer) need to be rescaled manually before converting. - image = Image.fromarray(((np.array(image)/(2**16))*2**8).astype(np.uint8)).convert('RGB') - - return image - -with open(sys.argv[2], 'w') as f: - writer = csv.writer(f) - with torch.no_grad(): - for image_path, annotations in input_json.items(): - image = Image.open(image_path) - image = normalize_image_mode(image) - for model_id, box in annotations.items(): - image_crop = image.crop(box) - image_crop_t = transform(image_crop).unsqueeze(0).to(device) - features = dinov2_vits14(image_crop_t) - writer.writerow([model_id, json.dumps(features[0].tolist())]) diff --git a/tests/Http/Controllers/Api/AnnotationCandidateControllerTest.php b/tests/Http/Controllers/Api/AnnotationCandidateControllerTest.php index 87b7263..29e98c4 100644 --- a/tests/Http/Controllers/Api/AnnotationCandidateControllerTest.php +++ b/tests/Http/Controllers/Api/AnnotationCandidateControllerTest.php @@ -3,9 +3,9 @@ namespace Biigle\Tests\Modules\Maia\Http\Controllers\Api; use ApiTestCase; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; use Biigle\Modules\Maia\AnnotationCandidateFeatureVector; use Biigle\Modules\Maia\Jobs\ConvertAnnotationCandidates; +use Biigle\Modules\Maia\Jobs\ProcessObjectDetectedImage; use Biigle\Modules\Maia\MaiaJob; use Biigle\Modules\Maia\MaiaJobState as State; use Biigle\Tests\ImageAnnotationTest; @@ -171,9 +171,12 @@ public function testUpdatePoints() $this->putJson("/api/v1/maia/annotation-candidates/{$a->id}", ['points' => [10, 20, 30]]) ->assertStatus(200); - Queue::assertPushed(GenerateImageAnnotationPatch::class); + Queue::assertPushed(ProcessObjectDetectedImage::class, function ($job) use ($a) { + $this->assertEquals([$a->id], $job->only); + $this->assertFalse($job->skipFeatureVectors); - $this->markTestIncomplete('also push a job to regenerate the feature vector'); + return true; + }); } public function testIndexSimilarity() diff --git a/tests/Http/Controllers/Api/TrainingProposalControllerTest.php b/tests/Http/Controllers/Api/TrainingProposalControllerTest.php index 1fe7adb..fdc4cbe 100644 --- a/tests/Http/Controllers/Api/TrainingProposalControllerTest.php +++ b/tests/Http/Controllers/Api/TrainingProposalControllerTest.php @@ -3,8 +3,8 @@ namespace Biigle\Tests\Modules\Maia\Http\Controllers\Api; use ApiTestCase; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; use Biigle\Modules\Maia\Events\MaiaJobContinued; +use Biigle\Modules\Maia\Jobs\ProcessNoveltyDetectedImage; use Biigle\Modules\Maia\MaiaJob; use Biigle\Modules\Maia\MaiaJobState as State; use Biigle\Modules\Maia\TrainingProposalFeatureVector; @@ -153,9 +153,12 @@ public function testUpdatePoints() $this->putJson("/api/v1/maia/training-proposals/{$a->id}", ['points' => [10, 20, 30]]) ->assertStatus(200); - Queue::assertPushed(GenerateImageAnnotationPatch::class); + Queue::assertPushed(ProcessNoveltyDetectedImage::class, function ($job) use ($a) { + $this->assertEquals([$a->id], $job->only); + $this->assertFalse($job->skipFeatureVectors); - $this->markTestIncomplete('also push a job to regenerate the feature vector'); + return true; + }); } public function testIndexSimilarity() diff --git a/tests/Jobs/ConvertAnnotationCandidatesTest.php b/tests/Jobs/ConvertAnnotationCandidatesTest.php index a5bcba3..8d5b133 100644 --- a/tests/Jobs/ConvertAnnotationCandidatesTest.php +++ b/tests/Jobs/ConvertAnnotationCandidatesTest.php @@ -2,7 +2,7 @@ namespace Biigle\Tests\Modules\Maia\Jobs; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; +use Biigle\Modules\Largo\Jobs\ProcessAnnotatedImage; use Biigle\Modules\Maia\Jobs\ConvertAnnotationCandidates; use Biigle\Tests\ImageAnnotationTest; use Biigle\Tests\LabelTest; @@ -47,6 +47,11 @@ public function testHandle() $this->assertEquals($annotation->id, $c2->fresh()->annotation_id); - Queue::assertPushed(GenerateImageAnnotationPatch::class); + Queue::assertPushed(ProcessAnnotatedImage::class, function ($job) use ($c1, $a) { + $this->assertEquals($c1->image_id, $job->file->id); + $this->assertEquals([$a->id], $job->only); + + return true; + }); } } diff --git a/tests/Jobs/GenerateAnnotationCandidatePatchesTest.php b/tests/Jobs/GenerateAnnotationCandidatePatchesTest.php index 332107e..be8157a 100644 --- a/tests/Jobs/GenerateAnnotationCandidatePatchesTest.php +++ b/tests/Jobs/GenerateAnnotationCandidatePatchesTest.php @@ -2,7 +2,8 @@ namespace Biigle\Tests\Modules\Maia\Jobs; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; +use Biigle\Image; +use Biigle\Modules\Maia\Jobs\ProcessObjectDetectedImage; use Biigle\Modules\Maia\Jobs\GenerateAnnotationCandidatePatches; use Biigle\Modules\Maia\MaiaJob; use Biigle\Modules\Maia\AnnotationCandidate; @@ -13,18 +14,27 @@ class GenerateAnnotationCandidatePatchesTest extends TestCase { public function testHandle() { - $job = MaiaJob::factory()->create(); - $tp = AnnotationCandidate::factory()->create(['job_id' => $job->id]); - $j = new GenerateAnnotationCandidatePatches($job); + $image = Image::factory()->create(); + $job = MaiaJob::factory()->create(['volume_id' => $image->volume_id]); + $tp = AnnotationCandidate::factory()->create([ + 'job_id' => $job->id, + 'image_id' => $image->id, + ]); + $j = new GenerateAnnotationCandidatePatches($tp->job); $j->handle(); - Queue::assertPushed(GenerateImageAnnotationPatch::class); + Queue::assertPushed(ProcessObjectDetectedImage::class, function ($job) { + $this->assertTrue($job->skipFeatureVectors); + + return true; + }); } public function testHandleEmpty() { - $job = MaiaJob::factory()->create(); + $image = Image::factory()->create(); + $job = MaiaJob::factory()->create(['volume_id' => $image->volume_id]); $j = new GenerateAnnotationCandidatePatches($job); $j->handle(); - Queue::assertNotPushed(GenerateImageAnnotationPatch::class); + Queue::assertNotPushed(ProcessObjectDetectedImage::class); } } diff --git a/tests/Jobs/GenerateAnnotationFeatureVectorsTest.php b/tests/Jobs/GenerateAnnotationFeatureVectorsTest.php index e985f8c..2dee54b 100644 --- a/tests/Jobs/GenerateAnnotationFeatureVectorsTest.php +++ b/tests/Jobs/GenerateAnnotationFeatureVectorsTest.php @@ -411,9 +411,9 @@ class GenerateProposalFeatureVectorsStub extends GenerateTrainingProposalFeature public $outputPath; public $output = []; - protected function python(array $input, string $outputPath) + protected function python(string $inputPath, string $outputPath) { - $this->input = $input; + $this->input = json_decode(File::get($inputPath), true); $this->outputPath = $outputPath; $csv = implode("\n", array_map(fn ($row) => implode(',', $row), $this->output)); File::put($outputPath, $csv); @@ -426,9 +426,9 @@ class GenerateCandidateFeatureVectorsStub extends GenerateAnnotationCandidateFea public $outputPath; public $output = []; - protected function python(array $input, string $outputPath) + protected function python(string $inputPath, string $outputPath) { - $this->input = $input; + $this->input = json_decode(File::get($inputPath), true); $this->outputPath = $outputPath; $csv = implode("\n", array_map(fn ($row) => implode(',', $row), $this->output)); File::put($outputPath, $csv); diff --git a/tests/Jobs/GenerateTrainingProposalPatchesTest.php b/tests/Jobs/GenerateTrainingProposalPatchesTest.php index 88506c0..a52c171 100644 --- a/tests/Jobs/GenerateTrainingProposalPatchesTest.php +++ b/tests/Jobs/GenerateTrainingProposalPatchesTest.php @@ -2,7 +2,8 @@ namespace Biigle\Tests\Modules\Maia\Jobs; -use Biigle\Modules\Largo\Jobs\GenerateImageAnnotationPatch; +use Biigle\Image; +use Biigle\Modules\Maia\Jobs\ProcessNoveltyDetectedImage; use Biigle\Modules\Maia\Jobs\GenerateTrainingProposalPatches; use Biigle\Modules\Maia\MaiaJob; use Biigle\Modules\Maia\TrainingProposal; @@ -13,18 +14,27 @@ class GenerateTrainingProposalPatchesTest extends TestCase { public function testHandle() { - $job = MaiaJob::factory()->create(); - $tp = TrainingProposal::factory()->create(['job_id' => $job->id]); - $j = new GenerateTrainingProposalPatches($job); + $image = Image::factory()->create(); + $job = MaiaJob::factory()->create(['volume_id' => $image->volume_id]); + $tp = TrainingProposal::factory()->create([ + 'job_id' => $job->id, + 'image_id' => $image->id, + ]); + $j = new GenerateTrainingProposalPatches($tp->job); $j->handle(); - Queue::assertPushed(GenerateImageAnnotationPatch::class); + Queue::assertPushed(ProcessNoveltyDetectedImage::class, function ($job) { + $this->assertTrue($job->skipFeatureVectors); + + return true; + }); } public function testHandleEmpty() { - $job = MaiaJob::factory()->create(); + $image = Image::factory()->create(); + $job = MaiaJob::factory()->create(['volume_id' => $image->volume_id]); $j = new GenerateTrainingProposalPatches($job); $j->handle(); - Queue::assertNotPushed(GenerateImageAnnotationPatch::class); + Queue::assertNotPushed(ProcessNoveltyDetectedImage::class); } } diff --git a/tests/Jobs/ProcessNoveltyDetectedImageTest.php b/tests/Jobs/ProcessNoveltyDetectedImageTest.php new file mode 100644 index 0000000..e7ddb1a --- /dev/null +++ b/tests/Jobs/ProcessNoveltyDetectedImageTest.php @@ -0,0 +1,96 @@ +getImageMock(); + $proposal = TrainingProposal::factory()->create([ + 'points' => [200, 200, 10], + ]); + $job = new ProcessNoveltyDetectedImageStub($proposal->image, $proposal->job, + targetDisk: 'test' + ); + $job->mock = $image; + + $image->shouldReceive('crop')->once()->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + + $job->handleFile($proposal->image, 'abc'); + $prefix = fragment_uuid_path($proposal->image->uuid); + $content = $disk->get("{$prefix}/{$proposal->id}.jpg"); + $this->assertEquals('abc123', $content); + } + + public function testGenerateFeatureVector() + { + Storage::fake('test'); + $image = $this->getImageMock(); + $image->shouldReceive('crop')->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + $proposal = TrainingProposal::factory()->create([ + 'points' => [200, 200, 10], + ]); + $job = new ProcessNoveltyDetectedImageStub($proposal->image, $proposal->job, + targetDisk: 'test' + ); + $job->mock = $image; + $job->output = [[$proposal->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($proposal->image, 'abc'); + + $input = $job->input; + $this->assertCount(1, $input); + $filename = array_keys($input)[0]; + $this->assertArrayHasKey($proposal->id, $input[$filename]); + $box = $input[$filename][$proposal->id]; + $this->assertEquals([190, 190, 210, 210], $box); + + $vectors = TrainingProposalFeatureVector::where('id', $proposal->id)->get(); + $this->assertCount(1, $vectors); + $this->assertEquals($proposal->job_id, $vectors[0]->job_id); + $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); + } + + protected function getImageMock($times = 1) + { + $image = Mockery::mock(); + $image->width = 1000; + $image->height = 750; + $image->shouldReceive('resize') + ->times($times) + ->andReturn($image); + + return $image; + } +} + +class ProcessNoveltyDetectedImageStub extends ProcessNoveltyDetectedImage +{ + public $input; + public $outputPath; + public $output = []; + + public function getVipsImage($path) + { + return $this->mock; + } + + protected function python(string $inputPath, string $outputPath) + { + $this->input = json_decode(File::get($inputPath), true); + $this->outputPath = $outputPath; + $csv = implode("\n", array_map(fn ($row) => implode(',', $row), $this->output)); + File::put($outputPath, $csv); + } +} diff --git a/tests/Jobs/ProcessObjectDetectedImageTest.php b/tests/Jobs/ProcessObjectDetectedImageTest.php new file mode 100644 index 0000000..415df56 --- /dev/null +++ b/tests/Jobs/ProcessObjectDetectedImageTest.php @@ -0,0 +1,96 @@ +getImageMock(); + $candidate = AnnotationCandidate::factory()->create([ + 'points' => [200, 200, 10], + ]); + $job = new ProcessObjectDetectedImageStub($candidate->image, $candidate->job, + targetDisk: 'test' + ); + $job->mock = $image; + + $image->shouldReceive('crop')->once()->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + + $job->handleFile($candidate->image, 'abc'); + $prefix = fragment_uuid_path($candidate->image->uuid); + $content = $disk->get("{$prefix}/{$candidate->id}.jpg"); + $this->assertEquals('abc123', $content); + } + + public function testGenerateFeatureVector() + { + Storage::fake('test'); + $image = $this->getImageMock(); + $image->shouldReceive('crop')->andReturn($image); + $image->shouldReceive('writeToBuffer')->andReturn('abc123'); + $candidate = AnnotationCandidate::factory()->create([ + 'points' => [200, 200, 10], + ]); + $job = new ProcessObjectDetectedImageStub($candidate->image, $candidate->job, + targetDisk: 'test' + ); + $job->mock = $image; + $job->output = [[$candidate->id, '"'.json_encode(range(0, 383)).'"']]; + $job->handleFile($candidate->image, 'abc'); + + $input = $job->input; + $this->assertCount(1, $input); + $filename = array_keys($input)[0]; + $this->assertArrayHasKey($candidate->id, $input[$filename]); + $box = $input[$filename][$candidate->id]; + $this->assertEquals([190, 190, 210, 210], $box); + + $vectors = AnnotationCandidateFeatureVector::where('id', $candidate->id)->get(); + $this->assertCount(1, $vectors); + $this->assertEquals($candidate->job_id, $vectors[0]->job_id); + $this->assertEquals(range(0, 383), $vectors[0]->vector->toArray()); + } + + protected function getImageMock($times = 1) + { + $image = Mockery::mock(); + $image->width = 1000; + $image->height = 750; + $image->shouldReceive('resize') + ->times($times) + ->andReturn($image); + + return $image; + } +} + +class ProcessObjectDetectedImageStub extends ProcessObjectDetectedImage +{ + public $input; + public $outputPath; + public $output = []; + + public function getVipsImage($path) + { + return $this->mock; + } + + protected function python(string $inputPath, string $outputPath) + { + $this->input = json_decode(File::get($inputPath), true); + $this->outputPath = $outputPath; + $csv = implode("\n", array_map(fn ($row) => implode(',', $row), $this->output)); + File::put($outputPath, $csv); + } +}