Merge pull request #609 from wri/feat/TM-1402-tree-species-design-int…

…egration

[TM-1402] Updates to the v2_tree_species -> tree_species_research relationship

app/Console/Commands/ImportTreeSpeciesAssociations.php

@@ -3,7 +3,10 @@
 namespace App\Console\Commands;
 
 use App\Console\Commands\Traits\Abortable;
+use App\Console\Commands\Traits\AbortException;
+use App\Console\Commands\Traits\ExceptionLevel;
 use App\Models\V2\TreeSpecies\TreeSpecies;
+use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
 use Illuminate\Console\Command;
 use Symfony\Component\Process\Process;
 
@@ -45,17 +48,36 @@ public function handle()
             $this->parseHeaders(fgetcsv($fileHandle));
 
             $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
+                $abortExceptions = [];
                 while ($csvRow = fgetcsv($fileHandle)) {
                     $treeSpeciesUuid = $csvRow[$this->treeSpeciesUuidColumn];
                     $taxonId = $csvRow[$this->taxonIdColumn];
 
                     if ($taxonId != 'NA') {
-                        TreeSpecies::isUuid($treeSpeciesUuid)->update(['taxon_id' => $taxonId]);
+                        try {
+                            $research = TreeSpeciesResearch::find($taxonId);
+                            $this->assert($research != null, "Taxon ID not found: $taxonId", ExceptionLevel::Warning);
+
+                            TreeSpecies::isUuid($treeSpeciesUuid)->update([
+                                'taxon_id' => $taxonId,
+                                'name' => $research->name,
+                            ]);
+                        } catch (AbortException $e) {
+                            $abortExceptions[] = $e;
+                        }
                     }
+
                     $progressBar->advance();
                 }
 
                 $progressBar->finish();
+
+                if (! empty($abortExceptions)) {
+                    $this->warn("Errors and warnings encountered during parsing CSV Rows:\n");
+                    foreach ($abortExceptions as $error) {
+                        $this->logException($error);
+                    }
+                }
             });
 
             fclose($fileHandle);
@@ -65,6 +87,7 @@ public function handle()
     protected function parseHeaders(array $headerRow): void
     {
         foreach ($headerRow as $index => $header) {
+            $header = trim($header, "\xEF\xBB\xBF\"");
             if ($header == 'tree_species_uuid') {
                 $this->treeSpeciesUuidColumn = $index;
             } elseif ($header == 'taxon_id') {
@@ -73,7 +96,7 @@ protected function parseHeaders(array $headerRow): void
         }
 
         $this->assert(
-            $this->treeSpeciesUuidColumn != null && $this->taxonIdColumn != null,
+            is_numeric($this->treeSpeciesUuidColumn) && is_numeric($this->taxonIdColumn),
             'Not all required columns were found'
         );
     }

app/Console/Commands/OneOff/PopulateTreeSpeciesResearch.php

@@ -4,8 +4,10 @@
 
 use App\Console\Commands\Traits\Abortable;
 use App\Console\Commands\Traits\AbortException;
+use App\Console\Commands\Traits\ExceptionLevel;
 use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
 use Illuminate\Console\Command;
+use Symfony\Component\Process\Process;
 
 class PopulateTreeSpeciesResearch extends Command
 {
@@ -33,6 +35,7 @@ class PopulateTreeSpeciesResearch extends Command
         'family' => 'family',
         'genus' => 'genus',
         'specificEpithet' => 'specific_epithet',
+        'infraspecificEpithet' => 'infraspecific_epithet',
     ];
 
     // Populated by parseHeaders(), a mapping of DB colum name to the index in each row where that data is expected to
@@ -45,21 +48,49 @@ class PopulateTreeSpeciesResearch extends Command
     public function handle()
     {
         $this->executeAbortableScript(function () {
+            $process = new Process(['wc', '-l', $this->argument('file')]);
+            $process->run();
+            $this->assert($process->isSuccessful(), "WC failed {$process->getErrorOutput()}");
+
+            $lines = ((int)explode(' ', $process->getOutput())[0]) - 1;
+
             $fileHandle = fopen($this->argument('file'), 'r');
-            $this->parseHeaders(fgetcsv($fileHandle, separator: "\t"));
+            $this->parseHeaders(fgetcsv($fileHandle));
 
-            // The input file at the time of this writing has 1618549 rows of data
-            $this->withProgressBar(1618549, function ($progressBar) use ($fileHandle) {
-                while ($csvRow = fgetcsv($fileHandle, separator: "\t")) {
+            $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
+                $abortExceptions = [];
+                while ($csvRow = fgetcsv($fileHandle)) {
                     $data = [];
                     foreach ($this->columns as $column => $index) {
                         $data[$column] = $csvRow[$index];
                     }
-                    TreeSpeciesResearch::create($data);
+
+                    try {
+                        $existing = TreeSpeciesResearch::where('scientific_name', $data['scientific_name'])->first();
+                        $this->assert(
+                            $existing == null,
+                            'Scientific name already exists, skipping: ' . json_encode([
+                                'existing_id' => $existing?->taxon_id,
+                                'new_id' => $data['taxon_id'],
+                                'scientific_name' => $data['scientific_name'],
+                            ], JSON_PRETTY_PRINT),
+                            ExceptionLevel::Warning
+                        );
+                        TreeSpeciesResearch::create($data);
+                    } catch (AbortException $e) {
+                        $abortExceptions[] = $e;
+                    }
                     $progressBar->advance();
                 }
 
                 $progressBar->finish();
+
+                if (! empty($abortExceptions)) {
+                    $this->warn("Errors and warnings encountered during parsing CSV Rows:\n");
+                    foreach ($abortExceptions as $error) {
+                        $this->logException($error);
+                    }
+                }
             });
 
             fclose($fileHandle);
@@ -72,7 +103,8 @@ public function handle()
     protected function parseHeaders(array $headerRow): void
     {
         foreach ($headerRow as $index => $header) {
-            $header = trim($header);
+            // Excel puts some garbage at the beginning of the file that we need to filter out.
+            $header = trim($header, "\xEF\xBB\xBF\"");
 
             if (array_key_exists($header, self::COLUMN_MAPPING)) {
                 $this->columns[self::COLUMN_MAPPING[$header]] = $index;

app/Console/Commands/OneOff/UpdateTreeCollections.php

@@ -2,6 +2,7 @@
 
 namespace App\Console\Commands\OneOff;
 
+use App\Models\V2\Forms\FormQuestion;
 use App\Models\V2\Organisation;
 use App\Models\V2\ProjectPitch;
 use App\Models\V2\Projects\Project;
@@ -62,5 +63,21 @@ public function handle()
                 $updateRequest->update(['content' => $content]);
             }
         });
+
+        $this->info('Updating form fields');
+        FormQuestion::withoutTimestamps(function () {
+            $relationSets = data_get(config('wri.linked-fields.models'), '*.relations');
+            foreach ($relationSets as $relations) {
+                foreach ($relations as $linkedFieldKey => $properties) {
+                    if ($properties['input_type'] != 'treeSpecies') {
+                        continue;
+                    }
+
+                    FormQuestion::withTrashed()
+                        ->where('linked_field_key', $linkedFieldKey)
+                        ->update(['collection' => $properties['collection']]);
+                }
+            }
+        });
     }
 }

app/Http/Resources/V2/TreeSpecies/TreeSpeciesResource.php

@@ -18,6 +18,7 @@ public function toArray($request)
             'amount' => $this->amount,
             'type' => $this->type,
             'collection' => $this->collection,
+            'taxon_id' => $this->taxon_id,
         ];
     }
 }

config/wri/linked-fields.php

@@ -151,6 +151,7 @@
                     'label' => 'Tree Species',
                     'resource' => 'App\Http\Resources\V2\TreeSpecies\TreeSpeciesResource',
                     'input_type' => 'treeSpecies',
+                    'collection' => 'historical-tree-species'
                 ],
                 'org-leadership-team' => [
                     'property' => 'leadershipTeam',

database/migrations/2024_12_10_220655_add_infraspecific_epithet_to_tree_species_research.php

@@ -0,0 +1,29 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('tree_species_research', function (Blueprint $table) {
+            $table->string('infraspecific_epithet');
+            $table->unique('scientific_name');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('tree_species_research', function (Blueprint $table) {
+            $table->dropColumn('infraspecific_epithet');
+            $table->dropIndex('tree_species_research_scientific_name_unique');
+        });
+    }
+};