Merge pull request #613 from wri/epic/TM-1398-tree-species

[TM-1398] Merge tree species epic

.github/workflows/pull-request.yml

@@ -1,7 +1,6 @@
 name: pull-request
 on:
   pull_request:
-    branches: [main, staging, release/**]
 jobs:
   lintTest:
     runs-on: ubuntu-latest

app/Console/Commands/ImportTreeSpeciesAssociations.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Console\Commands\Traits\Abortable;
+use App\Console\Commands\Traits\AbortException;
+use App\Console\Commands\Traits\ExceptionLevel;
+use App\Models\V2\TreeSpecies\TreeSpecies;
+use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
+use Illuminate\Console\Command;
+use Symfony\Component\Process\Process;
+
+class ImportTreeSpeciesAssociations extends Command
+{
+    use Abortable;
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'import-tree-species-associations {file}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Imports a CSV that links UUIDs from v2_tree_species to taxon_ids from tree_species_research';
+
+    protected int $treeSpeciesUuidColumn;
+
+    protected int $taxonIdColumn;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->executeAbortableScript(function () {
+            $process = new Process(['wc', '-l', $this->argument('file')]);
+            $process->run();
+            $this->assert($process->isSuccessful(), "WC failed {$process->getErrorOutput()}");
+
+            $lines = ((int)explode(' ', $process->getOutput())[0]) - 1;
+
+            $fileHandle = fopen($this->argument('file'), 'r');
+            $this->parseHeaders(fgetcsv($fileHandle));
+
+            $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
+                $abortExceptions = [];
+                while ($csvRow = fgetcsv($fileHandle)) {
+                    $treeSpeciesUuid = $csvRow[$this->treeSpeciesUuidColumn];
+                    $taxonId = $csvRow[$this->taxonIdColumn];
+
+                    if ($taxonId != 'NA') {
+                        try {
+                            $research = TreeSpeciesResearch::find($taxonId);
+                            $this->assert($research != null, "Taxon ID not found: $taxonId", ExceptionLevel::Warning);
+
+                            TreeSpecies::isUuid($treeSpeciesUuid)->update([
+                                'taxon_id' => $taxonId,
+                                'name' => $research->name,
+                            ]);
+                        } catch (AbortException $e) {
+                            $abortExceptions[] = $e;
+                        }
+                    }
+
+                    $progressBar->advance();
+                }
+
+                $progressBar->finish();
+
+                if (! empty($abortExceptions)) {
+                    $this->warn("Errors and warnings encountered during parsing CSV Rows:\n");
+                    foreach ($abortExceptions as $error) {
+                        $this->logException($error);
+                    }
+                }
+            });
+
+            fclose($fileHandle);
+        });
+    }
+
+    protected function parseHeaders(array $headerRow): void
+    {
+        foreach ($headerRow as $index => $header) {
+            $header = trim($header, "\xEF\xBB\xBF\"");
+            if ($header == 'tree_species_uuid') {
+                $this->treeSpeciesUuidColumn = $index;
+            } elseif ($header == 'taxon_id') {
+                $this->taxonIdColumn = $index;
+            }
+        }
+
+        $this->assert(
+            is_numeric($this->treeSpeciesUuidColumn) && is_numeric($this->taxonIdColumn),
+            'Not all required columns were found'
+        );
+    }
+}

app/Console/Commands/OneOff/PopulateTreeSpeciesResearch.php

@@ -4,8 +4,10 @@
 
 use App\Console\Commands\Traits\Abortable;
 use App\Console\Commands\Traits\AbortException;
+use App\Console\Commands\Traits\ExceptionLevel;
 use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
 use Illuminate\Console\Command;
+use Symfony\Component\Process\Process;
 
 class PopulateTreeSpeciesResearch extends Command
 {
@@ -33,6 +35,7 @@ class PopulateTreeSpeciesResearch extends Command
         'family' => 'family',
         'genus' => 'genus',
         'specificEpithet' => 'specific_epithet',
+        'infraspecificEpithet' => 'infraspecific_epithet',
     ];
 
     // Populated by parseHeaders(), a mapping of DB colum name to the index in each row where that data is expected to
@@ -45,21 +48,49 @@ class PopulateTreeSpeciesResearch extends Command
     public function handle()
     {
         $this->executeAbortableScript(function () {
+            $process = new Process(['wc', '-l', $this->argument('file')]);
+            $process->run();
+            $this->assert($process->isSuccessful(), "WC failed {$process->getErrorOutput()}");
+
+            $lines = ((int)explode(' ', $process->getOutput())[0]) - 1;
+
             $fileHandle = fopen($this->argument('file'), 'r');
-            $this->parseHeaders(fgetcsv($fileHandle, separator: "\t"));
+            $this->parseHeaders(fgetcsv($fileHandle));
 
-            // The input file at the time of this writing has 1618549 rows of data
-            $this->withProgressBar(1618549, function ($progressBar) use ($fileHandle) {
-                while ($csvRow = fgetcsv($fileHandle, separator: "\t")) {
+            $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
+                $abortExceptions = [];
+                while ($csvRow = fgetcsv($fileHandle)) {
                     $data = [];
                     foreach ($this->columns as $column => $index) {
                         $data[$column] = $csvRow[$index];
                     }
-                    TreeSpeciesResearch::create($data);
+
+                    try {
+                        $existing = TreeSpeciesResearch::where('scientific_name', $data['scientific_name'])->first();
+                        $this->assert(
+                            $existing == null,
+                            'Scientific name already exists, skipping: ' . json_encode([
+                                'existing_id' => $existing?->taxon_id,
+                                'new_id' => $data['taxon_id'],
+                                'scientific_name' => $data['scientific_name'],
+                            ], JSON_PRETTY_PRINT),
+                            ExceptionLevel::Warning
+                        );
+                        TreeSpeciesResearch::create($data);
+                    } catch (AbortException $e) {
+                        $abortExceptions[] = $e;
+                    }
                     $progressBar->advance();
                 }
 
                 $progressBar->finish();
+
+                if (! empty($abortExceptions)) {
+                    $this->warn("Errors and warnings encountered during parsing CSV Rows:\n");
+                    foreach ($abortExceptions as $error) {
+                        $this->logException($error);
+                    }
+                }
             });
 
             fclose($fileHandle);
@@ -72,7 +103,8 @@ public function handle()
     protected function parseHeaders(array $headerRow): void
     {
         foreach ($headerRow as $index => $header) {
-            $header = trim($header);
+            // Excel puts some garbage at the beginning of the file that we need to filter out.
+            $header = trim($header, "\xEF\xBB\xBF\"");
 
             if (array_key_exists($header, self::COLUMN_MAPPING)) {
                 $this->columns[self::COLUMN_MAPPING[$header]] = $index;

app/Console/Commands/OneOff/UpdateTreeCollections.php

@@ -2,6 +2,7 @@
 
 namespace App\Console\Commands\OneOff;
 
+use App\Models\V2\Forms\FormQuestion;
 use App\Models\V2\Organisation;
 use App\Models\V2\ProjectPitch;
 use App\Models\V2\Projects\Project;
@@ -62,5 +63,21 @@ public function handle()
                 $updateRequest->update(['content' => $content]);
             }
         });
+
+        $this->info('Updating form fields');
+        FormQuestion::withoutTimestamps(function () {
+            $relationSets = data_get(config('wri.linked-fields.models'), '*.relations');
+            foreach ($relationSets as $relations) {
+                foreach ($relations as $linkedFieldKey => $properties) {
+                    if ($properties['input_type'] != 'treeSpecies') {
+                        continue;
+                    }
+
+                    FormQuestion::withTrashed()
+                        ->where('linked_field_key', $linkedFieldKey)
+                        ->update(['collection' => $properties['collection']]);
+                }
+            }
+        });
     }
 }

app/Http/Resources/V2/TreeSpecies/TreeSpeciesResource.php

@@ -18,6 +18,7 @@ public function toArray($request)
             'amount' => $this->amount,
             'type' => $this->type,
             'collection' => $this->collection,
+            'taxon_id' => $this->taxon_id,
         ];
     }
 }

app/Models/V2/TreeSpecies/TreeSpecies.php

@@ -38,9 +38,7 @@ class TreeSpecies extends Model implements EntityRelationModel
         'speciesable_id',
         'collection',
         'hidden',
-
-        'old_id',
-        'old_model',
+        'taxon_id',
     ];
 
     public const COLLECTION_DIRECT_SEEDING = 'direct-seeding';
@@ -82,6 +80,11 @@ public function speciesable()
         return $this->morphTo();
     }
 
+    public function taxonomicSpecies()
+    {
+        return $this->belongsTo(TreeSpeciesResearch::class, 'taxon_id');
+    }
+
     public function getRouteKeyName()
     {
         return 'uuid';

config/wri/linked-fields.php

@@ -151,6 +151,7 @@
                     'label' => 'Tree Species',
                     'resource' => 'App\Http\Resources\V2\TreeSpecies\TreeSpeciesResource',
                     'input_type' => 'treeSpecies',
+                    'collection' => 'historical-tree-species'
                 ],
                 'org-leadership-team' => [
                     'property' => 'leadershipTeam',

database/migrations/2024_12_10_220655_add_infraspecific_epithet_to_tree_species_research.php

@@ -0,0 +1,29 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('tree_species_research', function (Blueprint $table) {
+            $table->string('infraspecific_epithet');
+            $table->unique('scientific_name');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('tree_species_research', function (Blueprint $table) {
+            $table->dropColumn('infraspecific_epithet');
+            $table->dropIndex('tree_species_research_scientific_name_unique');
+        });
+    }
+};