Merge pull request #621 from wri/release/utltimate-ulmus

[RELEASE] Utltimate Ulmus

.github/workflows/pull-request.yml

@@ -1,7 +1,6 @@
 name: pull-request
 on:
   pull_request:
-    branches: [main, staging, release/**]
 jobs:
   lintTest:
     runs-on: ubuntu-latest

app/Console/Commands/ImportTreeSpeciesAssociations.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Console\Commands\Traits\Abortable;
+use App\Console\Commands\Traits\AbortException;
+use App\Console\Commands\Traits\ExceptionLevel;
+use App\Models\V2\TreeSpecies\TreeSpecies;
+use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
+use Illuminate\Console\Command;
+use Symfony\Component\Process\Process;
+
+class ImportTreeSpeciesAssociations extends Command
+{
+    use Abortable;
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'import-tree-species-associations {file}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Imports a CSV that links UUIDs from v2_tree_species to taxon_ids from tree_species_research';
+
+    protected int $treeSpeciesUuidColumn;
+
+    protected int $taxonIdColumn;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->executeAbortableScript(function () {
+            $process = new Process(['wc', '-l', $this->argument('file')]);
+            $process->run();
+            $this->assert($process->isSuccessful(), "WC failed {$process->getErrorOutput()}");
+
+            $lines = ((int)explode(' ', $process->getOutput())[0]) - 1;
+
+            $fileHandle = fopen($this->argument('file'), 'r');
+            $this->parseHeaders(fgetcsv($fileHandle));
+
+            $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
+                $abortExceptions = [];
+                while ($csvRow = fgetcsv($fileHandle)) {
+                    $treeSpeciesUuid = $csvRow[$this->treeSpeciesUuidColumn];
+                    $taxonId = $csvRow[$this->taxonIdColumn];
+
+                    if ($taxonId != 'NA') {
+                        try {
+                            $research = TreeSpeciesResearch::find($taxonId);
+                            $this->assert($research != null, "Taxon ID not found: $taxonId", ExceptionLevel::Warning);
+
+                            TreeSpecies::isUuid($treeSpeciesUuid)->update([
+                                'taxon_id' => $taxonId,
+                                'name' => $research->scientific_name,
+                            ]);
+                        } catch (AbortException $e) {
+                            $abortExceptions[] = $e;
+                        }
+                    }
+
+                    $progressBar->advance();
+                }
+
+                $progressBar->finish();
+
+                if (! empty($abortExceptions)) {
+                    $this->warn("Errors and warnings encountered during parsing CSV Rows:\n");
+                    foreach ($abortExceptions as $error) {
+                        $this->logException($error);
+                    }
+                }
+            });
+
+            fclose($fileHandle);
+        });
+    }
+
+    protected function parseHeaders(array $headerRow): void
+    {
+        foreach ($headerRow as $index => $header) {
+            $header = trim($header, "\xEF\xBB\xBF\"");
+            if ($header == 'tree_species_uuid') {
+                $this->treeSpeciesUuidColumn = $index;
+            } elseif ($header == 'taxon_id') {
+                $this->taxonIdColumn = $index;
+            }
+        }
+
+        $this->assert(
+            is_numeric($this->treeSpeciesUuidColumn) && is_numeric($this->taxonIdColumn),
+            'Not all required columns were found'
+        );
+    }
+}

app/Console/Commands/OneOff/AssociateExactMatchTrees.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Console\Commands\OneOff;
+
+use App\Models\V2\TreeSpecies\TreeSpecies;
+use Illuminate\Console\Command;
+
+class AssociateExactMatchTrees extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'one-off:associate-exact-match-trees';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update tree species rows without a taxon_id but that do have an exact match in the backbone.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        TreeSpecies::withoutTimestamps(function () {
+            $query = TreeSpecies::withTrashed()
+                ->join('tree_species_research', 'v2_tree_species.name', '=', 'tree_species_research.scientific_name')
+                ->where('v2_tree_species.taxon_id', null);
+            $this->withProgressBar((clone $query)->count(), function ($progressBar) use ($query) {
+                $query->chunkById(100, function ($trees) use ($progressBar) {
+                    foreach ($trees as $tree) {
+                        TreeSpecies::where('id', $tree->id)->update(['taxon_id' => $tree->taxon_id]);
+                        $progressBar->advance();
+                    }
+                });
+            });
+        });
+    }
+}

app/Console/Commands/OneOff/PopulateTreeSpeciesResearch.php

@@ -0,0 +1,132 @@
+<?php
+
+namespace App\Console\Commands\OneOff;
+
+use App\Console\Commands\Traits\Abortable;
+use App\Console\Commands\Traits\AbortException;
+use App\Console\Commands\Traits\ExceptionLevel;
+use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
+use Illuminate\Console\Command;
+use Illuminate\Support\Carbon;
+use Symfony\Component\Process\Process;
+
+class PopulateTreeSpeciesResearch extends Command
+{
+    use Abortable;
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'one-off:populate-tree-species-research {file}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Given an input file, populates the tree_species_research table';
+
+    // The names of the columns we require for inserting into the DB. Key is column name in the header row of the CSV,
+    // value is the column name in the DB definition
+    protected const COLUMN_MAPPING = [
+        'taxonID' => 'taxon_id',
+        'scientificName' => 'scientific_name',
+        'family' => 'family',
+        'genus' => 'genus',
+        'specificEpithet' => 'specific_epithet',
+        'infraspecificEpithet' => 'infraspecific_epithet',
+    ];
+
+    // Populated by parseHeaders(), a mapping of DB colum name to the index in each row where that data is expected to
+    // exist
+    protected $columns = [];
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->executeAbortableScript(function () {
+            $process = new Process(['wc', '-l', $this->argument('file')]);
+            $process->run();
+            $this->assert($process->isSuccessful(), "WC failed {$process->getErrorOutput()}");
+
+            $lines = ((int)explode(' ', $process->getOutput())[0]) - 1;
+
+            $fileHandle = fopen($this->argument('file'), 'r');
+            $this->parseHeaders(fgetcsv($fileHandle));
+
+            $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
+                $abortExceptions = [];
+                $bulkInsert = [];
+                while ($csvRow = fgetcsv($fileHandle)) {
+                    $data = [];
+                    foreach ($this->columns as $column => $index) {
+                        $data[$column] = $csvRow[$index] == 'NA' ? null : $csvRow[$index];
+                    }
+
+                    // These don't get set automatically with bulk insert
+                    $now = Carbon::now();
+                    $data['created_at'] = $now;
+                    $data['updated_at'] = $now;
+
+                    try {
+                        $existing = TreeSpeciesResearch::where('scientific_name', $data['scientific_name'])->first();
+                        $this->assert(
+                            $existing == null,
+                            'Scientific name already exists, skipping: ' . json_encode([
+                                'existing_id' => $existing?->taxon_id,
+                                'new_id' => $data['taxon_id'],
+                                'scientific_name' => $data['scientific_name'],
+                                'infraspecific_epithet' => $data['infraspecific_epithet'],
+                            ], JSON_PRETTY_PRINT),
+                            ExceptionLevel::Warning
+                        );
+
+                        $bulkInsert[] = $data;
+                        if (count($bulkInsert) >= 1000) {
+                            TreeSpeciesResearch::insert($bulkInsert);
+                            $bulkInsert = [];
+                        }
+                    } catch (AbortException $e) {
+                        $abortExceptions[] = $e;
+                    }
+                    $progressBar->advance();
+                }
+
+                $progressBar->finish();
+
+                if (! empty($abortExceptions)) {
+                    $this->warn("Errors and warnings encountered during parsing CSV Rows:\n");
+                    foreach ($abortExceptions as $error) {
+                        $this->logException($error);
+                    }
+                }
+            });
+
+            fclose($fileHandle);
+        });
+    }
+
+    /**
+     * @throws AbortException
+     */
+    protected function parseHeaders(array $headerRow): void
+    {
+        foreach ($headerRow as $index => $header) {
+            // Excel puts some garbage at the beginning of the file that we need to filter out.
+            $header = trim($header, "\xEF\xBB\xBF\"");
+
+            if (array_key_exists($header, self::COLUMN_MAPPING)) {
+                $this->columns[self::COLUMN_MAPPING[$header]] = $index;
+            }
+        }
+
+        $this->assert(
+            count(self::COLUMN_MAPPING) === count($this->columns),
+            'Not all required columns were found'
+        );
+    }
+}

app/Console/Commands/OneOff/UpdateTreeCollections.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Console\Commands\OneOff;
+
+use App\Models\V2\Forms\FormQuestion;
+use App\Models\V2\Organisation;
+use App\Models\V2\ProjectPitch;
+use App\Models\V2\Projects\Project;
+use App\Models\V2\Sites\SiteReport;
+use App\Models\V2\TreeSpecies\TreeSpecies;
+use App\Models\V2\UpdateRequests\UpdateRequest;
+use Illuminate\Console\Command;
+
+class UpdateTreeCollections extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'one-off:update-tree-collections';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Removes old / missing collection names and updates to a correct value.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->info('Updating collections in v2_tree_species');
+        TreeSpecies::withoutTimestamps(function () {
+            TreeSpecies::withTrashed()->where('speciesable_type', ProjectPitch::class)
+                ->update(['collection' => TreeSpecies::COLLECTION_PLANTED]);
+            TreeSpecies::withTrashed()->where('speciesable_type', Project::class)->where('collection', 'primary')
+                ->update(['collection' => TreeSpecies::COLLECTION_PLANTED]);
+            TreeSpecies::withTrashed()->where('speciesable_type', Organisation::class)
+                ->update(['collection' => TreeSpecies::COLLECTION_HISTORICAL]);
+            TreeSpecies::withTrashed()->where('speciesable_type', SiteReport::class)->where('collection', null)
+                ->update(['collection' => TreeSpecies::COLLECTION_NON_TREE]);
+        });
+
+        $this->info('Updating collections in v2_update_requests content');
+        // This is kind of a hassle; fortunately, the only model type above that has bad data embedded in update requests
+        // is Project
+        UpdateRequest::withoutTimestamps(function () {
+            $updateRequests = UpdateRequest::where('updaterequestable_type', Project::class)
+                ->where('content', 'LIKE', '%"collection":"primary"%')
+                ->get();
+            foreach ($updateRequests as $updateRequest) {
+                $content = $updateRequest->content;
+                foreach (array_keys($content) as $key) {
+                    $collections = data_get($content, "$key.*.collection");
+                    if (is_array($collections) && in_array('primary', $collections)) {
+                        data_set($content, "$key.*.collection", TreeSpecies::COLLECTION_PLANTED);
+                    }
+                }
+
+                $updateRequest->update(['content' => $content]);
+            }
+        });
+
+        $this->info('Updating form fields');
+        FormQuestion::withoutTimestamps(function () {
+            $relationSets = data_get(config('wri.linked-fields.models'), '*.relations');
+            foreach ($relationSets as $relations) {
+                foreach ($relations as $linkedFieldKey => $properties) {
+                    if ($properties['input_type'] != 'treeSpecies') {
+                        continue;
+                    }
+
+                    FormQuestion::withTrashed()
+                        ->where('linked_field_key', $linkedFieldKey)
+                        ->update(['collection' => $properties['collection']]);
+                }
+            }
+        });
+    }
+}