Merge pull request #614 from wri/feat/TM-1400-research-insert-fixes

[TM-1400] Do bulk inserts for a faster script execution.
wri · Dec 16, 2024 · 7c5641a · 7c5641a
2 parents 0b41066 + 62db653
commit 7c5641a
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/app/Console/Commands/ImportTreeSpeciesAssociations.php b/app/Console/Commands/ImportTreeSpeciesAssociations.php
@@ -60,7 +60,7 @@ public function handle()
 
                             TreeSpecies::isUuid($treeSpeciesUuid)->update([
                                 'taxon_id' => $taxonId,
-                                'name' => $research->name,
+                                'name' => $research->scientific_name,
                             ]);
                         } catch (AbortException $e) {
                             $abortExceptions[] = $e;

diff --git a/app/Console/Commands/OneOff/PopulateTreeSpeciesResearch.php b/app/Console/Commands/OneOff/PopulateTreeSpeciesResearch.php
@@ -7,6 +7,7 @@
 use App\Console\Commands\Traits\ExceptionLevel;
 use App\Models\V2\TreeSpecies\TreeSpeciesResearch;
 use Illuminate\Console\Command;
+use Illuminate\Support\Carbon;
 use Symfony\Component\Process\Process;
 
 class PopulateTreeSpeciesResearch extends Command
@@ -59,10 +60,15 @@ public function handle()
 
             $this->withProgressBar($lines, function ($progressBar) use ($fileHandle) {
                 $abortExceptions = [];
+                $bulkInsert = [];
                 while ($csvRow = fgetcsv($fileHandle)) {
                     $data = [];
+                    $now = Carbon::now();
                     foreach ($this->columns as $column => $index) {
                         $data[$column] = $csvRow[$index];
+                        // These don't get set automatically with bulk insert
+                        $data['created_at'] = $now;
+                        $data['updated_at'] = $now;
                     }
 
                     try {
@@ -73,10 +79,16 @@ public function handle()
                                 'existing_id' => $existing?->taxon_id,
                                 'new_id' => $data['taxon_id'],
                                 'scientific_name' => $data['scientific_name'],
+                                'infraspecific_epithet' => $data['infraspecific_epithet'],
                             ], JSON_PRETTY_PRINT),
                             ExceptionLevel::Warning
                         );
-                        TreeSpeciesResearch::create($data);
+
+                        $bulkInsert[] = $data;
+                        if (count($bulkInsert) >= 1000) {
+                            TreeSpeciesResearch::insert($bulkInsert);
+                            $bulkInsert = [];
+                        }
                     } catch (AbortException $e) {
                         $abortExceptions[] = $e;
                     }