[RELEASE] X-Rayed Xylocarpus

.gitignore

@@ -20,3 +20,4 @@ swagger.json
 .php-cs-fixer.cache
 .phpunit.result.cache
 .DS_Store
+/data

app/Console/Commands/BulkWorkdayImport.php

@@ -7,9 +7,10 @@
 use App\Console\Commands\Traits\ExceptionLevel;
 use App\Models\SiteSubmission;
 use App\Models\Submission;
+use App\Models\V2\Demographics\Demographic;
+use App\Models\V2\Demographics\DemographicCollections;
 use App\Models\V2\Projects\ProjectReport;
 use App\Models\V2\Sites\SiteReport;
-use App\Models\V2\Workdays\Workday;
 use Illuminate\Console\Command;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Str;
@@ -34,27 +35,27 @@ class BulkWorkdayImport extends Command
 
     protected const COLLECTIONS = [
         'sites' => [
-            'Paid_site_establishment' => Workday::COLLECTION_SITE_PAID_SITE_ESTABLISHMENT,
-            'Vol_site_establishment' => Workday::COLLECTION_SITE_VOLUNTEER_SITE_ESTABLISHMENT,
-            'Paid_planting' => Workday::COLLECTION_SITE_PAID_PLANTING,
-            'Vol_planting' => Workday::COLLECTION_SITE_VOLUNTEER_PLANTING,
-            'Paid_monitoring' => Workday::COLLECTION_SITE_PAID_SITE_MONITORING,
-            'Vol_monitoring' => Workday::COLLECTION_SITE_VOLUNTEER_SITE_MONITORING,
-            'Paid_maintenance' => Workday::COLLECTION_SITE_PAID_SITE_MAINTENANCE,
-            'Vol_maintenance' => Workday::COLLECTION_SITE_VOLUNTEER_SITE_MAINTENANCE,
-            'Paid_other' => Workday::COLLECTION_SITE_PAID_OTHER,
-            'Vol_other' => Workday::COLLECTION_SITE_VOLUNTEER_OTHER,
+            'Paid_site_establishment' => DemographicCollections::PAID_SITE_ESTABLISHMENT,
+            'Vol_site_establishment' => DemographicCollections::VOLUNTEER_SITE_ESTABLISHMENT,
+            'Paid_planting' => DemographicCollections::PAID_PLANTING,
+            'Vol_planting' => DemographicCollections::VOLUNTEER_PLANTING,
+            'Paid_monitoring' => DemographicCollections::PAID_SITE_MONITORING,
+            'Vol_monitoring' => DemographicCollections::VOLUNTEER_SITE_MONITORING,
+            'Paid_maintenance' => DemographicCollections::PAID_SITE_MAINTENANCE,
+            'Vol_maintenance' => DemographicCollections::VOLUNTEER_SITE_MAINTENANCE,
+            'Paid_other' => DemographicCollections::PAID_OTHER,
+            'Vol_other' => DemographicCollections::VOLUNTEER_OTHER,
         ],
 
         'projects' => [
-            'Paid_project_management' => Workday::COLLECTION_PROJECT_PAID_PROJECT_MANAGEMENT,
-            'Vol_project_management' => Workday::COLLECTION_PROJECT_VOLUNTEER_PROJECT_MANAGEMENT,
-            'Paid_nursery_ops' => Workday::COLLECTION_PROJECT_PAID_NURSERY_OPERATIONS,
-            'Vol_nursery_ops' => Workday::COLLECTION_PROJECT_VOLUNTEER_NURSERY_OPERATIONS,
-            'Paid_seed_collection' => Workday::COLLECTION_PROJECT_PAID_NURSERY_OPERATIONS,
-            'Vol_seed_collection' => Workday::COLLECTION_PROJECT_VOLUNTEER_NURSERY_OPERATIONS,
-            'Paid_other' => Workday::COLLECTION_PROJECT_PAID_OTHER,
-            'Vol_other' => Workday::COLLECTION_PROJECT_VOLUNTEER_OTHER,
+            'Paid_project_management' => DemographicCollections::PAID_PROJECT_MANAGEMENT,
+            'Vol_project_management' => DemographicCollections::VOLUNTEER_PROJECT_MANAGEMENT,
+            'Paid_nursery_ops' => DemographicCollections::PAID_NURSERY_OPERATIONS,
+            'Vol_nursery_ops' => DemographicCollections::VOLUNTEER_NURSERY_OPERATIONS,
+            'Paid_seed_collection' => DemographicCollections::PAID_NURSERY_OPERATIONS,
+            'Vol_seed_collection' => DemographicCollections::VOLUNTEER_NURSERY_OPERATIONS,
+            'Paid_other' => DemographicCollections::PAID_OTHER,
+            'Vol_other' => DemographicCollections::VOLUNTEER_OTHER,
         ],
     ];
 
@@ -77,18 +78,18 @@ class BulkWorkdayImport extends Command
     ];
 
     protected const DEMOGRAPHICS = [
-        'women' => ['type' => 'gender', 'subtype' => null, 'name' => 'female'],
-        'men' => ['type' => 'gender', 'subtype' => null, 'name' => 'male'],
-        'non-binary' => ['type' => 'gender', 'subtype' => null, 'name' => 'non-binary'],
-        'nonbinary' => ['type' => 'gender', 'subtype' => null, 'name' => 'non-binary'],
-        'gender-unknown' => ['type' => 'gender', 'subtype' => null, 'name' => 'unknown'],
-        'no-gender' => ['type' => 'gender', 'subtype' => null, 'name' => 'unknown'],
-
-        'youth_15-24' => ['type' => 'age', 'subtype' => null, 'name' => 'youth'],
-        'adult_24-64' => ['type' => 'age', 'subtype' => null, 'name' => 'adult'],
-        'elder_65+' => ['type' => 'age', 'subtype' => null, 'name' => 'elder'],
-        'age-unknown' => ['type' => 'age', 'subtype' => null, 'name' => 'unknown'],
-        'age_unknown' => ['type' => 'age', 'subtype' => null, 'name' => 'unknown'],
+        'women' => ['type' => 'gender', 'subtype' => 'female', 'name' => null],
+        'men' => ['type' => 'gender', 'subtype' => 'male', 'name' => null],
+        'non-binary' => ['type' => 'gender', 'subtype' => 'non-binary', 'name' => null],
+        'nonbinary' => ['type' => 'gender', 'subtype' => 'non-binary', 'name' => null],
+        'gender-unknown' => ['type' => 'gender', 'subtype' => 'unknown', 'name' => null],
+        'no-gender' => ['type' => 'gender', 'subtype' => 'unknown', 'name' => null],
+
+        'youth_15-24' => ['type' => 'age', 'subtype' => 'youth', 'name' => null],
+        'adult_24-64' => ['type' => 'age', 'subtype' => 'adult', 'name' => null],
+        'elder_65+' => ['type' => 'age', 'subtype' => 'elder', 'name' => null],
+        'age-unknown' => ['type' => 'age', 'subtype' => 'unknown', 'name' => null],
+        'age_unknown' => ['type' => 'age', 'subtype' => 'unknown', 'name' => null],
 
         'indigenous' => ['type' => 'ethnicity', 'subtype' => 'indigenous', 'name' => null],
         'ethnicity-other' => ['type' => 'ethnicity', 'subtype' => 'other', 'name' => null],
@@ -278,8 +279,8 @@ protected function parseRow($csvRow, &$parseErrors): ?array
 
         // Check that all the demographics are balanced
         $collections = array_merge(
-            $this->modelConfig['model']::WORKDAY_COLLECTIONS['paid'],
-            $this->modelConfig['model']::WORKDAY_COLLECTIONS['volunteer'],
+            $this->modelConfig['model']::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['paid'],
+            $this->modelConfig['model']::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['volunteer'],
         );
         foreach ($collections as $collection) {
             if (empty($row[$collection])) {
@@ -348,8 +349,8 @@ protected function getData($collection, $demographic, $cell, $row): array
     protected function persistWorkdays($report, $data): void
     {
         $collections = array_merge(
-            get_class($report)::WORKDAY_COLLECTIONS['paid'],
-            get_class($report)::WORKDAY_COLLECTIONS['volunteer'],
+            get_class($report)::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['paid'],
+            get_class($report)::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['volunteer'],
         );
 
         $modelDescription = Str::replace('-', ' ', Str::title($report->shortName)) .
@@ -371,14 +372,15 @@ protected function persistWorkdays($report, $data): void
             }
 
             $this->info("Populating collection $collection\n");
-            $workday = Workday::create([
-                'workdayable_type' => get_class($report),
-                'workdayable_id' => $report->id,
+            $workday = Demographic::create([
+                'demographical_type' => get_class($report),
+                'demographical_id' => $report->id,
+                'type' => Demographic::WORKDAY_TYPE,
                 'collection' => $collection,
             ]);
 
             foreach ($data[$collection] as $demographicData) {
-                $workday->demographics()->create($demographicData);
+                $workday->entries()->create($demographicData);
             }
         }
 

app/Console/Commands/OneOff/FixDemographicNameType.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace App\Console\Commands\OneOff;
+
+use App\Models\V2\Demographics\DemographicEntry;
+use Illuminate\Console\Command;
+
+class FixDemographicNameType extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'one-off:fix-demographic-name-type';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Moves several system-defined names from the name column to type in demographic_entities';
+
+    protected const UPDATED_MAPPINGS = [
+        DemographicEntry::GENDER => DemographicEntry::GENDERS,
+        DemographicEntry::AGE => DemographicEntry::AGES,
+        DemographicEntry::CASTE => DemographicEntry::CASTES,
+    ];
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        foreach (self::UPDATED_MAPPINGS as $type => $subtypes) {
+            foreach ($subtypes as $subtype) {
+                $this->info("Updating subtype mapping [type=$type, subtype=$subtype]");
+                DemographicEntry::withTrashed()
+                    ->where(['type' => $type, 'name' => $subtype])
+                    ->update(['subtype' => $subtype, 'name' => null, 'updated_at' => 'now()']);
+            }
+        }
+    }
+}

app/Console/Commands/OneOff/FixDuplicateDemographics.php

@@ -3,7 +3,7 @@
 namespace App\Console\Commands\OneOff;
 
 use App\Models\V2\Demographics\Demographic;
-use App\Models\V2\Workdays\Workday;
+use App\Models\V2\Demographics\DemographicEntry;
 use Illuminate\Console\Command;
 
 class FixDuplicateDemographics extends Command
@@ -20,43 +20,42 @@ class FixDuplicateDemographics extends Command
      *
      * @var string
      */
-    protected $description = 'Fixes workday demographics that have been duplicated';
+    protected $description = 'Fixes demographics that have been duplicated';
 
     /**
      * Execute the console command.
      */
     public function handle()
     {
-        $workdays = [];
+        $demographics = [];
 
-        Demographic::selectRaw('demographical_id, type, subtype, name, count(*) as num, sum(amount) as sum')
+        DemographicEntry::selectRaw('demographic_id, type, subtype, name, count(*) as num, sum(amount) as sum')
             // group by is case insensitive, so in order to avoid turning up false positives, we cast to binary
             // to get the DB to recognize different casing.
-            ->groupByRaw('demographical_id, type, subtype, name, Cast(name as binary)')
-            ->where('demographical_type', Workday::class)
+            ->groupByRaw('demographic_id, type, subtype, name, Cast(name as binary)')
             ->orderByRaw('num desc')
-            ->chunk(10, function ($chunk) use (&$workdays) {
-                foreach ($chunk as $demographic) {
-                    if ($demographic->num == 1) {
+            ->chunk(10, function ($chunk) use (&$demographics) {
+                foreach ($chunk as $entry) {
+                    if ($entry->num == 1) {
                         return false;
                     }
 
-                    $demographic['rows'] = Demographic::where([
-                        'demographical_id' => $demographic->demographical_id,
-                        'demographical_type' => Workday::class,
-                        'type' => $demographic->type,
-                        'subtype' => $demographic->subtype,
-                        'name' => $demographic->name,
+                    $entry['rows'] = DemographicEntry::where([
+                        'demographic_id' => $entry->demographic_id,
+                        'type' => $entry->type,
+                        'subtype' => $entry->subtype,
+                        'name' => $entry->name,
                     ])->select('id', 'amount')->get()->toArray();
-                    $workdays[$demographic->demographical_id][] = $demographic->toArray();
+                    $demographics[$entry->demographic_id][] = $entry->toArray();
                 }
 
                 return true;
             });
 
-        foreach ($workdays as $workdayId => $stats) {
+        foreach ($demographics as $demographicId => $stats) {
             foreach ($stats as $stat) {
-                $workday = Workday::withTrashed()->find($workdayId);
+                /** @var Demographic $demographic */
+                $demographic = Demographic::withTrashed()->find($demographicId);
 
                 $rows = collect($stat['rows']);
                 $max = $rows->max('amount');
@@ -78,11 +77,12 @@ public function handle()
                 }
 
                 $info = [
-                    'workday_id' => $workdayId,
-                    'workdayable_type' => $workday->workdayable_type,
-                    'workdayable_id' => $workday->workdayable_id,
-                    'workdayable_uuid' => $workday->workdayable()->withTrashed()->first()->uuid,
-                    'collection' => $workday->collection,
+                    'demographic_id' => $demographicId,
+                    'demographical_type' => $demographic->demographical_type,
+                    'demographical_id' => $demographic->demographical_id,
+                    'demographical_uuid' => $demographic->demographical()->withTrashed()->first()->uuid,
+                    'demographic_type' => $demographic->type,
+                    'collection' => $demographic->collection,
                     'type' => $stat['type'],
                     'subtype' => $stat['subtype'],
                     'name' => $stat['name'],
@@ -97,7 +97,7 @@ public function handle()
                 unset($row);
                 foreach ($stat['rows'] as $row) {
                     if ($row['action'] == 'delete') {
-                        Demographic::find($row['id'])->delete();
+                        DemographicEntry::find($row['id'])->delete();
                     }
                 }
             }

app/Console/Commands/OneOff/FixDuplicateWorkdays.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace App\Console\Commands\OneOff;
+
+use App\Console\Commands\Traits\Abortable;
+use App\Console\Commands\Traits\AbortException;
+use App\Models\V2\Demographics\Demographic;
+use Illuminate\Console\Command;
+
+class FixDuplicateWorkdays extends Command
+{
+    use Abortable;
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'one-off:fix-duplicate-workdays';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Fixes instances of duplicated Workdays';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->executeAbortableScript(function () {
+            # Find all instances of workdays that are duplicated across the same type, id and collection.
+            $dupes = Demographic::select('demographical_type', 'demographical_id', 'collection')
+                ->type(Demographic::WORKDAY_TYPE)
+                ->groupBy('demographical_type', 'demographical_id', 'collection')
+                ->havingRaw('count(*) > ?', [1])
+                ->get();
+
+            $errors = [];
+            foreach ($dupes as $dupe) {
+                try {
+                    $this->removeDuplicates($dupe);
+                } catch (AbortException $e) {
+                    $errors[] = $e;
+                }
+            }
+
+            $this->info("Completed processing {$dupes->count()} duplicated workdays\n\n");
+
+            if (count($errors) > 0) {
+                $this->warn('Entries that were not resolved: ');
+                foreach ($errors as $error) {
+                    $this->warn($error->getMessage());
+                }
+            }
+        });
+    }
+
+    /**
+     * @throws AbortException
+     */
+    private function removeDuplicates($dupe)
+    {
+        $workdays = Demographic::where([
+            'demographical_type' => $dupe->demographical_type,
+            'demographical_id' => $dupe->demographical_id,
+            'type' => Demographic::WORKDAY_TYPE,
+            'collection' => $dupe->collection,
+        ])->get();
+
+        $type = explode('\\', $dupe->demographical_type);
+        $params = json_encode([
+            'type' => array_pop($type),
+            'id' => $dupe->demographical_id,
+            'uuid' => $workdays->first()->demographical->uuid,
+            'collection' => $dupe->collection,
+        ]);
+        $this->assert(
+            $workdays->map(fn ($w) => $w->visible)->unique()->count() == 1,
+            "Visible not identical: $params"
+        );
+        $this->assert(
+            $workdays->map(fn ($w) => $w->description)->unique()->count() == 1,
+            "Description not identical: $params"
+        );
+
+        // Some of these workdays have had updates. This check makes sure that either all the demographics have the
+        // same updated stamp (meaning that none have been updated, or they were all updated together), or that if
+        // there is something updated, the first one on the list is the most recently updated one, as we would expect.
+        $mostRecentUpdated = $workdays->sortBy('updated_at')->last();
+        $numberUpdatedDates = $workdays->map(fn ($w) => $w->updated_at)->unique()->count();
+        $this->assert($numberUpdatedDates == 1 || $mostRecentUpdated == $workdays->first(), "First is not the most recently updated: $params");
+
+        // If we made it through the checks above, it's considered safe to delete all the workdays after the first one.
+        $this->info("Removing dupes: [$dupe->demographical_type, $dupe->demographical_id, $dupe->collection]");
+        foreach ($workdays->slice(1) as $workday) {
+            $workday->entries()->delete();
+            $workday->delete();
+        }
+    }
+}