Skip to content

Commit

Permalink
Merge pull request #684 from wri/release/xrayed-xylocarpus
Browse files Browse the repository at this point in the history
[RELEASE] X-Rayed Xylocarpus
  • Loading branch information
roguenet authored Feb 6, 2025
2 parents e1722fa + 8defa4a commit 94f6a8a
Show file tree
Hide file tree
Showing 426 changed files with 6,187 additions and 15,401 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ swagger.json
.php-cs-fixer.cache
.phpunit.result.cache
.DS_Store
/data
80 changes: 41 additions & 39 deletions app/Console/Commands/BulkWorkdayImport.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@
use App\Console\Commands\Traits\ExceptionLevel;
use App\Models\SiteSubmission;
use App\Models\Submission;
use App\Models\V2\Demographics\Demographic;
use App\Models\V2\Demographics\DemographicCollections;
use App\Models\V2\Projects\ProjectReport;
use App\Models\V2\Sites\SiteReport;
use App\Models\V2\Workdays\Workday;
use Illuminate\Console\Command;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
Expand All @@ -34,27 +35,27 @@ class BulkWorkdayImport extends Command

protected const COLLECTIONS = [
'sites' => [
'Paid_site_establishment' => Workday::COLLECTION_SITE_PAID_SITE_ESTABLISHMENT,
'Vol_site_establishment' => Workday::COLLECTION_SITE_VOLUNTEER_SITE_ESTABLISHMENT,
'Paid_planting' => Workday::COLLECTION_SITE_PAID_PLANTING,
'Vol_planting' => Workday::COLLECTION_SITE_VOLUNTEER_PLANTING,
'Paid_monitoring' => Workday::COLLECTION_SITE_PAID_SITE_MONITORING,
'Vol_monitoring' => Workday::COLLECTION_SITE_VOLUNTEER_SITE_MONITORING,
'Paid_maintenance' => Workday::COLLECTION_SITE_PAID_SITE_MAINTENANCE,
'Vol_maintenance' => Workday::COLLECTION_SITE_VOLUNTEER_SITE_MAINTENANCE,
'Paid_other' => Workday::COLLECTION_SITE_PAID_OTHER,
'Vol_other' => Workday::COLLECTION_SITE_VOLUNTEER_OTHER,
'Paid_site_establishment' => DemographicCollections::PAID_SITE_ESTABLISHMENT,
'Vol_site_establishment' => DemographicCollections::VOLUNTEER_SITE_ESTABLISHMENT,
'Paid_planting' => DemographicCollections::PAID_PLANTING,
'Vol_planting' => DemographicCollections::VOLUNTEER_PLANTING,
'Paid_monitoring' => DemographicCollections::PAID_SITE_MONITORING,
'Vol_monitoring' => DemographicCollections::VOLUNTEER_SITE_MONITORING,
'Paid_maintenance' => DemographicCollections::PAID_SITE_MAINTENANCE,
'Vol_maintenance' => DemographicCollections::VOLUNTEER_SITE_MAINTENANCE,
'Paid_other' => DemographicCollections::PAID_OTHER,
'Vol_other' => DemographicCollections::VOLUNTEER_OTHER,
],

'projects' => [
'Paid_project_management' => Workday::COLLECTION_PROJECT_PAID_PROJECT_MANAGEMENT,
'Vol_project_management' => Workday::COLLECTION_PROJECT_VOLUNTEER_PROJECT_MANAGEMENT,
'Paid_nursery_ops' => Workday::COLLECTION_PROJECT_PAID_NURSERY_OPERATIONS,
'Vol_nursery_ops' => Workday::COLLECTION_PROJECT_VOLUNTEER_NURSERY_OPERATIONS,
'Paid_seed_collection' => Workday::COLLECTION_PROJECT_PAID_NURSERY_OPERATIONS,
'Vol_seed_collection' => Workday::COLLECTION_PROJECT_VOLUNTEER_NURSERY_OPERATIONS,
'Paid_other' => Workday::COLLECTION_PROJECT_PAID_OTHER,
'Vol_other' => Workday::COLLECTION_PROJECT_VOLUNTEER_OTHER,
'Paid_project_management' => DemographicCollections::PAID_PROJECT_MANAGEMENT,
'Vol_project_management' => DemographicCollections::VOLUNTEER_PROJECT_MANAGEMENT,
'Paid_nursery_ops' => DemographicCollections::PAID_NURSERY_OPERATIONS,
'Vol_nursery_ops' => DemographicCollections::VOLUNTEER_NURSERY_OPERATIONS,
'Paid_seed_collection' => DemographicCollections::PAID_NURSERY_OPERATIONS,
'Vol_seed_collection' => DemographicCollections::VOLUNTEER_NURSERY_OPERATIONS,
'Paid_other' => DemographicCollections::PAID_OTHER,
'Vol_other' => DemographicCollections::VOLUNTEER_OTHER,
],
];

Expand All @@ -77,18 +78,18 @@ class BulkWorkdayImport extends Command
];

protected const DEMOGRAPHICS = [
'women' => ['type' => 'gender', 'subtype' => null, 'name' => 'female'],
'men' => ['type' => 'gender', 'subtype' => null, 'name' => 'male'],
'non-binary' => ['type' => 'gender', 'subtype' => null, 'name' => 'non-binary'],
'nonbinary' => ['type' => 'gender', 'subtype' => null, 'name' => 'non-binary'],
'gender-unknown' => ['type' => 'gender', 'subtype' => null, 'name' => 'unknown'],
'no-gender' => ['type' => 'gender', 'subtype' => null, 'name' => 'unknown'],

'youth_15-24' => ['type' => 'age', 'subtype' => null, 'name' => 'youth'],
'adult_24-64' => ['type' => 'age', 'subtype' => null, 'name' => 'adult'],
'elder_65+' => ['type' => 'age', 'subtype' => null, 'name' => 'elder'],
'age-unknown' => ['type' => 'age', 'subtype' => null, 'name' => 'unknown'],
'age_unknown' => ['type' => 'age', 'subtype' => null, 'name' => 'unknown'],
'women' => ['type' => 'gender', 'subtype' => 'female', 'name' => null],
'men' => ['type' => 'gender', 'subtype' => 'male', 'name' => null],
'non-binary' => ['type' => 'gender', 'subtype' => 'non-binary', 'name' => null],
'nonbinary' => ['type' => 'gender', 'subtype' => 'non-binary', 'name' => null],
'gender-unknown' => ['type' => 'gender', 'subtype' => 'unknown', 'name' => null],
'no-gender' => ['type' => 'gender', 'subtype' => 'unknown', 'name' => null],

'youth_15-24' => ['type' => 'age', 'subtype' => 'youth', 'name' => null],
'adult_24-64' => ['type' => 'age', 'subtype' => 'adult', 'name' => null],
'elder_65+' => ['type' => 'age', 'subtype' => 'elder', 'name' => null],
'age-unknown' => ['type' => 'age', 'subtype' => 'unknown', 'name' => null],
'age_unknown' => ['type' => 'age', 'subtype' => 'unknown', 'name' => null],

'indigenous' => ['type' => 'ethnicity', 'subtype' => 'indigenous', 'name' => null],
'ethnicity-other' => ['type' => 'ethnicity', 'subtype' => 'other', 'name' => null],
Expand Down Expand Up @@ -278,8 +279,8 @@ protected function parseRow($csvRow, &$parseErrors): ?array

// Check that all the demographics are balanced
$collections = array_merge(
$this->modelConfig['model']::WORKDAY_COLLECTIONS['paid'],
$this->modelConfig['model']::WORKDAY_COLLECTIONS['volunteer'],
$this->modelConfig['model']::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['paid'],
$this->modelConfig['model']::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['volunteer'],
);
foreach ($collections as $collection) {
if (empty($row[$collection])) {
Expand Down Expand Up @@ -348,8 +349,8 @@ protected function getData($collection, $demographic, $cell, $row): array
protected function persistWorkdays($report, $data): void
{
$collections = array_merge(
get_class($report)::WORKDAY_COLLECTIONS['paid'],
get_class($report)::WORKDAY_COLLECTIONS['volunteer'],
get_class($report)::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['paid'],
get_class($report)::DEMOGRAPHIC_COLLECTIONS[Demographic::WORKDAY_TYPE]['volunteer'],
);

$modelDescription = Str::replace('-', ' ', Str::title($report->shortName)) .
Expand All @@ -371,14 +372,15 @@ protected function persistWorkdays($report, $data): void
}

$this->info("Populating collection $collection\n");
$workday = Workday::create([
'workdayable_type' => get_class($report),
'workdayable_id' => $report->id,
$workday = Demographic::create([
'demographical_type' => get_class($report),
'demographical_id' => $report->id,
'type' => Demographic::WORKDAY_TYPE,
'collection' => $collection,
]);

foreach ($data[$collection] as $demographicData) {
$workday->demographics()->create($demographicData);
$workday->entries()->create($demographicData);
}
}

Expand Down
44 changes: 44 additions & 0 deletions app/Console/Commands/OneOff/FixDemographicNameType.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
<?php

namespace App\Console\Commands\OneOff;

use App\Models\V2\Demographics\DemographicEntry;
use Illuminate\Console\Command;

class FixDemographicNameType extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'one-off:fix-demographic-name-type';

/**
* The console command description.
*
* @var string
*/
protected $description = 'Moves several system-defined names from the name column to type in demographic_entities';

protected const UPDATED_MAPPINGS = [
DemographicEntry::GENDER => DemographicEntry::GENDERS,
DemographicEntry::AGE => DemographicEntry::AGES,
DemographicEntry::CASTE => DemographicEntry::CASTES,
];

/**
* Execute the console command.
*/
public function handle()
{
foreach (self::UPDATED_MAPPINGS as $type => $subtypes) {
foreach ($subtypes as $subtype) {
$this->info("Updating subtype mapping [type=$type, subtype=$subtype]");
DemographicEntry::withTrashed()
->where(['type' => $type, 'name' => $subtype])
->update(['subtype' => $subtype, 'name' => null, 'updated_at' => 'now()']);
}
}
}
}
48 changes: 24 additions & 24 deletions app/Console/Commands/OneOff/FixDuplicateDemographics.php
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
namespace App\Console\Commands\OneOff;

use App\Models\V2\Demographics\Demographic;
use App\Models\V2\Workdays\Workday;
use App\Models\V2\Demographics\DemographicEntry;
use Illuminate\Console\Command;

class FixDuplicateDemographics extends Command
Expand All @@ -20,43 +20,42 @@ class FixDuplicateDemographics extends Command
*
* @var string
*/
protected $description = 'Fixes workday demographics that have been duplicated';
protected $description = 'Fixes demographics that have been duplicated';

/**
* Execute the console command.
*/
public function handle()
{
$workdays = [];
$demographics = [];

Demographic::selectRaw('demographical_id, type, subtype, name, count(*) as num, sum(amount) as sum')
DemographicEntry::selectRaw('demographic_id, type, subtype, name, count(*) as num, sum(amount) as sum')
// group by is case insensitive, so in order to avoid turning up false positives, we cast to binary
// to get the DB to recognize different casing.
->groupByRaw('demographical_id, type, subtype, name, Cast(name as binary)')
->where('demographical_type', Workday::class)
->groupByRaw('demographic_id, type, subtype, name, Cast(name as binary)')
->orderByRaw('num desc')
->chunk(10, function ($chunk) use (&$workdays) {
foreach ($chunk as $demographic) {
if ($demographic->num == 1) {
->chunk(10, function ($chunk) use (&$demographics) {
foreach ($chunk as $entry) {
if ($entry->num == 1) {
return false;
}

$demographic['rows'] = Demographic::where([
'demographical_id' => $demographic->demographical_id,
'demographical_type' => Workday::class,
'type' => $demographic->type,
'subtype' => $demographic->subtype,
'name' => $demographic->name,
$entry['rows'] = DemographicEntry::where([
'demographic_id' => $entry->demographic_id,
'type' => $entry->type,
'subtype' => $entry->subtype,
'name' => $entry->name,
])->select('id', 'amount')->get()->toArray();
$workdays[$demographic->demographical_id][] = $demographic->toArray();
$demographics[$entry->demographic_id][] = $entry->toArray();
}

return true;
});

foreach ($workdays as $workdayId => $stats) {
foreach ($demographics as $demographicId => $stats) {
foreach ($stats as $stat) {
$workday = Workday::withTrashed()->find($workdayId);
/** @var Demographic $demographic */
$demographic = Demographic::withTrashed()->find($demographicId);

$rows = collect($stat['rows']);
$max = $rows->max('amount');
Expand All @@ -78,11 +77,12 @@ public function handle()
}

$info = [
'workday_id' => $workdayId,
'workdayable_type' => $workday->workdayable_type,
'workdayable_id' => $workday->workdayable_id,
'workdayable_uuid' => $workday->workdayable()->withTrashed()->first()->uuid,
'collection' => $workday->collection,
'demographic_id' => $demographicId,
'demographical_type' => $demographic->demographical_type,
'demographical_id' => $demographic->demographical_id,
'demographical_uuid' => $demographic->demographical()->withTrashed()->first()->uuid,
'demographic_type' => $demographic->type,
'collection' => $demographic->collection,
'type' => $stat['type'],
'subtype' => $stat['subtype'],
'name' => $stat['name'],
Expand All @@ -97,7 +97,7 @@ public function handle()
unset($row);
foreach ($stat['rows'] as $row) {
if ($row['action'] == 'delete') {
Demographic::find($row['id'])->delete();
DemographicEntry::find($row['id'])->delete();
}
}
}
Expand Down
103 changes: 103 additions & 0 deletions app/Console/Commands/OneOff/FixDuplicateWorkdays.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
<?php

namespace App\Console\Commands\OneOff;

use App\Console\Commands\Traits\Abortable;
use App\Console\Commands\Traits\AbortException;
use App\Models\V2\Demographics\Demographic;
use Illuminate\Console\Command;

class FixDuplicateWorkdays extends Command
{
use Abortable;

/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'one-off:fix-duplicate-workdays';

/**
* The console command description.
*
* @var string
*/
protected $description = 'Fixes instances of duplicated Workdays';

/**
* Execute the console command.
*/
public function handle()
{
$this->executeAbortableScript(function () {
# Find all instances of workdays that are duplicated across the same type, id and collection.
$dupes = Demographic::select('demographical_type', 'demographical_id', 'collection')
->type(Demographic::WORKDAY_TYPE)
->groupBy('demographical_type', 'demographical_id', 'collection')
->havingRaw('count(*) > ?', [1])
->get();

$errors = [];
foreach ($dupes as $dupe) {
try {
$this->removeDuplicates($dupe);
} catch (AbortException $e) {
$errors[] = $e;
}
}

$this->info("Completed processing {$dupes->count()} duplicated workdays\n\n");

if (count($errors) > 0) {
$this->warn('Entries that were not resolved: ');
foreach ($errors as $error) {
$this->warn($error->getMessage());
}
}
});
}

/**
* @throws AbortException
*/
private function removeDuplicates($dupe)
{
$workdays = Demographic::where([
'demographical_type' => $dupe->demographical_type,
'demographical_id' => $dupe->demographical_id,
'type' => Demographic::WORKDAY_TYPE,
'collection' => $dupe->collection,
])->get();

$type = explode('\\', $dupe->demographical_type);
$params = json_encode([
'type' => array_pop($type),
'id' => $dupe->demographical_id,
'uuid' => $workdays->first()->demographical->uuid,
'collection' => $dupe->collection,
]);
$this->assert(
$workdays->map(fn ($w) => $w->visible)->unique()->count() == 1,
"Visible not identical: $params"
);
$this->assert(
$workdays->map(fn ($w) => $w->description)->unique()->count() == 1,
"Description not identical: $params"
);

// Some of these workdays have had updates. This check makes sure that either all the demographics have the
// same updated stamp (meaning that none have been updated, or they were all updated together), or that if
// there is something updated, the first one on the list is the most recently updated one, as we would expect.
$mostRecentUpdated = $workdays->sortBy('updated_at')->last();
$numberUpdatedDates = $workdays->map(fn ($w) => $w->updated_at)->unique()->count();
$this->assert($numberUpdatedDates == 1 || $mostRecentUpdated == $workdays->first(), "First is not the most recently updated: $params");

// If we made it through the checks above, it's considered safe to delete all the workdays after the first one.
$this->info("Removing dupes: [$dupe->demographical_type, $dupe->demographical_id, $dupe->collection]");
foreach ($workdays->slice(1) as $workday) {
$workday->entries()->delete();
$workday->delete();
}
}
}
Loading

0 comments on commit 94f6a8a

Please sign in to comment.