From ffdbff1a91ef77058f854c135fd177026739689b Mon Sep 17 00:00:00 2001 From: Limber Mamani <154026979+LimberHope@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:08:10 -0400 Subject: [PATCH 1/3] [TM-1466] remove cached functinality from Redis (#618) --- ...GetPolygonsIndicatorAnalysisController.php | 24 +++++++++---------- .../RunIndicatorAnalysisController.php | 22 +++++++---------- app/Jobs/RunIndicatorAnalysisJob.php | 5 +--- app/Services/RunIndicatorAnalysisService.php | 6 ++--- 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/app/Http/Controllers/V2/MonitoredData/GetPolygonsIndicatorAnalysisController.php b/app/Http/Controllers/V2/MonitoredData/GetPolygonsIndicatorAnalysisController.php index 85d6ee58..73961c48 100644 --- a/app/Http/Controllers/V2/MonitoredData/GetPolygonsIndicatorAnalysisController.php +++ b/app/Http/Controllers/V2/MonitoredData/GetPolygonsIndicatorAnalysisController.php @@ -78,7 +78,7 @@ public function __invoke(EntityModel $entity, string $slug) 'status' => $polygon->status, 'plantstart' => $polygon->plantstart ?? '-', 'site_name' => $polygon->site->name ?? '-', - 'size' => round($polygon->calc_area ?? 0, 3), + 'size' => round($polygon->calc_area ?? 0, 1), 'indicator_slug' => $indicator->indicator_slug, 'year_of_analysis' => $indicator->year_of_analysis, 'created_at' => $indicator->created_at, @@ -87,16 +87,16 @@ public function __invoke(EntityModel $entity, string $slug) ]; if (str_contains($slug, 'treeCoverLoss')) { $valueYears = json_decode($indicator->value, true); - $results['data']['2015'] = round((float) $valueYears['2015'], 3); - $results['data']['2016'] = round((float) $valueYears['2016'], 3); - $results['data']['2017'] = round((float) $valueYears['2017'], 3); - $results['data']['2018'] = round((float) $valueYears['2018'], 3); - $results['data']['2019'] = round((float) $valueYears['2019'], 3); - $results['data']['2020'] = round((float) $valueYears['2020'], 3); - $results['data']['2021'] = round((float) $valueYears['2021'], 3); - $results['data']['2022'] = round((float) $valueYears['2022'], 3); - $results['data']['2023'] = round((float) $valueYears['2023'], 3); - $results['data']['2024'] = round((float) $valueYears['2024'], 3); + $results['data']['2015'] = round((float) $valueYears['2015'], 1); + $results['data']['2016'] = round((float) $valueYears['2016'], 1); + $results['data']['2017'] = round((float) $valueYears['2017'], 1); + $results['data']['2018'] = round((float) $valueYears['2018'], 1); + $results['data']['2019'] = round((float) $valueYears['2019'], 1); + $results['data']['2020'] = round((float) $valueYears['2020'], 1); + $results['data']['2021'] = round((float) $valueYears['2021'], 1); + $results['data']['2022'] = round((float) $valueYears['2022'], 1); + $results['data']['2023'] = round((float) $valueYears['2023'], 1); + $results['data']['2024'] = round((float) $valueYears['2024'], 1); } if ($slug == 'restorationByEcoRegion') { @@ -123,7 +123,7 @@ public function processValuesHectares($values) $array = explode(',', str_replace('-', '_', $key)); $arrayTrim = array_map('trim', $array); foreach ($arrayTrim as $item) { - $separateKeys[$item] = round((float) $value, 3); + $separateKeys[$item] = round((float) $value, 1); } } diff --git a/app/Http/Controllers/V2/MonitoredData/RunIndicatorAnalysisController.php b/app/Http/Controllers/V2/MonitoredData/RunIndicatorAnalysisController.php index 2a55ba64..08307b03 100644 --- a/app/Http/Controllers/V2/MonitoredData/RunIndicatorAnalysisController.php +++ b/app/Http/Controllers/V2/MonitoredData/RunIndicatorAnalysisController.php @@ -8,7 +8,6 @@ use App\Models\DelayedJob; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; -use Illuminate\Support\Facades\Redis; class RunIndicatorAnalysisController extends Controller { @@ -16,20 +15,15 @@ public function __invoke(Request $request, string $slug) { try { $requestData = $request->all(); - $binary_data = Redis::get('run:indicator|'.$slug.'|'.json_encode($requestData['uuids'])); - if (! $binary_data) { - $delayedJob = DelayedJob::create(); - $job = new RunIndicatorAnalysisJob( - $delayedJob->id, - $requestData, - $slug - ); - dispatch($job); + $delayedJob = DelayedJob::create(); + $job = new RunIndicatorAnalysisJob( + $delayedJob->id, + $requestData, + $slug + ); + dispatch($job); - return (new DelayedJobResource($delayedJob))->additional(['message' => 'Analysis for '.$slug.' is being processed']); - } else { - return response()->json(['message' => 'Analysis for '.$slug.' is already processed'], 200); - } + return (new DelayedJobResource($delayedJob))->additional(['message' => 'Analysis for '.$slug.' is being processed']); } catch (\Exception $e) { Log::error('Error during analysis for ' . $slug . ' : ' . $e->getMessage()); diff --git a/app/Jobs/RunIndicatorAnalysisJob.php b/app/Jobs/RunIndicatorAnalysisJob.php index aabc716d..b7aa76ef 100644 --- a/app/Jobs/RunIndicatorAnalysisJob.php +++ b/app/Jobs/RunIndicatorAnalysisJob.php @@ -12,7 +12,6 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; -use Illuminate\Support\Facades\Redis; class RunIndicatorAnalysisJob implements ShouldQueue { @@ -40,9 +39,7 @@ public function handle(RunIndicatorAnalysisService $runIndicatorAnalysisService) { try { $delayedJob = DelayedJob::findOrFail($this->delayed_job_id); - - $binary_data = $runIndicatorAnalysisService->run($this->request, $this->slug); - Redis::set('run:indicator|'.$this->slug.'|'.json_encode($this->request['uuids']), $binary_data); + $runIndicatorAnalysisService->run($this->request, $this->slug); $delayedJob->update([ 'status' => DelayedJob::STATUS_SUCCEEDED, diff --git a/app/Services/RunIndicatorAnalysisService.php b/app/Services/RunIndicatorAnalysisService.php index 67843e8f..6822441b 100644 --- a/app/Services/RunIndicatorAnalysisService.php +++ b/app/Services/RunIndicatorAnalysisService.php @@ -85,7 +85,7 @@ public function run(array $request, string $slug) $response = $this->sendApiRequestIndicator(getenv('GFW_SECRET_KEY'), $slugMappings[$slug]['query_url'], $slugMappings[$slug]['sql'], $polygonGeometry['geo']); if (str_contains($slug, 'treeCoverLoss')) { - $processedTreeCoverLossValue = $this->processTreeCoverLossValue($response->json()['data']); + $processedTreeCoverLossValue = $this->processTreeCoverLossValue($response->json()['data'], $slugMappings[$slug]['indicator']); } if ($response->successful()) { @@ -142,11 +142,11 @@ public function sendApiRequestIndicator($secret_key, $query_url, $query_sql, $ge ]); } - public function processTreeCoverLossValue($data) + public function processTreeCoverLossValue($data, $indicator) { $processedTreeCoverLossValue = []; foreach ($data as $i) { - $processedTreeCoverLossValue[$i['umd_tree_cover_loss__year']] = $i['area__ha']; + $processedTreeCoverLossValue[$i[$indicator . '__year']] = $i['area__ha']; } return $processedTreeCoverLossValue; From e3d0e8c8a7d2cfa1be3d2eed73d8df2c87ebf84e Mon Sep 17 00:00:00 2001 From: Jorge Monroy Date: Tue, 17 Dec 2024 12:50:35 -0400 Subject: [PATCH 2/3] [TM-1531] delayed job with data (#617) * [TM-1531] entity record and creator to delayedJOb * [TM-1531] add useer to endpoint * [TM-1531] add entity data for polygons validations * [TM-1531] lint * [TM-1531] add is_cleared * [TM-1531] add to fix polygons entity * [TM-1531] store delayed data for uploads * [TM-1531] send mails when job for upload, check or fix is complete * [TM-1531] send correct user for mails * [TM-1531] modify column name on delayed jobs table * [TM-1531] change attribute name to progress message * [TM-1531] lint fix * [TM-1531] change to is_aknowledged value * [TM-1531] change created_by type, change to is_acknowledge * [TM-1531] add name to delayed jobs * [TM-1531] lint * [TM-1531] remove comment --------- Co-authored-by: cesarLima1 --- .../TerrafundClipGeometryController.php | 3 +++ .../TerrafundCreateGeometryController.php | 5 ++++ app/Models/DelayedJob.php | 2 +- .../2024_12_17_150134_add_name_to_jobs.php | 27 +++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 database/migrations/2024_12_17_150134_add_name_to_jobs.php diff --git a/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php b/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php index 04205eda..b9e699c8 100644 --- a/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php +++ b/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php @@ -32,6 +32,7 @@ public function clipOverlappingPolygonsBySite(string $uuid) 'entity_id' => $site->id, 'entity_type' => get_class($site), 'is_acknowledged' => false, + 'name' => 'Polygon Fix', ]); $job = new FixPolygonOverlapJob($delayedJob->id, $polygonUuids, $user->id); dispatch($job); @@ -89,6 +90,7 @@ public function clipOverlappingPolygonsOfProjectBySite(string $uuid) 'entity_type' => get_class($sitePolygon), 'created_by' => $user->id, 'is_acknowledged' => false, + 'name' => 'Polygon Fix', ]); $job = new FixPolygonOverlapJob($delayedJob->id, $uniquePolygonUuids, $user->id); dispatch($job); @@ -148,6 +150,7 @@ public function clipOverlappingPolygons(Request $request) 'entity_type' => get_class($entity), 'created_by' => $user->id, 'is_acknowledged' => false, + 'name' => 'Polygon Fix', ]); $job = new FixPolygonOverlapJob($delayedJob->id, $polygonUuids, $user->id); dispatch($job); diff --git a/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php b/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php index a2e41ca2..bddbd5c1 100755 --- a/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php +++ b/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php @@ -245,6 +245,7 @@ function ($attribute, $value, $fail) { 'entity_id' => $entity->id, 'entity_type' => get_class($entity), 'is_acknowledged' => false, + 'name' => 'Polygon Upload', ]); $job = new InsertGeojsonToDBJob( @@ -412,6 +413,7 @@ public function uploadShapefile(Request $request) 'entity_id' => $entity->id, 'entity_type' => get_class($entity), 'is_acknowledged' => false, + 'name' => 'Polygon Upload', ]); $job = new InsertGeojsonToDBJob( @@ -639,6 +641,7 @@ public function uploadGeoJSONFile(Request $request) 'entity_id' => $entity->id, 'entity_type' => get_class($entity), 'is_acknowledged' => false, + 'name' => 'Polygon Upload', ]); $job = new InsertGeojsonToDBJob( @@ -1251,6 +1254,7 @@ public function runSiteValidationPolygon(Request $request) 'entity_id' => $entity->id, 'entity_type' => get_class($entity), 'is_acknowledged' => false, + 'name' => 'Polygon validation', ]); $job = new RunSitePolygonsValidationJob($delayedJob->id, $sitePolygonsUuids); dispatch($job); @@ -1280,6 +1284,7 @@ public function runPolygonsValidation(Request $request) 'entity_id' => $entity->id, 'entity_type' => get_class($entity), 'is_acknowledged' => false, + 'name' => 'Polygon validation', ]); $job = new RunSitePolygonsValidationJob($delayedJob->id, $uuids); dispatch($job); diff --git a/app/Models/DelayedJob.php b/app/Models/DelayedJob.php index 497f4e0a..51e547df 100644 --- a/app/Models/DelayedJob.php +++ b/app/Models/DelayedJob.php @@ -18,7 +18,7 @@ class DelayedJob extends Model protected $table = 'delayed_jobs'; - protected $fillable = ['uuid', 'status', 'status_code', 'payload', 'entity_type', 'entity_id', 'created_by', 'is_acknowledged']; + protected $fillable = ['uuid', 'status', 'status_code', 'payload', 'entity_type', 'entity_id', 'created_by', 'is_acknowledged', 'name']; protected $casts = [ 'uuid' => 'string', diff --git a/database/migrations/2024_12_17_150134_add_name_to_jobs.php b/database/migrations/2024_12_17_150134_add_name_to_jobs.php new file mode 100644 index 00000000..edc2c5c1 --- /dev/null +++ b/database/migrations/2024_12_17_150134_add_name_to_jobs.php @@ -0,0 +1,27 @@ +string('name')->nullable()->after('id'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('delayed_jobs', function (Blueprint $table) { + $table->dropColumn('name'); + }); + } +}; From a5ee992656fb8ff24fcda31f62ab4b5da3d4ae91 Mon Sep 17 00:00:00 2001 From: Nathan Curtis Date: Tue, 17 Dec 2024 15:19:10 -0800 Subject: [PATCH 3/3] [TM-1579] A script to update v2_tree_species with a taxon_id when there is an exact name match. --- .../OneOff/AssociateExactMatchTrees.php | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 app/Console/Commands/OneOff/AssociateExactMatchTrees.php diff --git a/app/Console/Commands/OneOff/AssociateExactMatchTrees.php b/app/Console/Commands/OneOff/AssociateExactMatchTrees.php new file mode 100644 index 00000000..71e92d71 --- /dev/null +++ b/app/Console/Commands/OneOff/AssociateExactMatchTrees.php @@ -0,0 +1,43 @@ +join('tree_species_research', 'v2_tree_species.name', '=', 'tree_species_research.scientific_name') + ->where('v2_tree_species.taxon_id', null); + $this->withProgressBar((clone $query)->count(), function ($progressBar) use ($query) { + $query->chunkById(100, function ($trees) use ($progressBar) { + foreach ($trees as $tree) { + TreeSpecies::where('id', $tree->id)->update(['taxon_id' => $tree->taxon_id]); + $progressBar->advance(); + } + }); + }); + }); + } +}