diff --git a/app/Console/Commands/OneOff/AssociateExactMatchTrees.php b/app/Console/Commands/OneOff/AssociateExactMatchTrees.php new file mode 100644 index 000000000..71e92d71b --- /dev/null +++ b/app/Console/Commands/OneOff/AssociateExactMatchTrees.php @@ -0,0 +1,43 @@ +join('tree_species_research', 'v2_tree_species.name', '=', 'tree_species_research.scientific_name') + ->where('v2_tree_species.taxon_id', null); + $this->withProgressBar((clone $query)->count(), function ($progressBar) use ($query) { + $query->chunkById(100, function ($trees) use ($progressBar) { + foreach ($trees as $tree) { + TreeSpecies::where('id', $tree->id)->update(['taxon_id' => $tree->taxon_id]); + $progressBar->advance(); + } + }); + }); + }); + } +} diff --git a/app/Console/Commands/RecalculatePolygonAreas.php b/app/Console/Commands/RecalculatePolygonAreas.php new file mode 100644 index 000000000..508c3f879 --- /dev/null +++ b/app/Console/Commands/RecalculatePolygonAreas.php @@ -0,0 +1,94 @@ +option('only-active')) { + $query->active(); + } + + $sitePolygons = $query->cursor(); + + $processedCount = 0; + $errorCount = 0; + + $this->info('Starting polygon area recalculation...'); + $progressBar = $this->output->createProgressBar(); + $progressBar->start(); + + foreach ($sitePolygons as $sitePolygon) { + try { + $polygonGeometry = PolygonGeometry::where('uuid', $sitePolygon->poly_id) + ->select('uuid', DB::raw('ST_AsGeoJSON(geom) AS geojsonGeometry')) + ->first(); + if (! $polygonGeometry) { + $this->error("No geometry found for poly_id: {$sitePolygon->poly_id}"); + $errorCount++; + + continue; + } + $geometry = json_decode($polygonGeometry->geojsonGeometry, true); + + $calculatedArea = $areaService->getArea($geometry); + + $sitePolygon->calc_area = $calculatedArea; + $sitePolygon->save(); + + $processedCount++; + $progressBar->advance(); + } catch (\Exception $e) { + $this->error("Error processing polygon {$sitePolygon->id}: " . $e->getMessage()); + $errorCount++; + } + } + + DB::commit(); + + $progressBar->finish(); + $this->info("\n\nRecalculation complete!"); + $this->info("Processed: {$processedCount} polygons"); + $this->info("Errors: {$errorCount}"); + + } catch (\Exception $e) { + DB::rollBack(); + $this->error('Recalculation failed: ' . $e->getMessage()); + + return self::FAILURE; + } + + return self::SUCCESS; + } +} diff --git a/app/Console/Commands/UpdateValuesForIndicatorsCommand.php b/app/Console/Commands/UpdateValuesForIndicatorsCommand.php new file mode 100644 index 000000000..897ac3a37 --- /dev/null +++ b/app/Console/Commands/UpdateValuesForIndicatorsCommand.php @@ -0,0 +1,215 @@ + [ + 'sql' => 'SELECT umd_tree_cover_loss__year, SUM(area__ha) FROM results GROUP BY umd_tree_cover_loss__year', + 'query_url' => '/dataset/umd_tree_cover_loss/latest/query', + 'indicator' => 'umd_tree_cover_loss', + 'model' => IndicatorTreeCoverLoss::class, + 'table_name' => 'indicator_output_tree_cover_loss', + ], + 'treeCoverLossFires' => [ + 'sql' => 'SELECT umd_tree_cover_loss_from_fires__year, SUM(area__ha) FROM results GROUP BY umd_tree_cover_loss_from_fires__year', + 'query_url' => '/dataset/umd_tree_cover_loss_from_fires/latest/query', + 'indicator' => 'umd_tree_cover_loss_from_fires', + 'model' => IndicatorTreeCoverLoss::class, + 'table_name' => 'indicator_output_tree_cover_loss', + ], + 'restorationByEcoRegion' => [ + 'indicator' => 'wwf_terrestrial_ecoregions', + 'model' => IndicatorHectares::class, + 'table_name' => 'indicator_output_hectares', + ], + 'restorationByStrategy' => [ + 'indicator' => 'restoration_practice', + 'model' => IndicatorHectares::class, + 'table_name' => 'indicator_output_hectares', + ], + 'restorationByLandUse' => [ + 'indicator' => 'target_system', + 'model' => IndicatorHectares::class, + 'table_name' => 'indicator_output_hectares', + ], + ]; + + foreach ($slugMappings as $slug => $slugMapping) { + $uuids = []; + $processedCount = 0; + $errorCount = 0; + $this->info('Processing ' . $slug . '...'); + $progressBar = $this->output->createProgressBar(); + $progressBar->start(); + $data = $slugMapping['model']::with('sitePolygon') + ->where('indicator_slug', $slug) + ->select('id', 'site_polygon_id')->get(); + $uuids = $data->map(function ($item) { + return $item->sitePolygon ? $item->sitePolygon->poly_id : null; + })->filter()->toArray(); + + foreach ($uuids as $uuid) { + try { + $polygonGeometry = $this->getGeometry($uuid); + $registerExist = DB::table($slugMappings[$slug]['table_name'].' as i') + ->where('i.site_polygon_id', $polygonGeometry['site_polygon_id']) + ->where('i.indicator_slug', $slug) + ->where('i.year_of_analysis', Carbon::now()->year) + ->exists(); + + if (! $registerExist) { + continue; + } + + if (str_contains($slug, 'restorationBy')) { + $geojson = GeometryHelper::getPolygonGeojson($uuid); + $indicatorRestorationResponse = App::make(PythonService::class)->IndicatorPolygon($geojson, $slugMappings[$slug]['indicator'], getenv('GFW_SECRET_KEY')); + + if ($slug == 'restorationByEcoRegion') { + $value = json_encode($indicatorRestorationResponse['area'][$slugMappings[$slug]['indicator']]); + } else { + $value = $this->formatKeysValues($indicatorRestorationResponse['area'][$slugMappings[$slug]['indicator']]); + } + $data = [ + 'value' => $value, + ]; + $slugMappings[$slug]['model']::where('site_polygon_id', $polygonGeometry['site_polygon_id']) + ->where('indicator_slug', $slug) + ->where('year_of_analysis', Carbon::now()->year) + ->update($data); + + $processedCount++; + $progressBar->advance(); + + continue; + } + + $response = $this->sendApiRequestIndicator(getenv('GFW_SECRET_KEY'), $slugMappings[$slug]['query_url'], $slugMappings[$slug]['sql'], $polygonGeometry['geo']); + if (str_contains($slug, 'treeCoverLoss')) { + $processedTreeCoverLossValue = $this->processTreeCoverLossValue($response->json()['data'], $slugMappings[$slug]['indicator']); + } + + if ($response->successful()) { + if (str_contains($slug, 'treeCoverLoss')) { + $data = $this->generateTreeCoverLossData($processedTreeCoverLossValue); + } else { + $data = [ + 'value' => json_encode($response->json()['data']), + ]; + } + + $slugMappings[$slug]['model']::where('site_polygon_id', $polygonGeometry['site_polygon_id']) + ->where('indicator_slug', $slug) + ->where('year_of_analysis', Carbon::now()->year) + ->update($data); + $processedCount++; + $progressBar->advance(); + } else { + Log::error('A problem occurred during the analysis of the geometry for the polygon: ' . $uuid); + } + } catch (\Exception $e) { + Log::error('Error in the analysis: ' . $e->getMessage()); + $errorCount++; + } + } + $progressBar->finish(); + $this->info("\n\n{$slug} updated successfully."); + $this->info("Processed: {$processedCount} polygons"); + $this->info("Errors: {$errorCount}"); + } + + return 0; + } + + public function getGeometry($polygonUuid) + { + $geojson = GeometryHelper::getMonitoredPolygonsGeojson($polygonUuid); + $geoJsonObject = json_decode($geojson['geometry']->geojsonGeometry, true); + + return [ + 'geo' => [ + 'type' => 'Polygon', + 'coordinates' => $geoJsonObject['coordinates'], + ], + 'site_polygon_id' => $geojson['site_polygon_id'], + ]; + } + + public function sendApiRequestIndicator($secret_key, $query_url, $query_sql, $geometry) + { + return Http::withHeaders([ + 'content-type' => 'application/json', + 'x-api-key' => $secret_key, + ])->post('https://data-api.globalforestwatch.org' . $query_url, [ + 'sql' => $query_sql, + 'geometry' => $geometry, + ]); + } + + public function processTreeCoverLossValue($data, $indicator) + { + $processedTreeCoverLossValue = []; + foreach ($data as $i) { + $processedTreeCoverLossValue[$i[$indicator . '__year']] = $i['area__ha']; + } + + return $processedTreeCoverLossValue; + } + + public function generateTreeCoverLossData($processedTreeCoverLossValue) + { + $yearsOfAnalysis = [2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024]; + $responseData = []; + foreach ($yearsOfAnalysis as $year) { + if (isset($processedTreeCoverLossValue[$year])) { + $responseData[$year] = $processedTreeCoverLossValue[$year]; + } else { + $responseData[$year] = 0.0; + } + } + + return [ + 'value' => json_encode($responseData), + ]; + } + + public function formatKeysValues($data) + { + $formattedData = []; + foreach ($data as $key => $value) { + $formattedKey = strtolower(str_replace(' ', '-', $key)); + $formattedData[$formattedKey] = $value; + } + + return json_encode($formattedData); + } +} diff --git a/app/Helpers/PolygonGeometryHelper.php b/app/Helpers/PolygonGeometryHelper.php index 44c5085e3..7f6bc28d8 100644 --- a/app/Helpers/PolygonGeometryHelper.php +++ b/app/Helpers/PolygonGeometryHelper.php @@ -5,7 +5,7 @@ use App\Models\V2\Projects\Project; use App\Models\V2\Sites\Site; use App\Models\V2\Sites\SitePolygon; -use Illuminate\Support\Facades\DB; +use App\Services\AreaCalculationService; use Illuminate\Support\Facades\Log; class PolygonGeometryHelper @@ -16,12 +16,8 @@ public static function updateEstAreainSitePolygon($polygonGeometry, $geometry) $sitePolygon = SitePolygon::where('poly_id', $polygonGeometry->uuid)->first(); if ($sitePolygon) { - $geojson = json_encode($geometry); - $areaSqDegrees = DB::selectOne("SELECT ST_Area(ST_GeomFromGeoJSON('$geojson')) AS area")->area; - $latitude = DB::selectOne("SELECT ST_Y(ST_Centroid(ST_GeomFromGeoJSON('$geojson'))) AS latitude")->latitude; - $unitLatitude = 111320; - $areaSqMeters = $areaSqDegrees * pow($unitLatitude * cos(deg2rad($latitude)), 2); - $areaHectares = $areaSqMeters / 10000; + $areaCalculationService = app(AreaCalculationService::class); + $areaHectares = $areaCalculationService->getArea((array) $geometry->geometry); $sitePolygon->calc_area = $areaHectares; $sitePolygon->save(); diff --git a/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php b/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php index b9e699c8a..3408bf0c7 100644 --- a/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php +++ b/app/Http/Controllers/V2/Terrafund/TerrafundClipGeometryController.php @@ -29,8 +29,11 @@ public function clipOverlappingPolygonsBySite(string $uuid) $delayedJob = DelayedJobProgress::create([ 'processed_content' => 0, 'created_by' => $user->id, - 'entity_id' => $site->id, - 'entity_type' => get_class($site), + 'metadata' => [ + 'entity_id' => $site->id, + 'entity_type' => get_class($site), + 'entity_name' => $site->name, + ], 'is_acknowledged' => false, 'name' => 'Polygon Fix', ]); @@ -46,8 +49,8 @@ public function clipOverlappingPolygonsOfProjectBySite(string $uuid) ini_set('max_execution_time', self::MAX_EXECUTION_TIME); ini_set('memory_limit', '-1'); $user = Auth::user(); - $sitePolygon = Site::isUuid($uuid)->first(); - $projectId = $sitePolygon->project_id ?? null; + $site = Site::isUuid($uuid)->first(); + $projectId = $site->project_id ?? null; if (! $projectId) { return response()->json(['error' => 'Project not found for the given site UUID.'], 404); @@ -86,8 +89,11 @@ public function clipOverlappingPolygonsOfProjectBySite(string $uuid) $delayedJob = DelayedJobProgress::create([ 'processed_content' => 0, - 'entity_id' => $sitePolygon->id, - 'entity_type' => get_class($sitePolygon), + 'metadata' => [ + 'entity_id' => $site->id, + 'entity_type' => get_class($site), + 'entity_name' => $site->name, + ], 'created_by' => $user->id, 'is_acknowledged' => false, 'name' => 'Polygon Fix', @@ -146,8 +152,11 @@ public function clipOverlappingPolygons(Request $request) $user = Auth::user(); $delayedJob = DelayedJobProgress::create([ 'processed_content' => 0, - 'entity_id' => $entity->id, - 'entity_type' => get_class($entity), + 'metadata' => [ + 'entity_id' => $entity->id, + 'entity_type' => get_class($entity), + 'entity_name' => $entity->name, + ], 'created_by' => $user->id, 'is_acknowledged' => false, 'name' => 'Polygon Fix', diff --git a/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php b/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php index bddbd5c1e..a3b4bf6d8 100755 --- a/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php +++ b/app/Http/Controllers/V2/Terrafund/TerrafundCreateGeometryController.php @@ -242,8 +242,11 @@ function ($attribute, $value, $fail) { Redis::set($redis_key, $geojsonContent, 'EX', 7200); $delayedJob = DelayedJob::create([ 'created_by' => $user->id, - 'entity_id' => $entity->id, - 'entity_type' => get_class($entity), + 'metadata' => [ + 'entity_id' => $entity->id, + 'entity_type' => get_class($entity), + 'entity_name' => $entity->name, + ], 'is_acknowledged' => false, 'name' => 'Polygon Upload', ]); @@ -410,8 +413,11 @@ public function uploadShapefile(Request $request) Redis::set($redis_key, $geojsonContent, 'EX', 7200); $delayedJob = DelayedJob::create([ 'created_by' => $user->id, - 'entity_id' => $entity->id, - 'entity_type' => get_class($entity), + 'metadata' => [ + 'entity_id' => $entity->id, + 'entity_type' => get_class($entity), + 'entity_name' => $entity->name, + ], 'is_acknowledged' => false, 'name' => 'Polygon Upload', ]); @@ -638,8 +644,11 @@ public function uploadGeoJSONFile(Request $request) Redis::set($redis_key, $geojson_content, 'EX', 7200); $delayedJob = DelayedJob::create([ 'created_by' => $user->id, - 'entity_id' => $entity->id, - 'entity_type' => get_class($entity), + 'metadata' => [ + 'entity_id' => $entity->id, + 'entity_type' => get_class($entity), + 'entity_name' => $entity->name, + ], 'is_acknowledged' => false, 'name' => 'Polygon Upload', ]); @@ -1248,14 +1257,17 @@ public function runSiteValidationPolygon(Request $request) $entity = Site::where('uuid', $uuid)->firstOrFail(); $sitePolygonsUuids = GeometryHelper::getSitePolygonsUuids($uuid)->toArray(); $delayedJob = DelayedJobProgress::create([ - 'total_content' => count($sitePolygonsUuids), - 'processed_content' => 0, - 'created_by' => $user->id, + 'total_content' => count($sitePolygonsUuids), + 'processed_content' => 0, + 'created_by' => $user->id, + 'metadata' => [ 'entity_id' => $entity->id, 'entity_type' => get_class($entity), - 'is_acknowledged' => false, - 'name' => 'Polygon validation', - ]); + 'entity_name' => $entity->name, + ], + 'is_acknowledged' => false, + 'name' => 'Polygon Validation', + ]); $job = new RunSitePolygonsValidationJob($delayedJob->id, $sitePolygonsUuids); dispatch($job); @@ -1281,10 +1293,13 @@ public function runPolygonsValidation(Request $request) 'total_content' => count($uuids), 'processed_content' => 0, 'created_by' => $user->id, - 'entity_id' => $entity->id, - 'entity_type' => get_class($entity), + 'metadata' => [ + 'entity_id' => $entity->id, + 'entity_type' => get_class($entity), + 'entity_name' => $entity->name, + ], 'is_acknowledged' => false, - 'name' => 'Polygon validation', + 'name' => 'Polygon Validation', ]); $job = new RunSitePolygonsValidationJob($delayedJob->id, $uuids); dispatch($job); diff --git a/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php b/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php index 26ff7ac17..469efa40b 100644 --- a/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php +++ b/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php @@ -9,6 +9,7 @@ use App\Models\V2\Projects\ProjectPolygon; use App\Models\V2\Sites\SitePolygon; use App\Models\V2\User; +use App\Services\AreaCalculationService; use App\Services\PolygonService; use App\Services\SiteService; use Illuminate\Http\Request; @@ -329,10 +330,12 @@ public function createSitePolygon(string $uuid, string $siteUuid, Request $reque if (! $polygonGeometry) { return response()->json(['message' => 'No polygon geometry found for the given UUID.'], 404); } - $areaSqDegrees = DB::selectOne('SELECT ST_Area(geom) AS area FROM polygon_geometry WHERE uuid = :uuid', ['uuid' => $uuid])->area; - $latitude = DB::selectOne('SELECT ST_Y(ST_Centroid(geom)) AS latitude FROM polygon_geometry WHERE uuid = :uuid', ['uuid' => $uuid])->latitude; - $areaSqMeters = $areaSqDegrees * pow(111320 * cos(deg2rad($latitude)), 2); - $areaHectares = $areaSqMeters / 10000; + $polygonGeom = PolygonGeometry::where('uuid', $uuid) + ->select('uuid', DB::raw('ST_AsGeoJSON(geom) AS geojsonGeometry')) + ->first(); + $geometry = json_decode($polygonGeom->geojsonGeometry, true); + $areaCalculationService = app(AreaCalculationService::class); + $areaHectares = $areaCalculationService->getArea($geometry); $sitePolygon = new SitePolygon([ 'poly_name' => $validatedData['poly_name'], 'plantstart' => $validatedData['plantstart'], diff --git a/app/Http/Resources/V2/SiteReports/SiteReportResource.php b/app/Http/Resources/V2/SiteReports/SiteReportResource.php index aa28b7c53..9e865c21e 100644 --- a/app/Http/Resources/V2/SiteReports/SiteReportResource.php +++ b/app/Http/Resources/V2/SiteReports/SiteReportResource.php @@ -65,6 +65,7 @@ public function toArray($request) 'created_by' => $this->handleCreatedBy(), 'regeneration_description' => $this->regeneration_description, 'total_non_tree_species_planted_count' => $this->total_non_tree_species_planted_count, + 'total_tree_replanting_count' => $this->total_tree_replanting_count, 'pct_survival_to_date' => $this->pct_survival_to_date, 'survival_calculation' => $this->survival_calculation, diff --git a/app/Jobs/FixPolygonOverlapJob.php b/app/Jobs/FixPolygonOverlapJob.php index 16850a39b..ab1e2f238 100644 --- a/app/Jobs/FixPolygonOverlapJob.php +++ b/app/Jobs/FixPolygonOverlapJob.php @@ -6,6 +6,7 @@ use App\Mail\PolygonOperationsComplete; use App\Models\DelayedJob; use App\Models\DelayedJobProgress; +use App\Models\V2\Sites\Site; use App\Services\PolygonService; use Exception; use Illuminate\Bus\Queueable; @@ -68,7 +69,9 @@ public function handle(): void try { $delayedJob = DelayedJobProgress::findOrFail($this->delayed_job_id); $user = Auth::user(); - $site = $delayedJob->entity; + $metadata = $delayedJob->metadata; + $entityId = $metadata['entity_id'] ?? null; + $site = Site::findOrFail($entityId); $userForMail = $delayedJob->creator; if ($user) { $polygonsClipped = App::make(PolygonService::class)->processClippedPolygons($this->polygonUuids, $this->delayed_job_id); diff --git a/app/Jobs/InsertGeojsonToDBJob.php b/app/Jobs/InsertGeojsonToDBJob.php index a6caf7582..8809163e8 100755 --- a/app/Jobs/InsertGeojsonToDBJob.php +++ b/app/Jobs/InsertGeojsonToDBJob.php @@ -4,6 +4,7 @@ use App\Mail\PolygonOperationsComplete; use App\Models\DelayedJob; +use App\Models\V2\Sites\Site; use App\Services\PolygonService; use App\Services\SiteService; use Exception; @@ -54,7 +55,10 @@ public function handle(PolygonService $service) { $delayedJob = DelayedJob::findOrFail($this->delayed_job_id); $user = $delayedJob->creator; - $site = $delayedJob->entity; + $metadata = $delayedJob->metadata; + $entityId = $metadata['entity_id'] ?? null; + + $site = Site::findOrFail($entityId); try { $geojsonContent = Redis::get($this->redis_key); diff --git a/app/Jobs/RunSitePolygonsValidationJob.php b/app/Jobs/RunSitePolygonsValidationJob.php index 34f4f5a98..bcd1b43a2 100644 --- a/app/Jobs/RunSitePolygonsValidationJob.php +++ b/app/Jobs/RunSitePolygonsValidationJob.php @@ -5,6 +5,7 @@ use App\Mail\PolygonOperationsComplete; use App\Models\DelayedJob; use App\Models\DelayedJobProgress; +use App\Models\V2\Sites\Site; use App\Services\PolygonValidationService; use Exception; use Illuminate\Bus\Queueable; @@ -53,7 +54,20 @@ public function handle(PolygonValidationService $validationService) try { $delayedJob = DelayedJobProgress::findOrFail($this->delayed_job_id); $user = $delayedJob->creator; - $site = $delayedJob->entity; + $metadata = $delayedJob->metadata; + + $entityId = $metadata['entity_id'] ?? null; + + if ($entityId) { + $site = Site::findOrFail($entityId); + } else { + Log::error('entityId is null, unable to find site'); + } + + if (! $site) { + throw new Exception('Site not found for the given site UUID.'); + } + foreach ($this->sitePolygonsUuids as $polygonUuid) { $request = new Request(['uuid' => $polygonUuid]); $validationService->validateOverlapping($request); @@ -78,6 +92,8 @@ public function handle(PolygonValidationService $validationService) 'progress' => 100, ]); + Log::info('site available? ' . $site); + Mail::to($user->email_address) ->send(new PolygonOperationsComplete( $site, @@ -95,5 +111,6 @@ public function handle(PolygonValidationService $validationService) 'status_code' => Response::HTTP_INTERNAL_SERVER_ERROR, ]); } + } } diff --git a/app/Mail/PolygonOperationsComplete.php b/app/Mail/PolygonOperationsComplete.php index 3d7b74726..b08ec4abf 100644 --- a/app/Mail/PolygonOperationsComplete.php +++ b/app/Mail/PolygonOperationsComplete.php @@ -29,7 +29,12 @@ public function __construct($site, $operation, $user, $completedAt) ]) ->setCta('polygon-validation.cta'); - $this->link = '/sites/' . $site->uuid; + if ($user->hasRole('project-developer')) { + $this->link = '/site/' . $site->uuid; + } else { + $this->link = '/admin#/site/' . $site->uuid . '/show'; + } + $this->transactional = true; } } diff --git a/app/Models/DelayedJob.php b/app/Models/DelayedJob.php index 51e547df7..25b599e34 100644 --- a/app/Models/DelayedJob.php +++ b/app/Models/DelayedJob.php @@ -18,17 +18,13 @@ class DelayedJob extends Model protected $table = 'delayed_jobs'; - protected $fillable = ['uuid', 'status', 'status_code', 'payload', 'entity_type', 'entity_id', 'created_by', 'is_acknowledged', 'name']; + protected $fillable = ['uuid', 'status', 'status_code', 'payload', 'metadata', 'created_by', 'is_acknowledged', 'name']; protected $casts = [ 'uuid' => 'string', + 'metadata' => 'json', ]; - public function entity() - { - return $this->morphTo(); - } - public function creator() { return $this->belongsTo(User::class, 'created_by'); diff --git a/app/Models/Traits/UsesLinkedFields.php b/app/Models/Traits/UsesLinkedFields.php index 68be03d2a..818fcf3ac 100644 --- a/app/Models/Traits/UsesLinkedFields.php +++ b/app/Models/Traits/UsesLinkedFields.php @@ -293,8 +293,10 @@ private function syncRelation(string $property, string $inputType, $data, bool $ if ($model != null) { $model->update($entry); } else { - // protection against updating a deleted entry - unset($entry['uuid']); + // protection against clashing with a deleted entry + if (! empty($entry['uuid']) && $entity->$property()->onlyTrashed()->where('uuid', $entry['uuid'])->exists()) { + unset($entry['uuid']); + } $entity->$property()->create($entry); } } diff --git a/app/Models/V2/Sites/SiteReport.php b/app/Models/V2/Sites/SiteReport.php index bfa5eb831..97f0a78ce 100644 --- a/app/Models/V2/Sites/SiteReport.php +++ b/app/Models/V2/Sites/SiteReport.php @@ -307,6 +307,11 @@ public function getTotalNonTreeSpeciesPlantedCountAttribute(): int return $this->nonTreeSpecies()->visible()->sum('amount'); } + public function getTotalTreeReplantingCountAttribute(): int + { + return $this->replantingTreeSpecies()->visible()->sum('amount'); + } + public function getTotalSeedsPlantedCountAttribute(): int { return $this->seedings()->visible()->sum('amount'); diff --git a/app/Models/V2/TreeSpecies/TreeSpecies.php b/app/Models/V2/TreeSpecies/TreeSpecies.php index ff4dc0729..56b2827f5 100644 --- a/app/Models/V2/TreeSpecies/TreeSpecies.php +++ b/app/Models/V2/TreeSpecies/TreeSpecies.php @@ -32,6 +32,7 @@ class TreeSpecies extends Model implements EntityRelationModel public $table = 'v2_tree_species'; protected $fillable = [ + 'uuid', 'name', 'amount', 'speciesable_type', @@ -44,6 +45,7 @@ class TreeSpecies extends Model implements EntityRelationModel public const COLLECTION_DIRECT_SEEDING = 'direct-seeding'; public const COLLECTION_PLANTED = 'tree-planted'; public const COLLECTION_NON_TREE = 'non-tree'; + public const COLLECTION_REPLANTING = 'replanting'; public const COLLECTION_NURSERY = 'nursery-seedling'; public const COLLECTION_HISTORICAL = 'historical-tree-species'; @@ -51,6 +53,7 @@ class TreeSpecies extends Model implements EntityRelationModel self::COLLECTION_DIRECT_SEEDING => 'Direct Seeding', self::COLLECTION_PLANTED => 'Planted', self::COLLECTION_NON_TREE => 'Non Tree', + self::COLLECTION_REPLANTING => 'Replanting', self::COLLECTION_NURSERY => 'Nursery Seedling', self::COLLECTION_HISTORICAL => 'Historical Tree Species', ]; diff --git a/app/Services/AreaCalculationService.php b/app/Services/AreaCalculationService.php new file mode 100644 index 000000000..6b27c5392 --- /dev/null +++ b/app/Services/AreaCalculationService.php @@ -0,0 +1,85 @@ + 'Feature', + 'geometry' => $geometry, + 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']], + ]); + + $inputGeojson = tempnam(sys_get_temp_dir(), 'input_') . '.geojson'; + $outputGeojson = tempnam(sys_get_temp_dir(), 'output_') . '.geojson'; + + try { + file_put_contents($inputGeojson, $geojson); + + $process = new Process([ + 'python3', + base_path() . '/resources/python/polygon-area/app.py', + $inputGeojson, + $outputGeojson, + ]); + + $process->run(); + + if (! $process->isSuccessful()) { + Log::error('Area calculation failed: ' . $process->getErrorOutput()); + + throw new \RuntimeException('Area calculation failed: ' . $process->getErrorOutput()); + } + + $result = json_decode(file_get_contents($outputGeojson), true); + + return $result['area_hectares']; + + } catch (\Exception $e) { + Log::error('Error calculating area: ' . $e->getMessage()); + + throw $e; + } finally { + @unlink($inputGeojson); + @unlink($outputGeojson); + } + } + + public function getGeomAndArea(array $geometry): array + { + $geojson = json_encode([ + 'type' => 'Feature', + 'geometry' => $geometry, + 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']], + ]); + + $geom = DB::raw("ST_GeomFromGeoJSON('$geojson')"); + $areaHectares = $this->calculateArea($geometry); + + return ['geom' => $geom, 'area' => $areaHectares]; + } + + public function getArea(array $geometry): float + { + if ($geometry['type'] === 'MultiPolygon') { + $totalArea = 0; + foreach ($geometry['coordinates'] as $polygon) { + $polygonGeometry = [ + 'type' => 'Polygon', + 'coordinates' => $polygon, + ]; + $totalArea += $this->calculateArea($polygonGeometry); + } + + return $totalArea; + } + + return $this->calculateArea($geometry); + } +} diff --git a/app/Services/PolygonService.php b/app/Services/PolygonService.php index 5213b9af9..67b04218e 100755 --- a/app/Services/PolygonService.php +++ b/app/Services/PolygonService.php @@ -284,20 +284,9 @@ protected function getGeom(array $geometry) protected function getGeomAndArea(array $geometry): array { - // Convert geometry to GeoJSON string - $geojson = json_encode(['type' => 'Feature', 'geometry' => $geometry, 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']]]); - - // Get GeoJSON data in the database - $geom = DB::raw("ST_GeomFromGeoJSON('$geojson')"); - $areaSqDegrees = DB::selectOne("SELECT ST_Area(ST_GeomFromGeoJSON('$geojson')) AS area")->area; - $latitude = DB::selectOne("SELECT ST_Y(ST_Centroid(ST_GeomFromGeoJSON('$geojson'))) AS latitude")->latitude; - // 111320 is the length of one degree of latitude in meters at the equator - $unitLatitude = 111320; - $areaSqMeters = $areaSqDegrees * pow($unitLatitude * cos(deg2rad($latitude)), 2); + $areaCalculationService = app(AreaCalculationService::class); - $areaHectares = $areaSqMeters / 10000; - - return ['geom' => $geom, 'area' => $areaHectares]; + return $areaCalculationService->getGeomAndArea($geometry); } protected function insertSinglePolygon(array $geometry): array @@ -552,13 +541,20 @@ public function insertGeojsonToDBFromContent(string $geojsonData, ?string $entit } catch (Exception $e) { $errorMessage = $e->getMessage(); - $decodedErrorMessage = json_decode($errorMessage, true); + $decodedError = json_decode($errorMessage, true); + if (json_last_error() === JSON_ERROR_NONE) { - return ['error' => $decodedErrorMessage]; + Log::error('Validation error', ['error' => $decodedError]); + + return [ + 'error' => json_encode($decodedError), + ]; } else { - Log::info('Error inserting geojson to DB', ['error' => $errorMessage]); + Log::error('Validation error', ['error' => $errorMessage]); - return ['error' => $errorMessage]; + return [ + 'error' => $errorMessage, + ]; } } } diff --git a/app/Services/RunIndicatorAnalysisService.php b/app/Services/RunIndicatorAnalysisService.php index 6822441b7..19608be1f 100644 --- a/app/Services/RunIndicatorAnalysisService.php +++ b/app/Services/RunIndicatorAnalysisService.php @@ -10,6 +10,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; +use Symfony\Component\Process\Process; class RunIndicatorAnalysisService { @@ -133,13 +134,61 @@ public function getGeometry($polygonUuid) public function sendApiRequestIndicator($secret_key, $query_url, $query_sql, $geometry) { - return Http::withHeaders([ + $response = Http::withHeaders([ 'content-type' => 'application/json', 'x-api-key' => $secret_key, ])->post('https://data-api.globalforestwatch.org' . $query_url, [ 'sql' => $query_sql, 'geometry' => $geometry, ]); + + if ($response->successful()) { + $gfwDataFile = tempnam(sys_get_temp_dir(), 'gfw_') . '.json'; + $geometryFile = tempnam(sys_get_temp_dir(), 'geom_') . '.json'; + $outputFile = tempnam(sys_get_temp_dir(), 'output_') . '.json'; + + try { + file_put_contents($gfwDataFile, json_encode($response->json())); + file_put_contents($geometryFile, json_encode($geometry)); + + $process = new Process([ + 'python3', + base_path() . '/resources/python/gfw-area-adjustment/app.py', + $gfwDataFile, + $geometryFile, + $outputFile, + ]); + + $process->run(); + + if (! $process->isSuccessful()) { + Log::error('Area adjustment failed: ' . $process->getErrorOutput()); + + return $response; + } + + $adjustedData = json_decode(file_get_contents($outputFile), true); + + return new \Illuminate\Http\Client\Response( + new \GuzzleHttp\Psr7\Response( + 200, + ['Content-Type' => 'application/json'], + json_encode($adjustedData) + ) + ); + + } catch (\Exception $e) { + Log::error('Error adjusting areas: ' . $e->getMessage()); + + return $response; + } finally { + @unlink($gfwDataFile); + @unlink($geometryFile); + @unlink($outputFile); + } + } + + return $response; } public function processTreeCoverLossValue($data, $indicator) diff --git a/database/migrations/2024_12_20_153701_add_metadata_and_remove_entity_columns.php b/database/migrations/2024_12_20_153701_add_metadata_and_remove_entity_columns.php new file mode 100644 index 000000000..22ab00098 --- /dev/null +++ b/database/migrations/2024_12_20_153701_add_metadata_and_remove_entity_columns.php @@ -0,0 +1,31 @@ +json('metadata')->nullable()->after('payload')->comment('Stores additional information for the delayed job.'); + + $table->dropColumn(['entity_id', 'entity_type']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('delayed_jobs', function (Blueprint $table) { + $table->dropColumn('metadata'); + $table->unsignedBigInteger('entity_id')->nullable()->after('name'); + $table->string('entity_type')->nullable()->after('entityId'); + }); + } +}; diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 56eaa8143..46ab3fab7 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -1,7 +1,11 @@ ## PHP FROM php:8.2-apache AS php -RUN apt-get update -RUN apt-get install -y \ + +# Set GDAL version +ENV GDAL_VERSION=3.4.3 + +# Install basic dependencies +RUN apt-get update && apt-get install -y \ libxml2-dev \ libonig-dev \ libpng-dev \ @@ -10,10 +14,36 @@ RUN apt-get install -y \ libmagickwand-dev \ mariadb-client \ libzip-dev \ - gdal-bin \ - libgdal-dev \ python3.11-venv \ - exiftool + python3.11-dev \ + exiftool \ + build-essential \ + wget \ + cmake \ + sqlite3 \ + libsqlite3-dev \ + libspatialite-dev \ + libpq-dev \ + libcurl4-gnutls-dev \ + libproj-dev \ + libgeos-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install GDAL 3.4.3 from source +RUN wget https://github.com/OSGeo/gdal/releases/download/v${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} \ + && ./configure \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + && cd .. \ + && rm -rf gdal-${GDAL_VERSION} gdal-${GDAL_VERSION}.tar.gz + +# Set GDAL environment variables +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal +ENV C_INCLUDE_PATH=/usr/include/gdal +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH RUN docker-php-ext-configure gd --with-freetype --with-jpeg RUN docker-php-ext-install \ @@ -40,12 +70,16 @@ RUN a2enmod rewrite COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf COPY docker/php.ini /usr/local/etc/php/php.ini -## Python -RUN python3 -m venv /opt/python -COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt +# Python virtual environment setup +RUN python3.11 -m venv /opt/python ENV PATH="/opt/python/bin:${PATH}" + +# Install Python dependencies in the correct order +COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt +RUN pip3 install --upgrade pip wheel setuptools RUN pip3 install -r /root/voronoi-requirements.txt + RUN chmod -R a+rx /opt/python USER www-data ENV PATH="/opt/python/bin:${PATH}" -USER root +USER root \ No newline at end of file diff --git a/resources/python/gfw-area-adjustment/app.py b/resources/python/gfw-area-adjustment/app.py new file mode 100644 index 000000000..b586a60d5 --- /dev/null +++ b/resources/python/gfw-area-adjustment/app.py @@ -0,0 +1,101 @@ +import sys +import json +import geopandas as gpd +from shapely.geometry import shape, Polygon +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def calculate_correction_factor(reference_geometry): + """ + Calculate the correction factor by comparing WGS84 and ESRI:54009 areas + All areas are converted to hectares (1 hectare = 10000 square meters) + """ + try: + gdf_wgs84 = gpd.GeoDataFrame(geometry=[reference_geometry], crs="EPSG:4326") + + gdf_projected = gdf_wgs84.to_crs('ESRI:54009') + area_projected_ha = gdf_projected.area[0] / 10000 + + if area_projected_ha < 0.0001: + logger.warning("Area is very small, using correction factor of 1") + return 1.0 + + area_geodesic_ha = gdf_wgs84.geometry.to_crs('+proj=cea').area[0] / 10000 + + correction_factor = area_projected_ha / area_geodesic_ha if area_geodesic_ha > 0 else 1 + + if correction_factor > 10 or correction_factor < 0.1: + logger.warning(f"Extreme correction factor detected: {correction_factor}. Using 1.0 instead.") + return 1.0 + + return correction_factor + + except Exception as e: + logger.error(f"Error calculating correction factor: {str(e)}") + raise + +def adjust_gfw_data(gfw_data, reference_geometry): + """ + Adjust GFW area values using the correction factor from reference geometry + """ + try: + if isinstance(reference_geometry, str): + reference_geometry = json.loads(reference_geometry) + + if 'type' in reference_geometry and reference_geometry['type'] == 'Feature': + geometry = shape(reference_geometry['geometry']) + elif 'type' in reference_geometry and reference_geometry['type'] == 'FeatureCollection': + geometry = shape(reference_geometry['features'][0]['geometry']) + else: + geometry = shape(reference_geometry) + + correction_factor = calculate_correction_factor(geometry) + + if isinstance(gfw_data, str): + gfw_data = json.loads(gfw_data) + + adjusted_data = { + "data": [], + "status": gfw_data.get("status", "success") + } + + for entry in gfw_data.get("data", []): + adjusted_entry = entry.copy() + if entry["area__ha"] > 0.0001: + adjusted_entry["area__ha"] = round(entry["area__ha"] * correction_factor, 5) + adjusted_data["data"].append(adjusted_entry) + + return adjusted_data + + except Exception as e: + logger.error(f"Error adjusting GFW data: {str(e)}") + raise + +def main(): + try: + if len(sys.argv) != 4: + raise ValueError("Script requires GFW data, reference geometry, and output file paths as arguments") + + gfw_data_file = sys.argv[1] + reference_geometry_file = sys.argv[2] + output_file = sys.argv[3] + + with open(gfw_data_file, 'r') as f: + gfw_data = json.load(f) + + with open(reference_geometry_file, 'r') as f: + reference_geometry = json.load(f) + + result = adjust_gfw_data(gfw_data, reference_geometry) + + with open(output_file, 'w') as f: + json.dump(result, f) + + except Exception as e: + logger.error(f"Error processing data: {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/resources/python/polygon-area/app.py b/resources/python/polygon-area/app.py new file mode 100644 index 000000000..4af6e65b8 --- /dev/null +++ b/resources/python/polygon-area/app.py @@ -0,0 +1,60 @@ +import sys +import json +import geopandas as gpd +from shapely.geometry import shape +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def calculate_area(geometry): + """ + Calculate area in hectares for a given geometry + """ + try: + gdf = gpd.GeoDataFrame(geometry=[geometry], crs="EPSG:4326") + gdf_projected = gdf.to_crs('ESRI:54009') + + area_hectares = gdf_projected.geometry.area[0] / 10000 + + return area_hectares + except Exception as e: + logger.error(f"Error calculating area: {str(e)}") + raise + +def main(): + try: + if len(sys.argv) != 3: + raise ValueError("Script requires input and output file paths as arguments") + + input_file = sys.argv[1] + output_file = sys.argv[2] + + with open(input_file, 'r') as f: + geojson_data = json.load(f) + + if 'type' in geojson_data and geojson_data['type'] == 'Feature': + geometry = shape(geojson_data['geometry']) + elif 'type' in geojson_data and geojson_data['type'] == 'FeatureCollection': + geometry = shape(geojson_data['features'][0]['geometry']) + else: + geometry = shape(geojson_data) + + area = calculate_area(geometry) + + result = { + 'area_hectares': area, + 'original_geometry': geojson_data + } + + with open(output_file, 'w') as f: + json.dump(result, f) + + print(area) + + except Exception as e: + logger.error(f"Error processing geometry: {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/resources/python/polygon-indicator/tree_cover_indicator.py b/resources/python/polygon-indicator/tree_cover_indicator.py index 80090da0f..7ec3b08f6 100644 --- a/resources/python/polygon-indicator/tree_cover_indicator.py +++ b/resources/python/polygon-indicator/tree_cover_indicator.py @@ -25,11 +25,9 @@ def get_gfw_data(geometry, session, dataset, params): def calculate_area(feature): geometry = shape(feature["geometry"]) gdf = gpd.GeoDataFrame(geometry=[geometry], crs="EPSG:4326") - gdf = gdf.to_crs("EPSG:3857") - area_m2 = gdf.geometry.area.values[ - 0 - ] # Directly get the area in square meters as a float - area_ha = area_m2 / 10**4 # Convert to hectares + gdf_projected = gdf.to_crs('ESRI:54009') + + area_ha = gdf_projected.geometry.area[0] / 10000 return area_ha diff --git a/resources/python/polygon-voronoi/requirements.txt b/resources/python/polygon-voronoi/requirements.txt index 0d1ba9bb3..5885dc848 100755 --- a/resources/python/polygon-voronoi/requirements.txt +++ b/resources/python/polygon-voronoi/requirements.txt @@ -1,3 +1,12 @@ pyproj==3.4.1 numpy==1.26.4 shapely==2.0.1 +fiona==1.10.1 +pandas==2.1.3 +geopandas==1.0.1 +rasterio==1.4.1 +exactextract==0.2.0 +rasterstats==0.20.0 +pyyaml==6.0.2 +requests==2.32.3 +boto3==1.35.43 \ No newline at end of file