From afdc0b85c3d435a151239c454d241c0847279104 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Wed, 18 Dec 2024 15:16:17 -0400 Subject: [PATCH 01/14] [TM-1467] Refactor area calculation methods --- app/Helpers/PolygonGeometryHelper.php | 10 +-- .../TerrafundEditGeometryController.php | 11 ++- app/Services/AreaCalculationService.php | 85 +++++++++++++++++++ app/Services/PolygonService.php | 15 +--- resources/python/polygon-area/app.py | 60 +++++++++++++ .../polygon-indicator/tree_cover_indicator.py | 8 +- 6 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 app/Services/AreaCalculationService.php create mode 100644 resources/python/polygon-area/app.py diff --git a/app/Helpers/PolygonGeometryHelper.php b/app/Helpers/PolygonGeometryHelper.php index 44c5085e3..7f6bc28d8 100644 --- a/app/Helpers/PolygonGeometryHelper.php +++ b/app/Helpers/PolygonGeometryHelper.php @@ -5,7 +5,7 @@ use App\Models\V2\Projects\Project; use App\Models\V2\Sites\Site; use App\Models\V2\Sites\SitePolygon; -use Illuminate\Support\Facades\DB; +use App\Services\AreaCalculationService; use Illuminate\Support\Facades\Log; class PolygonGeometryHelper @@ -16,12 +16,8 @@ public static function updateEstAreainSitePolygon($polygonGeometry, $geometry) $sitePolygon = SitePolygon::where('poly_id', $polygonGeometry->uuid)->first(); if ($sitePolygon) { - $geojson = json_encode($geometry); - $areaSqDegrees = DB::selectOne("SELECT ST_Area(ST_GeomFromGeoJSON('$geojson')) AS area")->area; - $latitude = DB::selectOne("SELECT ST_Y(ST_Centroid(ST_GeomFromGeoJSON('$geojson'))) AS latitude")->latitude; - $unitLatitude = 111320; - $areaSqMeters = $areaSqDegrees * pow($unitLatitude * cos(deg2rad($latitude)), 2); - $areaHectares = $areaSqMeters / 10000; + $areaCalculationService = app(AreaCalculationService::class); + $areaHectares = $areaCalculationService->getArea((array) $geometry->geometry); $sitePolygon->calc_area = $areaHectares; $sitePolygon->save(); diff --git a/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php b/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php index 26ff7ac17..469efa40b 100644 --- a/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php +++ b/app/Http/Controllers/V2/Terrafund/TerrafundEditGeometryController.php @@ -9,6 +9,7 @@ use App\Models\V2\Projects\ProjectPolygon; use App\Models\V2\Sites\SitePolygon; use App\Models\V2\User; +use App\Services\AreaCalculationService; use App\Services\PolygonService; use App\Services\SiteService; use Illuminate\Http\Request; @@ -329,10 +330,12 @@ public function createSitePolygon(string $uuid, string $siteUuid, Request $reque if (! $polygonGeometry) { return response()->json(['message' => 'No polygon geometry found for the given UUID.'], 404); } - $areaSqDegrees = DB::selectOne('SELECT ST_Area(geom) AS area FROM polygon_geometry WHERE uuid = :uuid', ['uuid' => $uuid])->area; - $latitude = DB::selectOne('SELECT ST_Y(ST_Centroid(geom)) AS latitude FROM polygon_geometry WHERE uuid = :uuid', ['uuid' => $uuid])->latitude; - $areaSqMeters = $areaSqDegrees * pow(111320 * cos(deg2rad($latitude)), 2); - $areaHectares = $areaSqMeters / 10000; + $polygonGeom = PolygonGeometry::where('uuid', $uuid) + ->select('uuid', DB::raw('ST_AsGeoJSON(geom) AS geojsonGeometry')) + ->first(); + $geometry = json_decode($polygonGeom->geojsonGeometry, true); + $areaCalculationService = app(AreaCalculationService::class); + $areaHectares = $areaCalculationService->getArea($geometry); $sitePolygon = new SitePolygon([ 'poly_name' => $validatedData['poly_name'], 'plantstart' => $validatedData['plantstart'], diff --git a/app/Services/AreaCalculationService.php b/app/Services/AreaCalculationService.php new file mode 100644 index 000000000..6b27c5392 --- /dev/null +++ b/app/Services/AreaCalculationService.php @@ -0,0 +1,85 @@ + 'Feature', + 'geometry' => $geometry, + 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']], + ]); + + $inputGeojson = tempnam(sys_get_temp_dir(), 'input_') . '.geojson'; + $outputGeojson = tempnam(sys_get_temp_dir(), 'output_') . '.geojson'; + + try { + file_put_contents($inputGeojson, $geojson); + + $process = new Process([ + 'python3', + base_path() . '/resources/python/polygon-area/app.py', + $inputGeojson, + $outputGeojson, + ]); + + $process->run(); + + if (! $process->isSuccessful()) { + Log::error('Area calculation failed: ' . $process->getErrorOutput()); + + throw new \RuntimeException('Area calculation failed: ' . $process->getErrorOutput()); + } + + $result = json_decode(file_get_contents($outputGeojson), true); + + return $result['area_hectares']; + + } catch (\Exception $e) { + Log::error('Error calculating area: ' . $e->getMessage()); + + throw $e; + } finally { + @unlink($inputGeojson); + @unlink($outputGeojson); + } + } + + public function getGeomAndArea(array $geometry): array + { + $geojson = json_encode([ + 'type' => 'Feature', + 'geometry' => $geometry, + 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']], + ]); + + $geom = DB::raw("ST_GeomFromGeoJSON('$geojson')"); + $areaHectares = $this->calculateArea($geometry); + + return ['geom' => $geom, 'area' => $areaHectares]; + } + + public function getArea(array $geometry): float + { + if ($geometry['type'] === 'MultiPolygon') { + $totalArea = 0; + foreach ($geometry['coordinates'] as $polygon) { + $polygonGeometry = [ + 'type' => 'Polygon', + 'coordinates' => $polygon, + ]; + $totalArea += $this->calculateArea($polygonGeometry); + } + + return $totalArea; + } + + return $this->calculateArea($geometry); + } +} diff --git a/app/Services/PolygonService.php b/app/Services/PolygonService.php index 5213b9af9..d59339581 100755 --- a/app/Services/PolygonService.php +++ b/app/Services/PolygonService.php @@ -284,20 +284,9 @@ protected function getGeom(array $geometry) protected function getGeomAndArea(array $geometry): array { - // Convert geometry to GeoJSON string - $geojson = json_encode(['type' => 'Feature', 'geometry' => $geometry, 'crs' => ['type' => 'name', 'properties' => ['name' => 'EPSG:4326']]]); - - // Get GeoJSON data in the database - $geom = DB::raw("ST_GeomFromGeoJSON('$geojson')"); - $areaSqDegrees = DB::selectOne("SELECT ST_Area(ST_GeomFromGeoJSON('$geojson')) AS area")->area; - $latitude = DB::selectOne("SELECT ST_Y(ST_Centroid(ST_GeomFromGeoJSON('$geojson'))) AS latitude")->latitude; - // 111320 is the length of one degree of latitude in meters at the equator - $unitLatitude = 111320; - $areaSqMeters = $areaSqDegrees * pow($unitLatitude * cos(deg2rad($latitude)), 2); - - $areaHectares = $areaSqMeters / 10000; + $areaCalculationService = app(AreaCalculationService::class); - return ['geom' => $geom, 'area' => $areaHectares]; + return $areaCalculationService->getGeomAndArea($geometry); } protected function insertSinglePolygon(array $geometry): array diff --git a/resources/python/polygon-area/app.py b/resources/python/polygon-area/app.py new file mode 100644 index 000000000..4af6e65b8 --- /dev/null +++ b/resources/python/polygon-area/app.py @@ -0,0 +1,60 @@ +import sys +import json +import geopandas as gpd +from shapely.geometry import shape +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def calculate_area(geometry): + """ + Calculate area in hectares for a given geometry + """ + try: + gdf = gpd.GeoDataFrame(geometry=[geometry], crs="EPSG:4326") + gdf_projected = gdf.to_crs('ESRI:54009') + + area_hectares = gdf_projected.geometry.area[0] / 10000 + + return area_hectares + except Exception as e: + logger.error(f"Error calculating area: {str(e)}") + raise + +def main(): + try: + if len(sys.argv) != 3: + raise ValueError("Script requires input and output file paths as arguments") + + input_file = sys.argv[1] + output_file = sys.argv[2] + + with open(input_file, 'r') as f: + geojson_data = json.load(f) + + if 'type' in geojson_data and geojson_data['type'] == 'Feature': + geometry = shape(geojson_data['geometry']) + elif 'type' in geojson_data and geojson_data['type'] == 'FeatureCollection': + geometry = shape(geojson_data['features'][0]['geometry']) + else: + geometry = shape(geojson_data) + + area = calculate_area(geometry) + + result = { + 'area_hectares': area, + 'original_geometry': geojson_data + } + + with open(output_file, 'w') as f: + json.dump(result, f) + + print(area) + + except Exception as e: + logger.error(f"Error processing geometry: {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/resources/python/polygon-indicator/tree_cover_indicator.py b/resources/python/polygon-indicator/tree_cover_indicator.py index 80090da0f..7ec3b08f6 100644 --- a/resources/python/polygon-indicator/tree_cover_indicator.py +++ b/resources/python/polygon-indicator/tree_cover_indicator.py @@ -25,11 +25,9 @@ def get_gfw_data(geometry, session, dataset, params): def calculate_area(feature): geometry = shape(feature["geometry"]) gdf = gpd.GeoDataFrame(geometry=[geometry], crs="EPSG:4326") - gdf = gdf.to_crs("EPSG:3857") - area_m2 = gdf.geometry.area.values[ - 0 - ] # Directly get the area in square meters as a float - area_ha = area_m2 / 10**4 # Convert to hectares + gdf_projected = gdf.to_crs('ESRI:54009') + + area_ha = gdf_projected.geometry.area[0] / 10000 return area_ha From 708fd95cf334b9fd302f36a38c985b5393d82b15 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Wed, 18 Dec 2024 16:22:45 -0400 Subject: [PATCH 02/14] [TM-1467] update area for all polygons --- .../Commands/RecalculatePolygonAreas.php | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 app/Console/Commands/RecalculatePolygonAreas.php diff --git a/app/Console/Commands/RecalculatePolygonAreas.php b/app/Console/Commands/RecalculatePolygonAreas.php new file mode 100644 index 000000000..9539b5233 --- /dev/null +++ b/app/Console/Commands/RecalculatePolygonAreas.php @@ -0,0 +1,87 @@ +cursor(); + + $processedCount = 0; + $errorCount = 0; + + $this->info('Starting polygon area recalculation...'); + $progressBar = $this->output->createProgressBar(); + $progressBar->start(); + + foreach ($sitePolygons as $sitePolygon) { + try { + $polygonGeometry = PolygonGeometry::where('uuid', $sitePolygon->poly_id) + ->select('uuid', DB::raw('ST_AsGeoJSON(geom) AS geojsonGeometry')) + ->first(); + if (! $polygonGeometry) { + $this->error("No geometry found for poly_id: {$sitePolygon->poly_id}"); + $errorCount++; + + continue; + } + $geometry = json_decode($polygonGeometry->geojsonGeometry, true); + + $calculatedArea = $areaService->getArea($geometry); + + $sitePolygon->calc_area = $calculatedArea; + $sitePolygon->save(); + + $processedCount++; + $progressBar->advance(); + } catch (\Exception $e) { + $this->error("Error processing polygon {$sitePolygon->id}: " . $e->getMessage()); + $errorCount++; + } + } + + DB::commit(); + + $progressBar->finish(); + $this->info("\n\nRecalculation complete!"); + $this->info("Processed: {$processedCount} polygons"); + $this->info("Errors: {$errorCount}"); + + } catch (\Exception $e) { + DB::rollBack(); + $this->error('Recalculation failed: ' . $e->getMessage()); + + return self::FAILURE; + } + + return self::SUCCESS; + } +} From cd152b617fae3922e04f76dfa6922c8d13158abc Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 14:45:53 -0400 Subject: [PATCH 03/14] [TM-1467] add new requeriments for python --- resources/python/polygon-voronoi/requirements.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/resources/python/polygon-voronoi/requirements.txt b/resources/python/polygon-voronoi/requirements.txt index 0d1ba9bb3..0ee921a2a 100755 --- a/resources/python/polygon-voronoi/requirements.txt +++ b/resources/python/polygon-voronoi/requirements.txt @@ -1,3 +1,13 @@ pyproj==3.4.1 numpy==1.26.4 shapely==2.0.1 +geopandas==1.0.1 +pandas==2.1.3 +requests==2.32.3 +fiona==1.10.1 +exactextract==0.2.0 +rasterio==1.4.1 +gdal==3.4.1 +pyyaml==6.0.2 +rasterstats==0.20.0 +boto3==1.35.43 \ No newline at end of file From 37030d1c3b2066402f6ab3d11756a5f5d0099f95 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 14:57:22 -0400 Subject: [PATCH 04/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 56eaa8143..4bedd4e9f 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -12,7 +12,12 @@ RUN apt-get install -y \ libzip-dev \ gdal-bin \ libgdal-dev \ - python3.11-venv \ + python3-dev \ + python3-venv \ + python3-pip \ + python3-numpy \ + build-essential \ + libproj-dev \ exiftool RUN docker-php-ext-configure gd --with-freetype --with-jpeg From befc45daca8304e04dfc0507fd0bf1784f49311d Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 15:06:31 -0400 Subject: [PATCH 05/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 4bedd4e9f..8f02fdbc7 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -12,6 +12,7 @@ RUN apt-get install -y \ libzip-dev \ gdal-bin \ libgdal-dev \ + python3.11-venv \ python3-dev \ python3-venv \ python3-pip \ From 291cf15c8ca636cca12725fedb633b65a14bb699 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 15:15:07 -0400 Subject: [PATCH 06/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 8f02fdbc7..14f325058 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -1,5 +1,5 @@ -## PHP FROM php:8.2-apache AS php + RUN apt-get update RUN apt-get install -y \ libxml2-dev \ @@ -21,6 +21,15 @@ RUN apt-get install -y \ libproj-dev \ exiftool +# Add these lines to set GDAL environment variables +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal +ENV C_INCLUDE_PATH=/usr/include/gdal + +# Get GDAL version and set it as an environment variable +RUN GDAL_VERSION=$(gdal-config --version) && \ + echo "GDAL version: ${GDAL_VERSION}" && \ + export GDAL_VERSION + RUN docker-php-ext-configure gd --with-freetype --with-jpeg RUN docker-php-ext-install \ bcmath \ @@ -50,7 +59,12 @@ COPY docker/php.ini /usr/local/etc/php/php.ini RUN python3 -m venv /opt/python COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt ENV PATH="/opt/python/bin:${PATH}" + +# Install GDAL before other requirements +RUN pip3 install --no-cache-dir numpy +RUN pip3 install --no-cache-dir GDAL==$(gdal-config --version) RUN pip3 install -r /root/voronoi-requirements.txt + RUN chmod -R a+rx /opt/python USER www-data ENV PATH="/opt/python/bin:${PATH}" From 46af592c60f9e2967a34cea95be8fe9c761358f8 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 15:21:41 -0400 Subject: [PATCH 07/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 14f325058..4ea928881 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -19,17 +19,17 @@ RUN apt-get install -y \ python3-numpy \ build-essential \ libproj-dev \ - exiftool + exiftool \ + gcc \ + g++ \ + python3-gdal # Add this package -# Add these lines to set GDAL environment variables +# Add GDAL specific environment variables ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal +ENV GDAL_VERSION=3.6.1 -# Get GDAL version and set it as an environment variable -RUN GDAL_VERSION=$(gdal-config --version) && \ - echo "GDAL version: ${GDAL_VERSION}" && \ - export GDAL_VERSION - +# PHP Extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg RUN docker-php-ext-install \ bcmath \ @@ -60,9 +60,14 @@ RUN python3 -m venv /opt/python COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt ENV PATH="/opt/python/bin:${PATH}" -# Install GDAL before other requirements -RUN pip3 install --no-cache-dir numpy -RUN pip3 install --no-cache-dir GDAL==$(gdal-config --version) +# Install Python packages +RUN pip3 install --upgrade pip +RUN pip3 install --no-cache-dir numpy wheel +# Try installing GDAL with apt package first +RUN apt-get install -y python3-gdal +# If you still need to install via pip, use the specific version +RUN pip3 install --no-binary :all: GDAL==${GDAL_VERSION} +# Install remaining requirements RUN pip3 install -r /root/voronoi-requirements.txt RUN chmod -R a+rx /opt/python From 9b4965917f97437eae85fed89c7b6312908d86a1 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 15:30:58 -0400 Subject: [PATCH 08/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 2 +- resources/python/polygon-voronoi/requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 4ea928881..af5544c32 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -27,7 +27,7 @@ RUN apt-get install -y \ # Add GDAL specific environment variables ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -ENV GDAL_VERSION=3.6.1 +ENV GDAL_VERSION=3.4.1 # PHP Extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg diff --git a/resources/python/polygon-voronoi/requirements.txt b/resources/python/polygon-voronoi/requirements.txt index 0ee921a2a..aed93c15c 100755 --- a/resources/python/polygon-voronoi/requirements.txt +++ b/resources/python/polygon-voronoi/requirements.txt @@ -7,7 +7,6 @@ requests==2.32.3 fiona==1.10.1 exactextract==0.2.0 rasterio==1.4.1 -gdal==3.4.1 pyyaml==6.0.2 rasterstats==0.20.0 boto3==1.35.43 \ No newline at end of file From fadc5ef9f0f7bd84b0be46653648b036c4764654 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 15:40:21 -0400 Subject: [PATCH 09/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 19 ++++++++++++++----- .../python/polygon-voronoi/requirements.txt | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index af5544c32..bd3ce5299 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -22,13 +22,19 @@ RUN apt-get install -y \ exiftool \ gcc \ g++ \ - python3-gdal # Add this package + python3-gdal \ + proj-data \ + proj-bin # Add GDAL specific environment variables ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal ENV GDAL_VERSION=3.4.1 +# Set GDAL configuration +RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal +RUN export C_INCLUDE_PATH=/usr/include/gdal + # PHP Extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg RUN docker-php-ext-install \ @@ -62,15 +68,18 @@ ENV PATH="/opt/python/bin:${PATH}" # Install Python packages RUN pip3 install --upgrade pip -RUN pip3 install --no-cache-dir numpy wheel -# Try installing GDAL with apt package first +RUN pip3 install --no-cache-dir numpy wheel setuptools + +# Install GDAL and its dependencies RUN apt-get install -y python3-gdal -# If you still need to install via pip, use the specific version +RUN gdal-config --version +ENV GDAL_CONFIG=/usr/bin/gdal-config RUN pip3 install --no-binary :all: GDAL==${GDAL_VERSION} + # Install remaining requirements RUN pip3 install -r /root/voronoi-requirements.txt RUN chmod -R a+rx /opt/python USER www-data ENV PATH="/opt/python/bin:${PATH}" -USER root +USER root \ No newline at end of file diff --git a/resources/python/polygon-voronoi/requirements.txt b/resources/python/polygon-voronoi/requirements.txt index aed93c15c..0ee921a2a 100755 --- a/resources/python/polygon-voronoi/requirements.txt +++ b/resources/python/polygon-voronoi/requirements.txt @@ -7,6 +7,7 @@ requests==2.32.3 fiona==1.10.1 exactextract==0.2.0 rasterio==1.4.1 +gdal==3.4.1 pyyaml==6.0.2 rasterstats==0.20.0 boto3==1.35.43 \ No newline at end of file From 51a397384619ff8e774b9449bd586c8c2bc27eeb Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 15:53:52 -0400 Subject: [PATCH 10/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 39 +++++++++---------- .../python/polygon-voronoi/requirements.txt | 1 - 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index bd3ce5299..88765ee36 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -1,7 +1,9 @@ FROM php:8.2-apache AS php -RUN apt-get update -RUN apt-get install -y \ +# Add backports for more recent GDAL version +RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list + +RUN apt-get update && apt-get install -y \ libxml2-dev \ libonig-dev \ libpng-dev \ @@ -24,16 +26,9 @@ RUN apt-get install -y \ g++ \ python3-gdal \ proj-data \ - proj-bin - -# Add GDAL specific environment variables -ENV CPLUS_INCLUDE_PATH=/usr/include/gdal -ENV C_INCLUDE_PATH=/usr/include/gdal -ENV GDAL_VERSION=3.4.1 - -# Set GDAL configuration -RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal -RUN export C_INCLUDE_PATH=/usr/include/gdal + proj-bin \ + libgdal28 \ + python3-gdal # PHP Extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg @@ -70,14 +65,18 @@ ENV PATH="/opt/python/bin:${PATH}" RUN pip3 install --upgrade pip RUN pip3 install --no-cache-dir numpy wheel setuptools -# Install GDAL and its dependencies -RUN apt-get install -y python3-gdal -RUN gdal-config --version -ENV GDAL_CONFIG=/usr/bin/gdal-config -RUN pip3 install --no-binary :all: GDAL==${GDAL_VERSION} - -# Install remaining requirements -RUN pip3 install -r /root/voronoi-requirements.txt +# Install remaining requirements EXCEPT GDAL (since we're using system GDAL) +RUN pip3 install pyproj==3.4.1 \ + shapely==2.0.1 \ + geopandas==1.0.1 \ + pandas==2.1.3 \ + requests==2.32.3 \ + fiona==1.10.1 \ + exactextract==0.2.0 \ + rasterio==1.4.1 \ + pyyaml==6.0.2 \ + rasterstats==0.20.0 \ + boto3==1.35.43 RUN chmod -R a+rx /opt/python USER www-data diff --git a/resources/python/polygon-voronoi/requirements.txt b/resources/python/polygon-voronoi/requirements.txt index 0ee921a2a..aed93c15c 100755 --- a/resources/python/polygon-voronoi/requirements.txt +++ b/resources/python/polygon-voronoi/requirements.txt @@ -7,7 +7,6 @@ requests==2.32.3 fiona==1.10.1 exactextract==0.2.0 rasterio==1.4.1 -gdal==3.4.1 pyyaml==6.0.2 rasterstats==0.20.0 boto3==1.35.43 \ No newline at end of file From 8761ebe7d68bd8f1c6a084f0157da2e496a02178 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 16:23:32 -0400 Subject: [PATCH 11/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 60 +++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 88765ee36..682121712 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -1,8 +1,5 @@ FROM php:8.2-apache AS php -# Add backports for more recent GDAL version -RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list - RUN apt-get update && apt-get install -y \ libxml2-dev \ libonig-dev \ @@ -12,23 +9,31 @@ RUN apt-get update && apt-get install -y \ libmagickwand-dev \ mariadb-client \ libzip-dev \ - gdal-bin \ - libgdal-dev \ + build-essential \ + software-properties-common \ + wget \ + curl \ + gcc \ + g++ \ + python3.11-dev \ python3.11-venv \ - python3-dev \ - python3-venv \ python3-pip \ python3-numpy \ - build-essential \ libproj-dev \ - exiftool \ - gcc \ - g++ \ - python3-gdal \ proj-data \ proj-bin \ - libgdal28 \ - python3-gdal + exiftool + +# Install GDAL 3.4.3 from source +RUN wget https://github.com/OSGeo/gdal/releases/download/v3.4.3/gdal-3.4.3.tar.gz && \ + tar -xvf gdal-3.4.3.tar.gz && \ + cd gdal-3.4.3 && \ + ./configure --with-python=/usr/bin/python3.11 && \ + make -j$(nproc) && \ + make install && \ + ldconfig && \ + cd .. && \ + rm -rf gdal-3.4.3 gdal-3.4.3.tar.gz # PHP Extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg @@ -56,27 +61,26 @@ RUN a2enmod rewrite COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf COPY docker/php.ini /usr/local/etc/php/php.ini -## Python -RUN python3 -m venv /opt/python +## Python setup +RUN python3.11 -m venv /opt/python COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt ENV PATH="/opt/python/bin:${PATH}" +# Set GDAL environment variables +ENV CPLUS_INCLUDE_PATH=/usr/local/include +ENV C_INCLUDE_PATH=/usr/local/include +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +ENV GDAL_VERSION=3.4.3 + # Install Python packages RUN pip3 install --upgrade pip RUN pip3 install --no-cache-dir numpy wheel setuptools -# Install remaining requirements EXCEPT GDAL (since we're using system GDAL) -RUN pip3 install pyproj==3.4.1 \ - shapely==2.0.1 \ - geopandas==1.0.1 \ - pandas==2.1.3 \ - requests==2.32.3 \ - fiona==1.10.1 \ - exactextract==0.2.0 \ - rasterio==1.4.1 \ - pyyaml==6.0.2 \ - rasterstats==0.20.0 \ - boto3==1.35.43 +# Install GDAL with specific version +RUN pip3 install GDAL==${GDAL_VERSION} + +# Install remaining requirements +RUN pip3 install -r /root/voronoi-requirements.txt RUN chmod -R a+rx /opt/python USER www-data From b903b963b2846e1031804013352a6459d6115ab4 Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 16:57:00 -0400 Subject: [PATCH 12/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 85 ++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index 682121712..b8db2b52d 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -1,5 +1,9 @@ FROM php:8.2-apache AS php +# Set GDAL version +ENV GDAL_VERSION=3.4.3 + +# Install basic dependencies RUN apt-get update && apt-get install -y \ libxml2-dev \ libonig-dev \ @@ -9,33 +13,38 @@ RUN apt-get update && apt-get install -y \ libmagickwand-dev \ mariadb-client \ libzip-dev \ + python3.11-venv \ + python3.11-dev \ + exiftool \ build-essential \ - software-properties-common \ wget \ - curl \ - gcc \ - g++ \ - python3.11-dev \ - python3.11-venv \ - python3-pip \ - python3-numpy \ + cmake \ + sqlite3 \ + libsqlite3-dev \ + libspatialite-dev \ + libpq-dev \ + libcurl4-gnutls-dev \ libproj-dev \ - proj-data \ - proj-bin \ - exiftool + libgeos-dev \ + && rm -rf /var/lib/apt/lists/* # Install GDAL 3.4.3 from source -RUN wget https://github.com/OSGeo/gdal/releases/download/v3.4.3/gdal-3.4.3.tar.gz && \ - tar -xvf gdal-3.4.3.tar.gz && \ - cd gdal-3.4.3 && \ - ./configure --with-python=/usr/bin/python3.11 && \ - make -j$(nproc) && \ - make install && \ - ldconfig && \ - cd .. && \ - rm -rf gdal-3.4.3 gdal-3.4.3.tar.gz +RUN wget https://github.com/OSGeo/gdal/releases/download/v${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} \ + && ./configure \ + && make -j$(nproc) \ + && make install \ + && ldconfig \ + && cd .. \ + && rm -rf gdal-${GDAL_VERSION} gdal-${GDAL_VERSION}.tar.gz + +# Set GDAL environment variables +ENV CPLUS_INCLUDE_PATH=/usr/include/gdal +ENV C_INCLUDE_PATH=/usr/include/gdal +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -# PHP Extensions +# Your existing PHP extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg RUN docker-php-ext-install \ bcmath \ @@ -56,31 +65,31 @@ RUN docker-php-ext-install exif RUN docker-php-ext-enable exif RUN mv "$PHP_INI_DIR/php.ini-development" "$PHP_INI_DIR/php.ini" -## APACHE +# Apache configuration RUN a2enmod rewrite COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf COPY docker/php.ini /usr/local/etc/php/php.ini -## Python setup +# Python virtual environment setup RUN python3.11 -m venv /opt/python -COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt ENV PATH="/opt/python/bin:${PATH}" -# Set GDAL environment variables -ENV CPLUS_INCLUDE_PATH=/usr/local/include -ENV C_INCLUDE_PATH=/usr/local/include -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -ENV GDAL_VERSION=3.4.3 - -# Install Python packages -RUN pip3 install --upgrade pip -RUN pip3 install --no-cache-dir numpy wheel setuptools - -# Install GDAL with specific version +# Install Python dependencies in the correct order +COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt +RUN pip3 install --upgrade pip wheel setuptools +RUN pip3 install numpy==1.26.4 +RUN pip3 install pyproj==3.4.1 RUN pip3 install GDAL==${GDAL_VERSION} - -# Install remaining requirements -RUN pip3 install -r /root/voronoi-requirements.txt +RUN pip3 install fiona==1.10.1 +RUN pip3 install shapely==2.0.1 +RUN pip3 install pandas==2.1.3 +RUN pip3 install geopandas==1.0.1 +RUN pip3 install rasterio==1.4.1 +RUN pip3 install exactextract==0.2.0 +RUN pip3 install rasterstats==0.20.0 +RUN pip3 install pyyaml==6.0.2 +RUN pip3 install requests==2.32.3 +RUN pip3 install boto3==1.35.43 RUN chmod -R a+rx /opt/python USER www-data From a7f9ad307cd2aeda7b2ea53bbf16b30b09f818ff Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 17:10:31 -0400 Subject: [PATCH 13/14] [TM-1467] add new requeriments for python --- docker/php.Dockerfile | 18 +++--------------- .../python/polygon-voronoi/requirements.txt | 10 +++++----- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/docker/php.Dockerfile b/docker/php.Dockerfile index b8db2b52d..46ab3fab7 100644 --- a/docker/php.Dockerfile +++ b/docker/php.Dockerfile @@ -1,3 +1,4 @@ +## PHP FROM php:8.2-apache AS php # Set GDAL version @@ -44,7 +45,6 @@ ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -# Your existing PHP extensions RUN docker-php-ext-configure gd --with-freetype --with-jpeg RUN docker-php-ext-install \ bcmath \ @@ -65,7 +65,7 @@ RUN docker-php-ext-install exif RUN docker-php-ext-enable exif RUN mv "$PHP_INI_DIR/php.ini-development" "$PHP_INI_DIR/php.ini" -# Apache configuration +## APACHE RUN a2enmod rewrite COPY docker/000-default.conf /etc/apache2/sites-available/000-default.conf COPY docker/php.ini /usr/local/etc/php/php.ini @@ -77,19 +77,7 @@ ENV PATH="/opt/python/bin:${PATH}" # Install Python dependencies in the correct order COPY resources/python/polygon-voronoi/requirements.txt /root/voronoi-requirements.txt RUN pip3 install --upgrade pip wheel setuptools -RUN pip3 install numpy==1.26.4 -RUN pip3 install pyproj==3.4.1 -RUN pip3 install GDAL==${GDAL_VERSION} -RUN pip3 install fiona==1.10.1 -RUN pip3 install shapely==2.0.1 -RUN pip3 install pandas==2.1.3 -RUN pip3 install geopandas==1.0.1 -RUN pip3 install rasterio==1.4.1 -RUN pip3 install exactextract==0.2.0 -RUN pip3 install rasterstats==0.20.0 -RUN pip3 install pyyaml==6.0.2 -RUN pip3 install requests==2.32.3 -RUN pip3 install boto3==1.35.43 +RUN pip3 install -r /root/voronoi-requirements.txt RUN chmod -R a+rx /opt/python USER www-data diff --git a/resources/python/polygon-voronoi/requirements.txt b/resources/python/polygon-voronoi/requirements.txt index aed93c15c..5885dc848 100755 --- a/resources/python/polygon-voronoi/requirements.txt +++ b/resources/python/polygon-voronoi/requirements.txt @@ -1,12 +1,12 @@ pyproj==3.4.1 numpy==1.26.4 shapely==2.0.1 -geopandas==1.0.1 -pandas==2.1.3 -requests==2.32.3 fiona==1.10.1 -exactextract==0.2.0 +pandas==2.1.3 +geopandas==1.0.1 rasterio==1.4.1 -pyyaml==6.0.2 +exactextract==0.2.0 rasterstats==0.20.0 +pyyaml==6.0.2 +requests==2.32.3 boto3==1.35.43 \ No newline at end of file From f6ecf94a908dc45632d89f73346058ed151c135c Mon Sep 17 00:00:00 2001 From: cesarLima1 Date: Thu, 19 Dec 2024 17:31:47 -0400 Subject: [PATCH 14/14] [TM-1467] add filter to get only active polygons in command --- app/Console/Commands/RecalculatePolygonAreas.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/Console/Commands/RecalculatePolygonAreas.php b/app/Console/Commands/RecalculatePolygonAreas.php index 9539b5233..508c3f879 100644 --- a/app/Console/Commands/RecalculatePolygonAreas.php +++ b/app/Console/Commands/RecalculatePolygonAreas.php @@ -16,7 +16,8 @@ class RecalculatePolygonAreas extends Command * @var string */ protected $signature = 'polygons:recalculate-areas - {--batch-size=100 : Number of records to process in each batch}'; + {--batch-size=100 : Number of records to process in each batch} + {--only-active : Process only active polygons}'; /** * The console command description. @@ -33,7 +34,13 @@ public function handle(AreaCalculationService $areaService) DB::beginTransaction(); try { - $sitePolygons = SitePolygon::query()->cursor(); + $query = SitePolygon::query(); + + if ($this->option('only-active')) { + $query->active(); + } + + $sitePolygons = $query->cursor(); $processedCount = 0; $errorCount = 0;