Merge pull request #79 from wri/develop

Merge changes (Python, package upgrades) up to production

.github/workflows/terraform_build.yaml

@@ -9,10 +9,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.7
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: '3.10'
     - name: Lint with flake8
       run: |
         pip install flake8

.isort.cfg

@@ -2,4 +2,4 @@
 line_length = 88
 multi_line_output = 3
 include_trailing_comma = True
-known_third_party = asyncclick,aws_xray_sdk,boto3,geobuf,http3,moz_sql_parser,numpy,pandas,pydantic,pytest,rasterio,setuptools,shapely
+known_third_party = asyncclick,aws_xray_sdk,boto3,geobuf,http3,mo_parsing,mo_sql_parsing,numpy,pandas,pydantic,pytest,rasterio,setuptools,shapely

.pre-commit-config.yaml

@@ -6,35 +6,37 @@ repos:
     hooks:
     - id: seed-isort-config
 -   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.7.0
+    rev: v5.10.1
     hooks:
     - id: isort
 -   repo: https://github.com/myint/docformatter
-    rev: v1.4
+    rev: v1.7.5
     hooks:
     - id: docformatter
       args: [--in-place]
 -   repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 23.7.0
     hooks:
     - id: black
-      language_version: python3.8
+      language_version: python3.10
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.4.0
     hooks:
     - id: detect-aws-credentials
     - id: detect-private-key
     - id: trailing-whitespace
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
     hooks:
     - id: flake8
+      language_version: python3.10
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.790
+    rev: v1.5.1
     hooks:
     -   id: mypy
+        language_version: python3.10
 -   repo: https://github.com/Yelp/detect-secrets
-    rev: v0.14.3
+    rev: v1.4.0
     hooks:
     -   id: detect-secrets
         args: ['--baseline', '.secrets.baseline'] # run: `pip install detect-secrets` to establish baseline

README.md

@@ -1,4 +1,4 @@
-# GFW raster analsyis in AWS Lambda
+# GFW raster analysis in AWS Lambda
 
 ### Functionality
 
@@ -280,18 +280,3 @@ The GFW data lake is now in production, so this service will soon point to that
 
 These alerts are currently unsupported because we don't rasterize these layers. Instead, we store all enriched points in an document dataset. You can do on-the-fly analysis for these via SQL. (TBD: do we want to just forward that through here so there's only one endpoint?)
 
-### Aggregation
-
-There will be a new `String` parameter called `agg` that will accept one of `day | week | month | year` and return results aggregated by that timeline.
-
-### Whitelist
-
-There will be a new endpoint that will return a whitelist of whether layers intersect the input geometry. Details TBD.
-
-### Misc Layers
-
-Need to decide if/how we will support miscellanious layers on GFW but not maintained in the data lake, like PRODES and Terra-i. TBD.
-
-### Lat/Lon Coordinates
-
-A new analysis will be added to retrieve lat/lon coordinates of points (e.g. for GLAD alerts).

docker/raster_analysis/.lambdaignore

@@ -1,4 +1,5 @@
 *-info/*
+*/tests/*
 python/awscli/examples/*
 python/boto3/*
 python/botocore/*
@@ -28,4 +29,9 @@ python/setuptools/*
 python/tzlocal/test_data/*
 python/urllib3/*
 python/websocket/tests/*
-python/wheel/*
+python/wheel/*
+share/cryptopp/*
+share/doc/*
+share/gtk-doc/*
+share/libxml2-2.9.10/*
+share/man/*

docker/raster_analysis/Dockerfile

@@ -1,4 +1,4 @@
-FROM lambgeo/lambda-gdal:3.2-python3.7
+FROM ghcr.io/lambgeo/lambda-gdal:3.6-python3.10
 
 WORKDIR /var/task
 ENV WORKDIR /var/task
@@ -11,19 +11,19 @@ COPY raster_analysis/ raster_analysis/
 COPY setup.py .
 COPY docker/raster_analysis/.lambdaignore .
 
-# installing dependencies to build package
+# Install dependencies to build package
 ENV SETUPTOOLS_USE_DISTUTILS stdlib
 RUN pip3 install . -t python
 
-#Precompile all python packages and remove .py files
+RUN yum install -y zip
+
+# Precompile all python packages and remove .py files
 RUN python -m compileall .
-RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-37//'); cp $f $n; done;
+RUN find python/ -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-310//'); cp $f $n; done;
 RUN find python/ -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf
 RUN find python/ -type f -a -name '*.py' -print0 | xargs -0 rm -f
 
 # Compress all source codes expect files listed in .lambdaignore
 RUN cat .lambdaignore | xargs zip -9qyr layer.zip python -x
 
-CMD ["/bin/bash"]
-
-
+CMD ["/bin/bash"]

docker/test/Dockerfile

@@ -1,4 +1,4 @@
-FROM lambgeo/lambda-gdal:3.2-python3.7
+FROM ghcr.io/lambgeo/lambda-gdal:3.6-python3.10
 
 # Make the dir and to install all packages into packages/
 COPY raster_analysis/ raster_analysis/

raster_analysis/data_environment.py

@@ -1,13 +1,13 @@
 # flake8: noqa
 import json
-from collections import Iterable, defaultdict
+from collections import defaultdict
+from collections.abc import Iterable
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union, cast
 
 from numpy import (
     ceil,
     datetime64,
-    float,
     float32,
     float64,
     floor,
@@ -107,7 +107,10 @@ def get_layer(self, name: str) -> Layer:
 
     def get_layer_grid(self, name: str) -> Grid:
         layer = self.get_layer(name)
-        return Grid.get_grid(layer.grid)
+        if isinstance(layer, SourceLayer):
+            return Grid.get_grid(layer.grid)
+        else:
+            raise Exception("Tried to get grid of a non-SourceLayer!")
 
     def has_layer(self, name: str) -> bool:
         try:
@@ -122,14 +125,12 @@ def get_layers(self, layer_names: List[str]) -> List[Layer]:
     def get_source_layers(self, layer_names: List[str]) -> List[SourceLayer]:
         layers = self.get_layers(layer_names)
 
-        source_layers = []
+        source_layers: List[SourceLayer] = []
         for layer in layers:
             if isinstance(layer, SourceLayer):
-                layer = cast(SourceLayer, layer)
                 source_layers.append(layer)
             elif isinstance(layer, DerivedLayer):
-                layer = cast(DerivedLayer, layer)
-                source_layer = self.get_layer(layer.source_layer)
+                source_layer: SourceLayer = self.get_layer(layer.source_layer)
                 source_layers.append(source_layer)
 
         return source_layers
@@ -182,17 +183,17 @@ def decode_layer(self, name: str, values: Series) -> Series:
     def get_source_uri(self, name: str, tile: BasePolygon):
         layer = self.get_layer(name)
 
-        source_uri = layer.source_uri
-
-        if layer.tile_scheme:
-            grid = Grid.get_grid(layer.grid)
+        if isinstance(layer, SourceLayer):
+            grid = Grid.get_grid(GridName(layer.grid))
             tile_id = grid.get_tile_id(tile, layer.tile_scheme)
             source_uri = layer.source_uri.format(tile_id=tile_id)
 
-        if "s3" in source_uri:
-            source_uri = source_uri.replace("s3://", "/vsis3/")
+            if "s3" in source_uri:
+                source_uri = source_uri.replace("s3://", "/vsis3/")
 
-        return source_uri
+            return source_uri
+        else:
+            raise Exception("Tried to call get_source_uri on a non-SourceLayer!")
 
     def get_pixel_encoding(self, name: str) -> Dict[int, Any]:
         layer = self.get_layer(name)
@@ -212,7 +213,7 @@ def get_pixel_encoding(self, name: str) -> Dict[int, Any]:
             return {}
 
     def has_default_value(self, name):
-        """A layer is defined as having a default value if it's encoding
+        """A layer is defined as having a default value if its encoding
         contains the default value symbol '_', or a mapping for 0.
 
         Otherwise, 0 is considered NoData and will be filtered out.

raster_analysis/geometry.py

@@ -1,5 +1,5 @@
 import sys
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 import geobuf
 from shapely.geometry import Polygon, mapping, shape
@@ -11,8 +11,8 @@
 class GeometryTile:
     def __init__(
         self,
-        source_geom: Any,
-        tile_geojson: Dict[str, Any] = None,
+        source_geom: str,
+        tile_geojson: Optional[Dict[str, Any]] = None,
         is_encoded: bool = False,
     ):
         if is_encoded:
@@ -21,7 +21,7 @@ def __init__(
             full_geom = shape(source_geom).buffer(0)
 
         self.geom: BasePolygon = full_geom
-        self.tile: Polygon = None
+        self.tile: Optional[Polygon] = None
 
         if tile_geojson:
             tile = shape(tile_geojson)