Scotland, Finland, Italy, Flanders, Spain, France and Norway DTM prov…

…iders * Scotland DTM provider * linting (as usual) * moar linting * and more linting * Finland provider * remove superfluous print statements * update docs on DTM providers that require a key * And Tinitaly provider * Add Flanders provider * Spain 5m DTM Provider * remove grequests * Add France provider * Norway provider * Ignore similar code in providers * Addressed PR feedback * forgot a file * try again after merging
iwatkot · Jan 18, 2025 · efbd296 · efbd296
1 parent 079aea8
commit efbd296
Show file tree

Hide file tree

Showing 18 changed files with 488 additions and 74 deletions.
diff --git a/.pylintrc b/.pylintrc
@@ -21,5 +21,6 @@ disable=raw-checker-failed,
         too-many-locals,
         redefined-builtin,
         too-many-instance-attributes,
+        duplicate-code,
         too-many-statements,
-        attribute-defined-outside-init
+        attribute-defined-outside-init
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -1,42 +1,42 @@
 {
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Streamlit: webui",
-            "type": "debugpy",
-            "request": "launch",
-            "module": "streamlit",
-            "args": ["run", "./webui/webui.py"],
-            "presentation": {
-                "reveal": "always",
-                "panel": "new"
-            },
-            "env": {
-                "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}",
-                "LOG_LEVEL": "DEBUG",
-            }
-        },
-        {
-            "name": "demo.py",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/demo.py",
-            "console": "integratedTerminal",
-            "justMyCode": true,
-            "env": {
-              "PYTHONPATH": "${workspaceFolder}"
-            }
-        },
-        {
-            "name": "Current File",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal",
-            "justMyCode": true,
-            "env": {
-              "PYTHONPATH": "${workspaceFolder}"
-            }
-        }
-    ]
-}
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Streamlit: webui",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "streamlit",
+      "args": ["run", "./webui/webui.py"],
+      "presentation": {
+        "reveal": "always",
+        "panel": "new"
+      },
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}",
+        "LOG_LEVEL": "DEBUG"
+      }
+    },
+    {
+      "name": "demo.py",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/demo.py",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      }
+    },
+    {
+      "name": "Current File",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}"
+      }
+    }
+  ]
+}
diff --git a/docs/dtm_providers.md b/docs/dtm_providers.md
@@ -122,7 +122,7 @@ Then, it determines which tiles are needed, downloads them all to a temporary fo
 Finally, it returns a list of file paths to the downloaded tiles.
 
 As you can see, it's pretty simple to implement a DTM provider. You can use any source of elevation data, as long as it's free and open.
-NOTE: DTM Providers which require API keys, paid subscriptions, or any other form of payment will not be considered for implementation in the generator.
+NOTE: If a DTM Provider requires an API key, paid subscription, or any other form of payment, you will be fully responsible for setting up your own access to the provider. The provider in the app will expose the settings needed to provide your authentication key or other required information.
 
 ### How DTM Provider can interact with the generator?
 

diff --git a/maps4fs/__init__.py b/maps4fs/__init__.py
@@ -8,6 +8,13 @@
 from maps4fs.generator.dtm.hessen import HessenProvider
 from maps4fs.generator.dtm.england import England1MProvider
 from maps4fs.generator.dtm.canada import CanadaProvider
+from maps4fs.generator.dtm.scotland import ScotlandProvider
+from maps4fs.generator.dtm.finland import FinlandProvider
+from maps4fs.generator.dtm.italy import ItalyProvider
+from maps4fs.generator.dtm.flanders import FlandersProvider
+from maps4fs.generator.dtm.spain import SpainProvider
+from maps4fs.generator.dtm.france import FranceProvider
+from maps4fs.generator.dtm.norway import NorwayProvider
 from maps4fs.generator.game import Game
 from maps4fs.generator.map import Map
 from maps4fs.generator.settings import (

diff --git a/maps4fs/generator/dtm/base/wcs.py b/maps4fs/generator/dtm/base/wcs.py
@@ -30,6 +30,18 @@ def get_wcs_parameters(self, tile: tuple[float, float, float, float]) -> dict:
             dict: The parameters for the WCS request.
         """
 
+    def get_wcs_instance_parameters(self) -> dict:
+        """Get the parameters for the WCS instance.
+
+        Returns:
+            dict: The parameters for the WCS instance.
+        """
+        return {
+            "url": self._url,
+            "version": self._wcs_version,
+            "timeout": 120,
+        }
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.shared_tiff_path = os.path.join(self._tile_directory, "shared")
@@ -53,12 +65,9 @@ def download_all_tiles(self, tiles: list[tuple[float, float, float, float]]) ->
             list: List of paths to the downloaded GeoTIFF files.
         """
         all_tif_files = []
-        wcs = WebCoverageService(
-            self._url,
-            version=self._wcs_version,
-            # auth=Authentication(verify=False),
-            timeout=600,
-        )
+        params = self.get_wcs_instance_parameters()
+        wcs = WebCoverageService(**params)
+
         for tile in tqdm(tiles, desc="Downloading tiles", unit="tile"):
             file_name = "_".join(map(str, tile)) + ".tif"
             file_path = os.path.join(self.shared_tiff_path, file_name)

diff --git a/maps4fs/generator/dtm/base/wms.py b/maps4fs/generator/dtm/base/wms.py
@@ -4,6 +4,7 @@
 import os
 
 from owslib.wms import WebMapService
+from tqdm import tqdm
 
 from maps4fs.generator.dtm import utils
 from maps4fs.generator.dtm.dtm import DTMProvider
@@ -58,7 +59,7 @@ def download_all_tiles(self, tiles: list[tuple[float, float, float, float]]) ->
             # auth=Authentication(verify=False),
             timeout=600,
         )
-        for tile in tiles:
+        for tile in tqdm(tiles, desc="Downloading tiles", unit="tile"):
             file_name = "_".join(map(str, tile)) + ".tif"
             file_path = os.path.join(self.shared_tiff_path, file_name)
             if not os.path.exists(file_path):

diff --git a/maps4fs/generator/dtm/dtm.py b/maps4fs/generator/dtm/dtm.py
@@ -12,13 +12,14 @@
 import numpy as np
 import osmnx as ox  # type: ignore
 import rasterio  # type: ignore
-import requests
 from pydantic import BaseModel
 from rasterio.enums import Resampling
 from rasterio.merge import merge
 from rasterio.warp import calculate_default_transform, reproject
+import requests
 from tqdm import tqdm
 
+
 from maps4fs.logger import Logger
 
 if TYPE_CHECKING:
@@ -395,28 +396,47 @@ def download_tif_files(self, urls: list[str], output_path: str) -> list[str]:
             list: List of paths to the downloaded GeoTIFF files.
         """
         tif_files: list[str] = []
-        for url in tqdm(urls, desc="Downloading tiles", unit="tile"):
+
+        existing_file_urls = [
+            f for f in urls if os.path.exists(os.path.join(output_path, os.path.basename(f)))
+        ]
+
+        for url in existing_file_urls:
+            self.logger.debug("File already exists: %s", os.path.basename(url))
             file_name = os.path.basename(url)
-            self.logger.debug("Retrieving TIFF: %s", file_name)
             file_path = os.path.join(output_path, file_name)
-            if not os.path.exists(file_path):
-                try:
-                    # Send a GET request to the file URL
-                    response = requests.get(url, stream=True, timeout=60)
-                    response.raise_for_status()  # Raise an error for HTTP status codes 4xx/5xx
-
-                    # Write the content of the response to the file
-                    with open(file_path, "wb") as file:
-                        for chunk in response.iter_content(chunk_size=8192):  # Download in chunks
-                            file.write(chunk)
-                    self.logger.debug("File downloaded successfully: %s", file_path)
-                except requests.exceptions.RequestException as e:
-                    self.logger.error("Failed to download file: %s", e)
-            else:
-                self.logger.debug("File already exists: %s", file_name)
             if file_name.endswith(".zip"):
                 file_path = self.unzip_img_from_tif(file_name, output_path)
             tif_files.append(file_path)
+
+        for url in tqdm(
+            (u for u in urls if u not in existing_file_urls),
+            desc="Downloading tiles",
+            unit="tile",
+            initial=len(tif_files),
+        ):
+            try:
+                file_name = os.path.basename(url)
+                file_path = os.path.join(output_path, file_name)
+                self.logger.debug("Retrieving TIFF: %s", file_name)
+
+                # Send a GET request to the file URL
+                response = requests.get(url, stream=True, timeout=60)
+                response.raise_for_status()  # Raise an error for HTTP status codes 4xx/5xx
+
+                # Write the content of the response to the file
+                with open(file_path, "wb") as file:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        file.write(chunk)
+
+                self.logger.debug("File downloaded successfully: %s", file_path)
+
+                if file_name.endswith(".zip"):
+                    file_path = self.unzip_img_from_tif(file_name, output_path)
+
+                tif_files.append(file_path)
+            except requests.exceptions.RequestException as e:
+                self.logger.error("Failed to download file: %s", e)
         return tif_files
 
     def unzip_img_from_tif(self, file_name: str, output_path: str) -> str:
@@ -585,7 +605,7 @@ def normalize_dem(self, data: np.ndarray) -> np.ndarray:
                 "Applying power factor: %s to the DEM data.",
                 power_factor,
             )
-            data = np.power(data, power_factor).astype(np.uint16)
+            data = np.power(data, power_factor)
 
         normalized_data = np.round(data * scaling_factor).astype(np.uint16)
         self.logger.debug(

diff --git a/maps4fs/generator/dtm/finland.py b/maps4fs/generator/dtm/finland.py
@@ -0,0 +1,53 @@
+"""This module contains provider of Finland data."""
+
+from owslib.util import Authentication
+
+from maps4fs.generator.dtm.base.wcs import WCSProvider
+from maps4fs.generator.dtm.dtm import DTMProvider, DTMProviderSettings
+
+
+class FinlandProviderSettings(DTMProviderSettings):
+    """Settings for the Finland provider."""
+
+    api_key: str = ""
+
+
+class FinlandProvider(WCSProvider, DTMProvider):
+    """Provider of Finland data."""
+
+    _code = "finland"
+    _name = "Finland"
+    _region = "FI"
+    _icon = "🇫🇮"
+    _resolution = 2
+    _settings = FinlandProviderSettings
+    _author = "[kbrandwijk](https://github.com/kbrandwijk)"
+    _is_community = True
+    _is_base = False
+    _extents = (70.09, 59.45, 31.59, 19.08)
+
+    _url = "https://avoin-karttakuva.maanmittauslaitos.fi/ortokuvat-ja-korkeusmallit/wcs/v2"
+    _wcs_version = "2.0.1"
+    _source_crs = "EPSG:3067"
+    _tile_size = 1000
+
+    _instructions = (
+        "ℹ️ This provider requires an API Key. See [here](https://www.maanmittausl"
+        "aitos.fi/rajapinnat/api-avaimen-ohje) for more information on how to create one, then "
+        "enter it below in the settings field for API Key."
+    )
+
+    def get_wcs_instance_parameters(self):
+        settings = super().get_wcs_instance_parameters()
+        settings["auth"] = Authentication(
+            username=self.user_settings.api_key, password=self.user_settings.api_key
+        )
+        return settings
+
+    def get_wcs_parameters(self, tile: tuple[float, float, float, float]) -> dict:
+        return {
+            "identifier": ["korkeusmalli_2m"],
+            "subsets": [("N", str(tile[0]), str(tile[2])), ("E", str(tile[1]), str(tile[3]))],
+            "format": "image/tiff",
+            "timeout": 600,
+        }
diff --git a/maps4fs/generator/dtm/flanders.py b/maps4fs/generator/dtm/flanders.py
@@ -0,0 +1,34 @@
+"""This module contains provider of Flanders data."""
+
+from maps4fs.generator.dtm.base.wcs import WCSProvider
+from maps4fs.generator.dtm.dtm import DTMProvider
+
+
+class FlandersProvider(WCSProvider, DTMProvider):
+    """Provider of Flanders data."""
+
+    _code = "flanders"
+    _name = "Flanders DHM II"
+    _region = "BE"
+    _icon = "🇧🇪"
+    _resolution = 1
+    _author = "[kbrandwijk](https://github.com/kbrandwijk)"
+    _is_community = True
+    _is_base = False
+    _extents = (51.5150730375579684, 50.6694992827160817, 5.9444417082210812, 2.5170092434134252)
+
+    _url = "https://geo.api.vlaanderen.be/el-dtm/wcs"
+    _wcs_version = "1.0.0"
+    _source_crs = "EPSG:4258"
+    _tile_size = 0.02
+
+    def get_wcs_parameters(self, tile: tuple[float, float, float, float]) -> dict:
+        return {
+            "identifier": "EL.GridCoverage.DTM",
+            "bbox": tile,
+            "format": "GeoTIFF",
+            "crs": "EPSG:4258",
+            "width": 1000,
+            "height": 1000,
+            "timeout": 600,
+        }