From 7c222ba15476cfea7e81080404d00d8d40bb0850 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Mon, 20 Nov 2023 09:40:16 +0100 Subject: [PATCH] require Python 3.10 and associated ruff updates (#39) This is a bit of a mixed bag of changes similar to which we merged in https://github.com/Deltares/Ribasim last weeks. First of all we now require at least Python 3.10. Before this was 3.9. The differences are not too big, but I don't think we need to spend out time supporting old Python versions at this moment, 3.10 is sufficiently old. Since we now tell ruff to target Python 3.10, and ask it to give syntax upgrade suggestions, we fix those. Most of this is nicer typing annotations, using e.g. `dict` instead of `from typing import Dict`. The ruff updates also includes some configuration to make it work well with notebooks. Lastly I removed the typos check. I don't think it is suitable for this repository as there are many Dutch terms used in the code and docs. --- .gitignore | 3 -- .pre-commit-config.yaml | 4 -- .typos.toml | 2 - .vscode/extensions.json | 3 +- .../{settings_template.json => settings.json} | 4 ++ environment.yml | 3 +- pixi.lock | 41 ++++++++--------- ruff.toml | 5 ++- src/hydamo/hydamo/datamodel.py | 45 +++++++++---------- src/hydamo/pyproject.toml | 2 +- src/ribasim_nl/pyproject.toml | 2 +- src/ribasim_nl/ribasim_nl/cloud.py | 25 +++++------ src/ribasim_nl/ribasim_nl/codes.py | 7 ++- src/ribasim_nl/ribasim_nl/geometry.py | 22 +++++---- 14 files changed, 79 insertions(+), 89 deletions(-) delete mode 100644 .typos.toml rename .vscode/{settings_template.json => settings.json} (66%) diff --git a/.gitignore b/.gitignore index d7eb5d87..a533bd82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ # pixi environments .pixi -# visual studio code -.vscode/settings.json - .ipynb_checkpoints # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7aacfaa4..09701b40 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,3 @@ repos: rev: 0.6.1 hooks: - id: nbstripout - - repo: https://github.com/crate-ci/typos - rev: v1.16.21 - hooks: - - id: typos diff --git a/.typos.toml b/.typos.toml deleted file mode 100644 index 4433e4cd..00000000 --- a/.typos.toml +++ /dev/null @@ -1,2 +0,0 @@ -[files] -extend-exclude = ["*.csv", "*.json", "*.ipynb", "cloud.py"] diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 6c254ab3..b9c05241 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,6 +3,7 @@ "ms-python.python", "ms-python.mypy-type-checker", "charliermarsh.ruff", - "njpwerner.autodocstring" + "njpwerner.autodocstring", + "quarto.quarto" ] } diff --git a/.vscode/settings_template.json b/.vscode/settings.json similarity index 66% rename from .vscode/settings_template.json rename to .vscode/settings.json index 24d488ba..bf473541 100644 --- a/.vscode/settings_template.json +++ b/.vscode/settings.json @@ -1,5 +1,9 @@ { "notebook.formatOnSave.enabled": true, + "notebook.codeActionsOnSave": { + "source.fixAll.ruff": true, + "source.organizeImports.ruff": true + }, "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, diff --git a/environment.yml b/environment.yml index 9c4e8a14..334d69a9 100644 --- a/environment.yml +++ b/environment.yml @@ -15,8 +15,6 @@ dependencies: - pandas - pandas-stubs - pip - - pip: - - quartodoc - pre-commit - pyarrow - pydantic=1 @@ -24,6 +22,7 @@ dependencies: - pytest - pytest-cov - python>=3.9 + - quartodoc - rasterstats - ruff - shapely>=2.0 diff --git a/pixi.lock b/pixi.lock index cc9d1346..c8c39b84 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,3 +1,4 @@ +version: 2 metadata: content_hash: linux-64: e90c2ee71ad70fc0a1c8302029533a7d1498f2bffcd0eaa8d2934700e775dc1d @@ -18816,7 +18817,7 @@ package: timestamp: 1691275738213 - platform: linux-64 name: quartodoc - version: 0.6.5 + version: 0.6.6 category: main manager: conda dependencies: @@ -18832,22 +18833,21 @@ package: - tabulate >=0.9.0 - typing-extensions >=4.4.0 - watchdog >=3.0.0 - url: https://conda.anaconda.org/conda-forge/noarch/quartodoc-0.6.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/quartodoc-0.6.6-pyhd8ed1ab_0.conda hash: - md5: b6a31dcfbb44a796bf9a3d938af29cf8 - sha256: 07d2a6273f01e97744ca78f1bf52d5fa7a534fe81239d341cbe51d5d7c7f0f9d + md5: 1eaa52c457eaae45459a45ee3f3fde98 + sha256: f7fddcb986f515530e05eeb687b485a8c0565b6163da77f2ad4d117f82687052 build: pyhd8ed1ab_0 arch: x86_64 subdir: linux-64 build_number: 0 license: MIT - license_family: MIT noarch: python - size: 57836 - timestamp: 1697861921173 + size: 58091 + timestamp: 1700237790215 - platform: osx-64 name: quartodoc - version: 0.6.5 + version: 0.6.6 category: main manager: conda dependencies: @@ -18863,22 +18863,21 @@ package: - tabulate >=0.9.0 - typing-extensions >=4.4.0 - watchdog >=3.0.0 - url: https://conda.anaconda.org/conda-forge/noarch/quartodoc-0.6.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/quartodoc-0.6.6-pyhd8ed1ab_0.conda hash: - md5: b6a31dcfbb44a796bf9a3d938af29cf8 - sha256: 07d2a6273f01e97744ca78f1bf52d5fa7a534fe81239d341cbe51d5d7c7f0f9d + md5: 1eaa52c457eaae45459a45ee3f3fde98 + sha256: f7fddcb986f515530e05eeb687b485a8c0565b6163da77f2ad4d117f82687052 build: pyhd8ed1ab_0 arch: x86_64 subdir: osx-64 build_number: 0 license: MIT - license_family: MIT noarch: python - size: 57836 - timestamp: 1697861921173 + size: 58091 + timestamp: 1700237790215 - platform: win-64 name: quartodoc - version: 0.6.5 + version: 0.6.6 category: main manager: conda dependencies: @@ -18894,19 +18893,18 @@ package: - tabulate >=0.9.0 - typing-extensions >=4.4.0 - watchdog >=3.0.0 - url: https://conda.anaconda.org/conda-forge/noarch/quartodoc-0.6.5-pyhd8ed1ab_0.conda + url: https://conda.anaconda.org/conda-forge/noarch/quartodoc-0.6.6-pyhd8ed1ab_0.conda hash: - md5: b6a31dcfbb44a796bf9a3d938af29cf8 - sha256: 07d2a6273f01e97744ca78f1bf52d5fa7a534fe81239d341cbe51d5d7c7f0f9d + md5: 1eaa52c457eaae45459a45ee3f3fde98 + sha256: f7fddcb986f515530e05eeb687b485a8c0565b6163da77f2ad4d117f82687052 build: pyhd8ed1ab_0 arch: x86_64 subdir: win-64 build_number: 0 license: MIT - license_family: MIT noarch: python - size: 57836 - timestamp: 1697861921173 + size: 58091 + timestamp: 1700237790215 - platform: linux-64 name: rasterio version: 1.3.9 @@ -23825,4 +23823,3 @@ package: license_family: BSD size: 343428 timestamp: 1693151615801 -version: 1 diff --git a/ruff.toml b/ruff.toml index 20e78ea8..2d0b1fbd 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,9 @@ -select = ["D", "E", "F", "NPY", "PD", "C4", "I"] +# See https://docs.astral.sh/ruff/rules/ +select = ["D", "E", "F", "NPY", "PD", "C4", "I", "UP"] ignore = ["D1", "D202", "D205", "D400", "D404", "E501", "PD002", "PD901"] fixable = ["I"] +extend-include = ["*.ipynb"] +target-version = "py310" [pydocstyle] convention = "numpy" diff --git a/src/hydamo/hydamo/datamodel.py b/src/hydamo/hydamo/datamodel.py index ddc321c0..388d76c8 100644 --- a/src/hydamo/hydamo/datamodel.py +++ b/src/hydamo/hydamo/datamodel.py @@ -5,7 +5,7 @@ import re import warnings from pathlib import Path -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Literal import fiona import geopandas as gpd @@ -40,7 +40,7 @@ "number": "float", } -default_properties: Dict[str, Any] = { +default_properties: dict[str, Any] = { "id": None, "dtype": "str", "required": False, @@ -48,18 +48,18 @@ } -def map_definition(definition: Dict[str, Any]) -> List[Dict[str, Any]]: +def map_definition(definition: dict[str, Any]) -> list[dict[str, Any]]: """ Parameters ---------- - definition : Dict + definition : dict[str, Any] HyDAMO definition as specified in the HyDAMO JSON specification. Returns ------- - List + list[dict[str, Any]] Validation schema for the HyDAMO class. """ @@ -70,7 +70,7 @@ def map_definition(definition: Dict[str, Any]) -> List[Dict[str, Any]]: for k, v in definition.items(): # convert geometry if shape if k == "shape": - properties: Dict[str, Any] = {"id": "geometry"} + properties: dict[str, Any] = {"id": "geometry"} dtype = v["type"] if not isinstance(dtype, list): dtype = [dtype] @@ -116,21 +116,20 @@ class ExtendedGeoDataFrame(gpd.GeoDataFrame): # type: ignore def __init__( self, - validation_schema: List[Dict[str, Any]], - geotype: Optional[ - List[ - Literal[ - "LineString", - "MultiLineString", - "Point", - "PointZ", - "Polygon", - "MultiPolygon", - ] + validation_schema: list[dict[str, Any]], + geotype: list[ + Literal[ + "LineString", + "MultiLineString", + "Point", + "PointZ", + "Polygon", + "MultiPolygon", ] - ], + ] + | None, layer_name: str = "", - required_columns: List[str] = [], + required_columns: list[str] = [], logger=logging, *args, **kwargs, @@ -144,7 +143,7 @@ def __init__( # else: kwargs["columns"] = required_columns - super(ExtendedGeoDataFrame, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.validation_schema = validation_schema self.required_columns = required_columns @@ -301,7 +300,7 @@ def __init__( self, version: str = "2.2", schemas_path: Path = SCHEMAS_DIR, - ignored_layers: List[str] = [ + ignored_layers: list[str] = [ "afvoeraanvoergebied", "imwa_geoobject", "leggerwatersysteem", @@ -311,7 +310,7 @@ def __init__( ): self.version = version self.schema_json = schemas_path.joinpath(f"HyDAMO_{version}.json") - self.layers: List[str] = [] + self.layers: list[str] = [] self.ignored_layers = ignored_layers self.init_datamodel() @@ -322,7 +321,7 @@ def data_layers(self): def init_datamodel(self) -> None: """Initialize DataModel from self.schemas_path.""" - self.validation_schemas: Dict[str, Any] = {} + self.validation_schemas: dict[str, Any] = {} # read schema as dict with open(self.schema_json) as src: diff --git a/src/hydamo/pyproject.toml b/src/hydamo/pyproject.toml index d120431c..fa29f8d9 100644 --- a/src/hydamo/pyproject.toml +++ b/src/hydamo/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] license = { text = "MIT" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "geopandas", ] diff --git a/src/ribasim_nl/pyproject.toml b/src/ribasim_nl/pyproject.toml index f55401af..75893b9c 100644 --- a/src/ribasim_nl/pyproject.toml +++ b/src/ribasim_nl/pyproject.toml @@ -12,7 +12,7 @@ authors = [ ] license = { text = "MIT" } -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ "geopandas", ] diff --git a/src/ribasim_nl/ribasim_nl/cloud.py b/src/ribasim_nl/ribasim_nl/cloud.py index 04f86b0b..586f8d3f 100644 --- a/src/ribasim_nl/ribasim_nl/cloud.py +++ b/src/ribasim_nl/ribasim_nl/cloud.py @@ -3,7 +3,6 @@ import shutil from dataclasses import dataclass, field from pathlib import Path -from typing import List, Tuple, Union from xml.etree import ElementTree import requests @@ -53,9 +52,9 @@ def is_dir(item): class CloudStorage: """Connect a local 'data_dir` to cloud-storage.""" - data_dir: Union[str, Path] = RIBASIM_NL_DATA_DIR + data_dir: str | Path = RIBASIM_NL_DATA_DIR user: str = RIBASIM_NL_CLOUD_USER - url: List[str] = BASE_URL + url: list[str] = BASE_URL password: str = field(repr=False, default=RIBASIM_NL_CLOUD_PASS) def __post_init__(self): @@ -89,18 +88,18 @@ def __post_init__(self): logger.info(f"{self.data_dir} is created") @property - def source_data(self) -> List[str]: + def source_data(self) -> list[str]: """List of all source_data (directories) in sub-folder 'Basisgegevens`.""" url = self.joinurl("Basisgegevens") return self.content(url) @property - def auth(self) -> Tuple[str, str]: + def auth(self) -> tuple[str, str]: """Auth tuple for requests""" return (self.user, self.password) @property - def water_authorities(self) -> List[str]: + def water_authorities(self) -> list[str]: """List of all water authorities (directories)""" return WATER_AUTHORITIES @@ -108,7 +107,7 @@ def validate_authority(self, authority): if authority not in self.water_authorities: raise ValueError(f"""'{authority}' not in {self.water_authorities}""") - def file_url(self, file_path: Union[str, Path]) -> str: + def file_url(self, file_path: str | Path) -> str: relative_path = Path(file_path).relative_to(self.data_dir) return f"{self.url}/{relative_path.as_posix()}" @@ -120,7 +119,7 @@ def file_path(self, file_url): relative_url = self.relative_url(file_url) return self.data_dir.joinpath(relative_url) - def relative_path(self, file_path: Union[str, Path]): + def relative_path(self, file_path: str | Path): return Path(file_path).relative_to(self.data_dir) def joinurl(self, *args: str): @@ -156,7 +155,7 @@ def download_file(self, file_url: str): with open(file_path, "wb") as f: f.write(r.content) - def content(self, url) -> Union[List[str], None]: + def content(self, url) -> list[str] | None: """List all content in a directory User can specify a path to the directory with additional arguments. @@ -171,7 +170,7 @@ def content(self, url) -> Union[List[str], None]: Returns ------- - List[str] + list[str] List of all content directories in a specified path """ @@ -203,7 +202,7 @@ def content(self, url) -> Union[List[str], None]: return content - def dirs(self, *args) -> List[str]: + def dirs(self, *args) -> list[str]: """List sub-directories in a directory User can specify a path to the directory with additional arguments. @@ -218,7 +217,7 @@ def dirs(self, *args) -> List[str]: Returns ------- - List[str] + list[str] List of directories in a specified path """ @@ -304,7 +303,7 @@ def download_verwerkt(self, authority: str, overwrite: bool = False): url = self.joinurl(authority, "verwerkt") self.download_content(url, overwrite=overwrite) - def download_basisgegevens(self, bronnen: List[str] = [], overwrite=True): + def download_basisgegevens(self, bronnen: list[str] = [], overwrite=True): """Download sources in the folder 'Basisgegevens'""" source_data = self.source_data if not bronnen: diff --git a/src/ribasim_nl/ribasim_nl/codes.py b/src/ribasim_nl/ribasim_nl/codes.py index e664d904..5cc77991 100644 --- a/src/ribasim_nl/ribasim_nl/codes.py +++ b/src/ribasim_nl/ribasim_nl/codes.py @@ -1,6 +1,5 @@ """Utilities for generating unique codes.""" from pathlib import Path -from typing import Dict, List, Union import pandas as pd from pandas import DataFrame @@ -46,7 +45,7 @@ def wbh_code_exists(wbh_code) -> bool: return wbh_code in codes_df.wbh_code.to_numpy() -def bgt_to_wbh_code(bgt_code) -> Union[str, None]: +def bgt_to_wbh_code(bgt_code) -> str | None: """Convert bgt_code to wbh_code if bgt_code exists""" wbh_code = None if bgt_code_exists(bgt_code): @@ -60,9 +59,9 @@ def bgt_to_wbh_code(bgt_code) -> Union[str, None]: def find_codes( organization: str, - administration_category: Union[str, None] = None, + administration_category: str | None = None, to_dict: bool = True, -) -> Union[Dict[str, List[Dict[str, str]]], DataFrame]: +) -> dict[str, list[dict[str, str]]] | DataFrame: codes = {} """Find codes associated with an organization""" codes_df = get_codes_df() diff --git a/src/ribasim_nl/ribasim_nl/geometry.py b/src/ribasim_nl/ribasim_nl/geometry.py index cb9b6e18..228dc103 100644 --- a/src/ribasim_nl/ribasim_nl/geometry.py +++ b/src/ribasim_nl/ribasim_nl/geometry.py @@ -1,6 +1,6 @@ # %% """Functions to apply on a shapely.geometry""" -from typing import List, Union, get_type_hints +from typing import get_type_hints from shapely.geometry import LineString, MultiPolygon, Point, Polygon from shapely.ops import polygonize, polylabel @@ -8,12 +8,12 @@ from ribasim_nl.generic import _validate_inputs -def basin_to_point(basin_polygon: Union[Polygon, MultiPolygon]) -> Point: +def basin_to_point(basin_polygon: Polygon | MultiPolygon) -> Point: """Return a representative point for the basin; centroid if it is within (Multi)Polygon or polylabel if not. Parameters ---------- - basin_polygon : Union[Polygon, MultiPolygon] + basin_polygon : Polygon | MultiPolygon (Multi)Polygon to get representative point for Returns @@ -34,19 +34,17 @@ def basin_to_point(basin_polygon: Union[Polygon, MultiPolygon]) -> Point: return point -def sort_basins( - basin_polygons: Union[MultiPolygon, List[Polygon]] -) -> Union[MultiPolygon, list]: +def sort_basins(basin_polygons: MultiPolygon | list[Polygon]) -> MultiPolygon | list: """Sort basins in a MultiPolygon or list of Polygons on .area in ascending order (small to large). Parameters ---------- - basin_polygons : Union[MultiPolygon, list] + basin_polygons : MultiPolygon | list[Polygon] MultiPolygon or list of polygons to be sorted Returns ------- - Union[MultiPolygon, list] + MultiPolygon | list MultiPolygon with sorted polygons """ is_multipolygon = isinstance(basin_polygons, MultiPolygon) @@ -108,18 +106,18 @@ def cut_basin(basin_polygon: Polygon, line: LineString) -> MultiPolygon: def drop_z( - geometry: Union[LineString, MultiPolygon, Point, Polygon] -) -> Union[Point, Polygon, MultiPolygon]: + geometry: LineString | MultiPolygon | Point | Polygon +) -> Point | Polygon | MultiPolygon: """Drop the z-coordinate of a geometry if it has. Parameters ---------- - geometry : Union[LineString, MultiPolygon, Point, Polygon] + geometry : LineString | MultiPolygon | Point | Polygon Input geometry Returns ------- - Union[Point, Polygon, MultiPolygon] + Point | Polygon | MultiPolygon Output geometry """