From 5c6a369fe42ea7cba43293137ba53581369bad24 Mon Sep 17 00:00:00 2001 From: deltamarnix <150045289+deltamarnix@users.noreply.github.com> Date: Tue, 14 Nov 2023 22:11:18 +0100 Subject: [PATCH] Support toml in QGIS plugin (#781) Fixes part of #629 (the results are not automatically associated yet). The Open button now supports toml instead of gpkg. The toml file checks for the "database" entry and loads that file as it used to do. The New button now also creates a toml file with the database entry pointing to the gpkg that is also created like before. And the start and end dates are 01-01-2020 until 01-01-2030. An option for another PR would be to add the start and end dates as fields in the main widget so that they can be adjusted. But that would mean that we need to keep the open toml file in memory and set the start and end date, and flush the file after those changes. Also, error handling is not fancy yet. If the "database" entry is missing from the toml file, you will get a python KeyError The folder `qgis` was renamed to `ribasim_qgis` for pylance to handle the imports better. --------- Co-authored-by: Martijn Visser --- .docker/compose.yml | 4 +- .../Ribasim_Ribasim_MakeQgisPlugin.xml | 2 +- docs/contribute/addnode.qmd | 2 +- {qgis => ribasim_qgis}/LICENSE | 0 {qgis => ribasim_qgis}/__init__.py | 0 {qgis => ribasim_qgis}/core/__init__.py | 0 {qgis => ribasim_qgis}/core/geopackage.py | 0 {qgis => ribasim_qgis}/core/nodes.py | 4 +- {qgis => ribasim_qgis}/core/topology.py | 1 - {qgis => ribasim_qgis}/icon.png | Bin {qgis => ribasim_qgis}/metadata.txt | 0 {qgis => ribasim_qgis}/resources.py | 0 {qgis => ribasim_qgis}/resources.qrc | 0 {qgis => ribasim_qgis}/ribasim_qgis.py | 0 {qgis => ribasim_qgis}/tests/__init__.py | 0 .../tests/test_load_plugin.py | 0 ribasim_qgis/tomllib/__init__.py | 13 + ribasim_qgis/tomllib/_parser.py | 694 ++++++++++++++++++ ribasim_qgis/tomllib/_re.py | 110 +++ ribasim_qgis/tomllib/_types.py | 13 + .../widgets/dataset_widget.py | 56 +- .../widgets/nodes_widget.py | 1 + .../widgets/results_widget.py | 0 .../widgets/ribasim_widget.py | 6 +- ruff.toml | 3 + 25 files changed, 881 insertions(+), 28 deletions(-) rename {qgis => ribasim_qgis}/LICENSE (100%) rename {qgis => ribasim_qgis}/__init__.py (100%) rename {qgis => ribasim_qgis}/core/__init__.py (100%) rename {qgis => ribasim_qgis}/core/geopackage.py (100%) rename {qgis => ribasim_qgis}/core/nodes.py (100%) rename {qgis => ribasim_qgis}/core/topology.py (99%) rename {qgis => ribasim_qgis}/icon.png (100%) rename {qgis => ribasim_qgis}/metadata.txt (100%) rename {qgis => ribasim_qgis}/resources.py (100%) rename {qgis => ribasim_qgis}/resources.qrc (100%) rename {qgis => ribasim_qgis}/ribasim_qgis.py (100%) rename {qgis => ribasim_qgis}/tests/__init__.py (100%) rename {qgis => ribasim_qgis}/tests/test_load_plugin.py (100%) create mode 100644 ribasim_qgis/tomllib/__init__.py create mode 100644 ribasim_qgis/tomllib/_parser.py create mode 100644 ribasim_qgis/tomllib/_re.py create mode 100644 ribasim_qgis/tomllib/_types.py rename {qgis => ribasim_qgis}/widgets/dataset_widget.py (86%) rename {qgis => ribasim_qgis}/widgets/nodes_widget.py (99%) rename {qgis => ribasim_qgis}/widgets/results_widget.py (100%) rename {qgis => ribasim_qgis}/widgets/ribasim_widget.py (98%) diff --git a/.docker/compose.yml b/.docker/compose.yml index 0d680c5fa..2405ad9c2 100644 --- a/.docker/compose.yml +++ b/.docker/compose.yml @@ -2,10 +2,10 @@ version: '2' services: qgis: - image: qgis/qgis:release-3_30 + image: qgis/qgis:release-3_28 container_name: qgis volumes: - - ../qgis/:/tests_directory/${PLUGIN_NAME} + - ../ribasim_qgis/:/tests_directory/${PLUGIN_NAME} environment: - CI=true - DISPLAY=:99 diff --git a/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_MakeQgisPlugin.xml b/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_MakeQgisPlugin.xml index 1d8aed453..fb9a9b29c 100644 --- a/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_MakeQgisPlugin.xml +++ b/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_MakeQgisPlugin.xml @@ -10,7 +10,7 @@ - diff --git a/docs/contribute/addnode.qmd b/docs/contribute/addnode.qmd index 4c5d8fbbb..5aab81531 100644 --- a/docs/contribute/addnode.qmd +++ b/docs/contribute/addnode.qmd @@ -168,7 +168,7 @@ In `python/ribasim/ribasim/geometry/node.py` add a color and shape description i # QGIS plugin -The script `qgis/core/nodes.py` has to be updated to specify how the new node type is displayed by the QGIS plugin. Specifically: +The script `ribasim_qgis/core/nodes.py` has to be updated to specify how the new node type is displayed by the QGIS plugin. Specifically: - Add a color and shape description in the `MARKERS` dictionary in `Node.renderer`, consistent with the entries added above; diff --git a/qgis/LICENSE b/ribasim_qgis/LICENSE similarity index 100% rename from qgis/LICENSE rename to ribasim_qgis/LICENSE diff --git a/qgis/__init__.py b/ribasim_qgis/__init__.py similarity index 100% rename from qgis/__init__.py rename to ribasim_qgis/__init__.py diff --git a/qgis/core/__init__.py b/ribasim_qgis/core/__init__.py similarity index 100% rename from qgis/core/__init__.py rename to ribasim_qgis/core/__init__.py diff --git a/qgis/core/geopackage.py b/ribasim_qgis/core/geopackage.py similarity index 100% rename from qgis/core/geopackage.py rename to ribasim_qgis/core/geopackage.py diff --git a/qgis/core/nodes.py b/ribasim_qgis/core/nodes.py similarity index 100% rename from qgis/core/nodes.py rename to ribasim_qgis/core/nodes.py index 7812bd095..28cd7c976 100644 --- a/qgis/core/nodes.py +++ b/ribasim_qgis/core/nodes.py @@ -25,8 +25,6 @@ from PyQt5.QtCore import Qt, QVariant from PyQt5.QtGui import QColor -from ribasim_qgis.core import geopackage - from qgis.core import ( Qgis, QgsCategorizedSymbolRenderer, @@ -43,6 +41,8 @@ QgsVectorLayerSimpleLabeling, ) +from ribasim_qgis.core import geopackage + class Input(abc.ABC): """Abstract base class for Ribasim input layers.""" diff --git a/qgis/core/topology.py b/ribasim_qgis/core/topology.py similarity index 99% rename from qgis/core/topology.py rename to ribasim_qgis/core/topology.py index a7090505a..017658af1 100644 --- a/qgis/core/topology.py +++ b/ribasim_qgis/core/topology.py @@ -1,7 +1,6 @@ from typing import Tuple import numpy as np - from qgis import processing from qgis.core import QgsVectorLayer from qgis.core.additions.edit import edit diff --git a/qgis/icon.png b/ribasim_qgis/icon.png similarity index 100% rename from qgis/icon.png rename to ribasim_qgis/icon.png diff --git a/qgis/metadata.txt b/ribasim_qgis/metadata.txt similarity index 100% rename from qgis/metadata.txt rename to ribasim_qgis/metadata.txt diff --git a/qgis/resources.py b/ribasim_qgis/resources.py similarity index 100% rename from qgis/resources.py rename to ribasim_qgis/resources.py diff --git a/qgis/resources.qrc b/ribasim_qgis/resources.qrc similarity index 100% rename from qgis/resources.qrc rename to ribasim_qgis/resources.qrc diff --git a/qgis/ribasim_qgis.py b/ribasim_qgis/ribasim_qgis.py similarity index 100% rename from qgis/ribasim_qgis.py rename to ribasim_qgis/ribasim_qgis.py diff --git a/qgis/tests/__init__.py b/ribasim_qgis/tests/__init__.py similarity index 100% rename from qgis/tests/__init__.py rename to ribasim_qgis/tests/__init__.py diff --git a/qgis/tests/test_load_plugin.py b/ribasim_qgis/tests/test_load_plugin.py similarity index 100% rename from qgis/tests/test_load_plugin.py rename to ribasim_qgis/tests/test_load_plugin.py diff --git a/ribasim_qgis/tomllib/__init__.py b/ribasim_qgis/tomllib/__init__.py new file mode 100644 index 000000000..d7e0167a4 --- /dev/null +++ b/ribasim_qgis/tomllib/__init__.py @@ -0,0 +1,13 @@ +# Copied from https://github.com/python/cpython/blob/v3.12.0/Lib/tomllib/__init__.py +# QGIS does not guarantee a toml reader + +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +__all__ = ("loads", "load", "TOMLDecodeError") + +from ._parser import TOMLDecodeError, load, loads + +# Pretend this exception was created here. +TOMLDecodeError.__module__ = __name__ diff --git a/ribasim_qgis/tomllib/_parser.py b/ribasim_qgis/tomllib/_parser.py new file mode 100644 index 000000000..7c1eb0dc6 --- /dev/null +++ b/ribasim_qgis/tomllib/_parser.py @@ -0,0 +1,694 @@ +# Copied from https://github.com/python/cpython/blob/v3.12.0/Lib/tomllib/_parser.py +# QGIS does not guarantee a toml reader + +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +from __future__ import annotations + +from collections.abc import Iterable +import string +from types import MappingProxyType +from typing import Any, BinaryIO, NamedTuple + +from ._re import ( + RE_DATETIME, + RE_LOCALTIME, + RE_NUMBER, + match_to_datetime, + match_to_localtime, + match_to_number, +) +from ._types import Key, ParseFloat, Pos + +ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) + +# Neither of these sets include quotation mark or backslash. They are +# currently handled as separate cases in the parser functions. +ILLEGAL_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t") +ILLEGAL_MULTILINE_BASIC_STR_CHARS = ASCII_CTRL - frozenset("\t\n") + +ILLEGAL_LITERAL_STR_CHARS = ILLEGAL_BASIC_STR_CHARS +ILLEGAL_MULTILINE_LITERAL_STR_CHARS = ILLEGAL_MULTILINE_BASIC_STR_CHARS + +ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS + +TOML_WS = frozenset(" \t") +TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") +BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") +HEXDIGIT_CHARS = frozenset(string.hexdigits) + +BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( + { + "\\b": "\u0008", # backspace + "\\t": "\u0009", # tab + "\\n": "\u000A", # linefeed + "\\f": "\u000C", # form feed + "\\r": "\u000D", # carriage return + '\\"': "\u0022", # quote + "\\\\": "\u005C", # backslash + } +) + + +class TOMLDecodeError(ValueError): + """An error raised if a document is not valid TOML.""" + + +def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: + """Parse TOML from a binary file object.""" + b = fp.read() + try: + s = b.decode() + except AttributeError: + raise TypeError( + "File must be opened in binary mode, e.g. use `open('foo.toml', 'rb')`" + ) from None + return loads(s, parse_float=parse_float) + + +def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # noqa: C901 + """Parse TOML from a string.""" + + # The spec allows converting "\r\n" to "\n", even in string + # literals. Let's do so to simplify parsing. + src = s.replace("\r\n", "\n") + pos = 0 + out = Output(NestedDict(), Flags()) + header: Key = () + parse_float = make_safe_parse_float(parse_float) + + # Parse one statement at a time + # (typically means one line in TOML source) + while True: + # 1. Skip line leading whitespace + pos = skip_chars(src, pos, TOML_WS) + + # 2. Parse rules. Expect one of the following: + # - end of file + # - end of line + # - comment + # - key/value pair + # - append dict to list (and move to its namespace) + # - create dict (and move to its namespace) + # Skip trailing whitespace when applicable. + try: + char = src[pos] + except IndexError: + break + if char == "\n": + pos += 1 + continue + if char in KEY_INITIAL_CHARS: + pos = key_value_rule(src, pos, out, header, parse_float) + pos = skip_chars(src, pos, TOML_WS) + elif char == "[": + try: + second_char: str | None = src[pos + 1] + except IndexError: + second_char = None + out.flags.finalize_pending() + if second_char == "[": + pos, header = create_list_rule(src, pos, out) + else: + pos, header = create_dict_rule(src, pos, out) + pos = skip_chars(src, pos, TOML_WS) + elif char != "#": + raise suffixed_err(src, pos, "Invalid statement") + + # 3. Skip comment + pos = skip_comment(src, pos) + + # 4. Expect end of line or end of file + try: + char = src[pos] + except IndexError: + break + if char != "\n": + raise suffixed_err( + src, pos, "Expected newline or end of document after a statement" + ) + pos += 1 + + return out.data.dict + + +class Flags: + """Flags that map to parsed keys/namespaces.""" + + # Marks an immutable namespace (inline array or inline table). + FROZEN = 0 + # Marks a nest that has been explicitly created and can no longer + # be opened using the "[table]" syntax. + EXPLICIT_NEST = 1 + + def __init__(self) -> None: + self._flags: dict[str, dict] = {} + self._pending_flags: set[tuple[Key, int]] = set() + + def add_pending(self, key: Key, flag: int) -> None: + self._pending_flags.add((key, flag)) + + def finalize_pending(self) -> None: + for key, flag in self._pending_flags: + self.set(key, flag, recursive=False) + self._pending_flags.clear() + + def unset_all(self, key: Key) -> None: + cont = self._flags + for k in key[:-1]: + if k not in cont: + return + cont = cont[k]["nested"] + cont.pop(key[-1], None) + + def set(self, key: Key, flag: int, *, recursive: bool) -> None: # noqa: A003 + cont = self._flags + key_parent, key_stem = key[:-1], key[-1] + for k in key_parent: + if k not in cont: + cont[k] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont = cont[k]["nested"] + if key_stem not in cont: + cont[key_stem] = {"flags": set(), "recursive_flags": set(), "nested": {}} + cont[key_stem]["recursive_flags" if recursive else "flags"].add(flag) + + def is_(self, key: Key, flag: int) -> bool: + if not key: + return False # document root has no flags + cont = self._flags + for k in key[:-1]: + if k not in cont: + return False + inner_cont = cont[k] + if flag in inner_cont["recursive_flags"]: + return True + cont = inner_cont["nested"] + key_stem = key[-1] + if key_stem in cont: + cont = cont[key_stem] + return flag in cont["flags"] or flag in cont["recursive_flags"] + return False + + +class NestedDict: + def __init__(self) -> None: + # The parsed content of the TOML document + self.dict: dict[str, Any] = {} + + def get_or_create_nest( + self, + key: Key, + *, + access_lists: bool = True, + ) -> dict: + cont: Any = self.dict + for k in key: + if k not in cont: + cont[k] = {} + cont = cont[k] + if access_lists and isinstance(cont, list): + cont = cont[-1] + if not isinstance(cont, dict): + raise KeyError("There is no nest behind this key") + return cont + + def append_nest_to_list(self, key: Key) -> None: + cont = self.get_or_create_nest(key[:-1]) + last_key = key[-1] + if last_key in cont: + list_ = cont[last_key] + if not isinstance(list_, list): + raise KeyError("An object other than list found behind this key") + list_.append({}) + else: + cont[last_key] = [{}] + + +class Output(NamedTuple): + data: NestedDict + flags: Flags + + +def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: + try: + while src[pos] in chars: + pos += 1 + except IndexError: + pass + return pos + + +def skip_until( + src: str, + pos: Pos, + expect: str, + *, + error_on: frozenset[str], + error_on_eof: bool, +) -> Pos: + try: + new_pos = src.index(expect, pos) + except ValueError: + new_pos = len(src) + if error_on_eof: + raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + + if not error_on.isdisjoint(src[pos:new_pos]): + while src[pos] not in error_on: + pos += 1 + raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + return new_pos + + +def skip_comment(src: str, pos: Pos) -> Pos: + try: + char: str | None = src[pos] + except IndexError: + char = None + if char == "#": + return skip_until( + src, pos + 1, "\n", error_on=ILLEGAL_COMMENT_CHARS, error_on_eof=False + ) + return pos + + +def skip_comments_and_array_ws(src: str, pos: Pos) -> Pos: + while True: + pos_before_skip = pos + pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) + pos = skip_comment(src, pos) + if pos == pos_before_skip: + return pos + + +def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: + pos += 1 # Skip "[" + pos = skip_chars(src, pos, TOML_WS) + pos, key = parse_key(src, pos) + + if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Cannot declare {key} twice") + out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) + try: + out.data.get_or_create_nest(key) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + + if not src.startswith("]", pos): + raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + return pos + 1, key + + +def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: + pos += 2 # Skip "[[" + pos = skip_chars(src, pos, TOML_WS) + pos, key = parse_key(src, pos) + + if out.flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + # Free the namespace now that it points to another empty list item... + out.flags.unset_all(key) + # ...but this key precisely is still prohibited from table declaration + out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) + try: + out.data.append_nest_to_list(key) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + + if not src.startswith("]]", pos): + raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + return pos + 2, key + + +def key_value_rule( + src: str, pos: Pos, out: Output, header: Key, parse_float: ParseFloat +) -> Pos: + pos, key, value = parse_key_value_pair(src, pos, parse_float) + key_parent, key_stem = key[:-1], key[-1] + abs_key_parent = header + key_parent + + relative_path_cont_keys = (header + key[:i] for i in range(1, len(key))) + for cont_key in relative_path_cont_keys: + # Check that dotted key syntax does not redefine an existing table + if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): + raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + # Containers in the relative path can't be opened with the table syntax or + # dotted key/value syntax in following table sections. + out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) + + if out.flags.is_(abs_key_parent, Flags.FROZEN): + raise suffixed_err( + src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + ) + + try: + nest = out.data.get_or_create_nest(abs_key_parent) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + if key_stem in nest: + raise suffixed_err(src, pos, "Cannot overwrite a value") + # Mark inline table and array namespaces recursively immutable + if isinstance(value, (dict, list)): + out.flags.set(header + key, Flags.FROZEN, recursive=True) + nest[key_stem] = value + return pos + + +def parse_key_value_pair( + src: str, pos: Pos, parse_float: ParseFloat +) -> tuple[Pos, Key, Any]: + pos, key = parse_key(src, pos) + try: + char: str | None = src[pos] + except IndexError: + char = None + if char != "=": + raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + pos, value = parse_value(src, pos, parse_float) + return pos, key, value + + +def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: + pos, key_part = parse_key_part(src, pos) + key: Key = (key_part,) + pos = skip_chars(src, pos, TOML_WS) + while True: + try: + char: str | None = src[pos] + except IndexError: + char = None + if char != ".": + return pos, key + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + pos, key_part = parse_key_part(src, pos) + key += (key_part,) + pos = skip_chars(src, pos, TOML_WS) + + +def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: + try: + char: str | None = src[pos] + except IndexError: + char = None + if char in BARE_KEY_CHARS: + start_pos = pos + pos = skip_chars(src, pos, BARE_KEY_CHARS) + return pos, src[start_pos:pos] + if char == "'": + return parse_literal_str(src, pos) + if char == '"': + return parse_one_line_basic_str(src, pos) + raise suffixed_err(src, pos, "Invalid initial character for a key part") + + +def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: + pos += 1 + return parse_basic_str(src, pos, multiline=False) + + +def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]: + pos += 1 + array: list = [] + + pos = skip_comments_and_array_ws(src, pos) + if src.startswith("]", pos): + return pos + 1, array + while True: + pos, val = parse_value(src, pos, parse_float) + array.append(val) + pos = skip_comments_and_array_ws(src, pos) + + c = src[pos : pos + 1] + if c == "]": + return pos + 1, array + if c != ",": + raise suffixed_err(src, pos, "Unclosed array") + pos += 1 + + pos = skip_comments_and_array_ws(src, pos) + if src.startswith("]", pos): + return pos + 1, array + + +def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]: + pos += 1 + nested_dict = NestedDict() + flags = Flags() + + pos = skip_chars(src, pos, TOML_WS) + if src.startswith("}", pos): + return pos + 1, nested_dict.dict + while True: + pos, key, value = parse_key_value_pair(src, pos, parse_float) + key_parent, key_stem = key[:-1], key[-1] + if flags.is_(key, Flags.FROZEN): + raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + try: + nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) + except KeyError: + raise suffixed_err(src, pos, "Cannot overwrite a value") from None + if key_stem in nest: + raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + nest[key_stem] = value + pos = skip_chars(src, pos, TOML_WS) + c = src[pos : pos + 1] + if c == "}": + return pos + 1, nested_dict.dict + if c != ",": + raise suffixed_err(src, pos, "Unclosed inline table") + if isinstance(value, (dict, list)): + flags.set(key, Flags.FROZEN, recursive=True) + pos += 1 + pos = skip_chars(src, pos, TOML_WS) + + +def parse_basic_str_escape( + src: str, pos: Pos, *, multiline: bool = False +) -> tuple[Pos, str]: + escape_id = src[pos : pos + 2] + pos += 2 + if multiline and escape_id in {"\\ ", "\\\t", "\\\n"}: + # Skip whitespace until next non-whitespace character or end of + # the doc. Error if non-whitespace is found before newline. + if escape_id != "\\\n": + pos = skip_chars(src, pos, TOML_WS) + try: + char = src[pos] + except IndexError: + return pos, "" + if char != "\n": + raise suffixed_err(src, pos, "Unescaped '\\' in a string") + pos += 1 + pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) + return pos, "" + if escape_id == "\\u": + return parse_hex_char(src, pos, 4) + if escape_id == "\\U": + return parse_hex_char(src, pos, 8) + try: + return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] + except KeyError: + raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + + +def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: + return parse_basic_str_escape(src, pos, multiline=True) + + +def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: + hex_str = src[pos : pos + hex_len] + if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): + raise suffixed_err(src, pos, "Invalid hex value") + pos += hex_len + hex_int = int(hex_str, 16) + if not is_unicode_scalar_value(hex_int): + raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + return pos, chr(hex_int) + + +def parse_literal_str(src: str, pos: Pos) -> tuple[Pos, str]: + pos += 1 # Skip starting apostrophe + start_pos = pos + pos = skip_until( + src, pos, "'", error_on=ILLEGAL_LITERAL_STR_CHARS, error_on_eof=True + ) + return pos + 1, src[start_pos:pos] # Skip ending apostrophe + + +def parse_multiline_str(src: str, pos: Pos, *, literal: bool) -> tuple[Pos, str]: + pos += 3 + if src.startswith("\n", pos): + pos += 1 + + if literal: + delim = "'" + end_pos = skip_until( + src, + pos, + "'''", + error_on=ILLEGAL_MULTILINE_LITERAL_STR_CHARS, + error_on_eof=True, + ) + result = src[pos:end_pos] + pos = end_pos + 3 + else: + delim = '"' + pos, result = parse_basic_str(src, pos, multiline=True) + + # Add at maximum two extra apostrophes/quotes if the end sequence + # is 4 or 5 chars long instead of just 3. + if not src.startswith(delim, pos): + return pos, result + pos += 1 + if not src.startswith(delim, pos): + return pos, result + delim + pos += 1 + return pos, result + (delim * 2) + + +def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: + if multiline: + error_on = ILLEGAL_MULTILINE_BASIC_STR_CHARS + parse_escapes = parse_basic_str_escape_multiline + else: + error_on = ILLEGAL_BASIC_STR_CHARS + parse_escapes = parse_basic_str_escape + result = "" + start_pos = pos + while True: + try: + char = src[pos] + except IndexError: + raise suffixed_err(src, pos, "Unterminated string") from None + if char == '"': + if not multiline: + return pos + 1, result + src[start_pos:pos] + if src.startswith('"""', pos): + return pos + 3, result + src[start_pos:pos] + pos += 1 + continue + if char == "\\": + result += src[start_pos:pos] + pos, parsed_escape = parse_escapes(src, pos) + result += parsed_escape + start_pos = pos + continue + if char in error_on: + raise suffixed_err(src, pos, f"Illegal character {char!r}") + pos += 1 + + +def parse_value( # noqa: C901 + src: str, pos: Pos, parse_float: ParseFloat +) -> tuple[Pos, Any]: + try: + char: str | None = src[pos] + except IndexError: + char = None + + # IMPORTANT: order conditions based on speed of checking and likelihood + + # Basic strings + if char == '"': + if src.startswith('"""', pos): + return parse_multiline_str(src, pos, literal=False) + return parse_one_line_basic_str(src, pos) + + # Literal strings + if char == "'": + if src.startswith("'''", pos): + return parse_multiline_str(src, pos, literal=True) + return parse_literal_str(src, pos) + + # Booleans + if char == "t": + if src.startswith("true", pos): + return pos + 4, True + if char == "f": + if src.startswith("false", pos): + return pos + 5, False + + # Arrays + if char == "[": + return parse_array(src, pos, parse_float) + + # Inline tables + if char == "{": + return parse_inline_table(src, pos, parse_float) + + # Dates and times + datetime_match = RE_DATETIME.match(src, pos) + if datetime_match: + try: + datetime_obj = match_to_datetime(datetime_match) + except ValueError as e: + raise suffixed_err(src, pos, "Invalid date or datetime") from e + return datetime_match.end(), datetime_obj + localtime_match = RE_LOCALTIME.match(src, pos) + if localtime_match: + return localtime_match.end(), match_to_localtime(localtime_match) + + # Integers and "normal" floats. + # The regex will greedily match any type starting with a decimal + # char, so needs to be located after handling of dates and times. + number_match = RE_NUMBER.match(src, pos) + if number_match: + return number_match.end(), match_to_number(number_match, parse_float) + + # Special floats + first_three = src[pos : pos + 3] + if first_three in {"inf", "nan"}: + return pos + 3, parse_float(first_three) + first_four = src[pos : pos + 4] + if first_four in {"-inf", "+inf", "-nan", "+nan"}: + return pos + 4, parse_float(first_four) + + raise suffixed_err(src, pos, "Invalid value") + + +def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: + """Return a `TOMLDecodeError` where error message is suffixed with + coordinates in source.""" + + def coord_repr(src: str, pos: Pos) -> str: + if pos >= len(src): + return "end of document" + line = src.count("\n", 0, pos) + 1 + if line == 1: + column = pos + 1 + else: + column = pos - src.rindex("\n", 0, pos) + return f"line {line}, column {column}" + + return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + + +def is_unicode_scalar_value(codepoint: int) -> bool: + return (0 <= codepoint <= 55295) or (57344 <= codepoint <= 1114111) + + +def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat: + """A decorator to make `parse_float` safe. + + `parse_float` must not return dicts or lists, because these types + would be mixed with parsed TOML tables and arrays, thus confusing + the parser. The returned decorated callable raises `ValueError` + instead of returning illegal types. + """ + # The default `float` callable never returns illegal types. Optimize it. + if parse_float is float: # type: ignore[comparison-overlap] + return float + + def safe_parse_float(float_str: str) -> Any: + float_value = parse_float(float_str) + if isinstance(float_value, (dict, list)): + raise ValueError("parse_float must not return dicts or lists") + return float_value + + return safe_parse_float diff --git a/ribasim_qgis/tomllib/_re.py b/ribasim_qgis/tomllib/_re.py new file mode 100644 index 000000000..b48d89c72 --- /dev/null +++ b/ribasim_qgis/tomllib/_re.py @@ -0,0 +1,110 @@ +# Copied from https://github.com/python/cpython/blob/v3.12.0/Lib/tomllib/_re.py +# QGIS does not guarantee a toml reader + +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +from __future__ import annotations + +from datetime import date, datetime, time, timedelta, timezone, tzinfo +from functools import lru_cache +import re +from typing import Any + +from ._types import ParseFloat + +# E.g. +# - 00:32:00.999999 +# - 00:32:00 +_TIME_RE_STR = r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])(?:\.([0-9]{1,6})[0-9]*)?" + +RE_NUMBER = re.compile( + r""" +0 +(?: + x[0-9A-Fa-f](?:_?[0-9A-Fa-f])* # hex + | + b[01](?:_?[01])* # bin + | + o[0-7](?:_?[0-7])* # oct +) +| +[+-]?(?:0|[1-9](?:_?[0-9])*) # dec, integer part +(?P + (?:\.[0-9](?:_?[0-9])*)? # optional fractional part + (?:[eE][+-]?[0-9](?:_?[0-9])*)? # optional exponent part +) +""", + flags=re.VERBOSE, +) +RE_LOCALTIME = re.compile(_TIME_RE_STR) +RE_DATETIME = re.compile( + rf""" +([0-9]{{4}})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01]) # date, e.g. 1988-10-27 +(?: + [Tt ] + {_TIME_RE_STR} + (?:([Zz])|([+-])([01][0-9]|2[0-3]):([0-5][0-9]))? # optional time offset +)? +""", + flags=re.VERBOSE, +) + + +def match_to_datetime(match: re.Match) -> datetime | date: + """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`. + + Raises ValueError if the match does not correspond to a valid date + or datetime. + """ + ( + year_str, + month_str, + day_str, + hour_str, + minute_str, + sec_str, + micros_str, + zulu_time, + offset_sign_str, + offset_hour_str, + offset_minute_str, + ) = match.groups() + year, month, day = int(year_str), int(month_str), int(day_str) + if hour_str is None: + return date(year, month, day) + hour, minute, sec = int(hour_str), int(minute_str), int(sec_str) + micros = int(micros_str.ljust(6, "0")) if micros_str else 0 + if offset_sign_str: + tz: tzinfo | None = cached_tz( + offset_hour_str, offset_minute_str, offset_sign_str + ) + elif zulu_time: + tz = timezone.utc + else: # local date-time + tz = None + return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) + + +@lru_cache(maxsize=None) +def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: + sign = 1 if sign_str == "+" else -1 + return timezone( + timedelta( + hours=sign * int(hour_str), + minutes=sign * int(minute_str), + ) + ) + + +def match_to_localtime(match: re.Match) -> time: + hour_str, minute_str, sec_str, micros_str = match.groups() + micros = int(micros_str.ljust(6, "0")) if micros_str else 0 + return time(int(hour_str), int(minute_str), int(sec_str), micros) + + +def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any: + if match.group("floatpart"): + return parse_float(match.group()) + return int(match.group(), 0) diff --git a/ribasim_qgis/tomllib/_types.py b/ribasim_qgis/tomllib/_types.py new file mode 100644 index 000000000..775a153c1 --- /dev/null +++ b/ribasim_qgis/tomllib/_types.py @@ -0,0 +1,13 @@ +# Copied from https://github.com/python/cpython/blob/v3.12.0/Lib/tomllib/_types.py +# QGIS does not guarantee a toml reader + +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021 Taneli Hukkinen +# Licensed to PSF under a Contributor Agreement. + +from typing import Any, Callable, Tuple + +# Type annotations +ParseFloat = Callable[[str], Any] +Key = Tuple[str, ...] +Pos = int diff --git a/qgis/widgets/dataset_widget.py b/ribasim_qgis/widgets/dataset_widget.py similarity index 86% rename from qgis/widgets/dataset_widget.py rename to ribasim_qgis/widgets/dataset_widget.py index a32096862..0d1a32b67 100644 --- a/qgis/widgets/dataset_widget.py +++ b/ribasim_qgis/widgets/dataset_widget.py @@ -4,6 +4,7 @@ This widget also allows enabling or disabling individual elements for a computation. """ +from datetime import datetime from pathlib import Path from typing import Any, List, Set @@ -24,12 +25,13 @@ QVBoxLayout, QWidget, ) -from ribasim_qgis.core.nodes import Edge, Node, load_nodes_from_geopackage -from ribasim_qgis.core.topology import derive_connectivity, explode_lines - from qgis.core import QgsMapLayer, QgsProject from qgis.core.additions.edit import edit +import ribasim_qgis.tomllib as tomllib +from ribasim_qgis.core.nodes import Edge, Node, load_nodes_from_geopackage +from ribasim_qgis.core.topology import derive_connectivity, explode_lines + class DatasetTreeWidget(QTreeWidget): def __init__(self, parent=None): @@ -129,12 +131,12 @@ def __init__(self, parent): self.dataset_tree.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding) self.dataset_line_edit = QLineEdit() self.dataset_line_edit.setEnabled(False) # Just used as a viewing port - self.new_geopackage_button = QPushButton("New") - self.open_geopackage_button = QPushButton("Open") + self.new_model_button = QPushButton("New") + self.open_model_button = QPushButton("Open") self.remove_button = QPushButton("Remove from Dataset") self.add_button = QPushButton("Add to QGIS") - self.new_geopackage_button.clicked.connect(self.new_geopackage) - self.open_geopackage_button.clicked.connect(self.open_geopackage) + self.new_model_button.clicked.connect(self.new_model) + self.open_model_button.clicked.connect(self.open_model) self.suppress_popup_checkbox = QCheckBox("Suppress attribute form pop-up") self.suppress_popup_checkbox.stateChanged.connect(self.suppress_popup_changed) self.remove_button.clicked.connect(self.remove_geopackage_layer) @@ -146,8 +148,8 @@ def __init__(self, parent): dataset_row = QHBoxLayout() layer_row = QHBoxLayout() dataset_row.addWidget(self.dataset_line_edit) - dataset_row.addWidget(self.open_geopackage_button) - dataset_row.addWidget(self.new_geopackage_button) + dataset_row.addWidget(self.open_model_button) + dataset_row.addWidget(self.new_model_button) dataset_layout.addLayout(dataset_row) dataset_layout.addWidget(self.dataset_tree) dataset_layout.addWidget(self.suppress_popup_checkbox) @@ -158,7 +160,7 @@ def __init__(self, parent): @property def path(self) -> str: - """Returns currently active path to GeoPackage""" + """Returns currently active path to Ribasim model (.toml)""" return self.dataset_line_edit.text() def explode_and_connect(self) -> None: @@ -241,7 +243,8 @@ def add_selection_to_qgis(self) -> None: def load_geopackage(self) -> None: """Load the layers of a GeoPackage into the Layers Panel""" self.dataset_tree.clear() - nodes = load_nodes_from_geopackage(self.path) + geo_path = self._get_database_path_from_model_file() + nodes = load_nodes_from_geopackage(geo_path) for node_layer in nodes.values(): self.dataset_tree.add_node_layer(node_layer) name = str(Path(self.path).stem) @@ -255,21 +258,38 @@ def load_geopackage(self) -> None: self.edge_layer.editingStopped.connect(self.explode_and_connect) return - def new_geopackage(self) -> None: - """Create a new GeoPackage file, and set it as the active dataset.""" - path, _ = QFileDialog.getSaveFileName(self, "Select file", "", "*.gpkg") + def _get_database_path_from_model_file(self) -> str: + with open(self.path, "rb") as f: + model_filename = tomllib.load(f)["database"] + return str(Path(self.path).parent.joinpath(model_filename)) + + def new_model(self) -> None: + """Create a new Ribasim model file, and set it as the active dataset.""" + path, _ = QFileDialog.getSaveFileName(self, "Select file", "", "*.toml") if path != "": # Empty string in case of cancel button press self.dataset_line_edit.setText(path) + geo_path = Path(self.path).parent.joinpath("database.gpkg") + self._write_new_model(geo_path.name) for input_type in (Node, Edge): - instance = input_type.create(path, self.parent.crs, names=[]) + instance = input_type.create(str(geo_path), self.parent.crs, names=[]) instance.write() self.load_geopackage() self.parent.toggle_node_buttons(True) - def open_geopackage(self) -> None: - """Open a GeoPackage file, containing Ribasim input.""" + def _write_new_model(self, database_name: str) -> None: + with open(self.path, "w") as f: + f.writelines( + [ + f'database = "{database_name}"\n', + f"starttime = {datetime(2020, 1, 1)}\n", + f"endtime = {datetime(2030, 1, 1)}\n", + ] + ) + + def open_model(self) -> None: + """Open a Ribasim model file.""" self.dataset_tree.clear() - path, _ = QFileDialog.getOpenFileName(self, "Select file", "", "*.gpkg") + path, _ = QFileDialog.getOpenFileName(self, "Select file", "", "*.toml") if path != "": # Empty string in case of cancel button press self.dataset_line_edit.setText(path) self.load_geopackage() diff --git a/qgis/widgets/nodes_widget.py b/ribasim_qgis/widgets/nodes_widget.py similarity index 99% rename from qgis/widgets/nodes_widget.py rename to ribasim_qgis/widgets/nodes_widget.py index 0d09d277c..bcce08be7 100644 --- a/qgis/widgets/nodes_widget.py +++ b/ribasim_qgis/widgets/nodes_widget.py @@ -1,6 +1,7 @@ from functools import partial from PyQt5.QtWidgets import QGridLayout, QPushButton, QVBoxLayout, QWidget + from ribasim_qgis.core.nodes import NODES diff --git a/qgis/widgets/results_widget.py b/ribasim_qgis/widgets/results_widget.py similarity index 100% rename from qgis/widgets/results_widget.py rename to ribasim_qgis/widgets/results_widget.py diff --git a/qgis/widgets/ribasim_widget.py b/ribasim_qgis/widgets/ribasim_widget.py similarity index 98% rename from qgis/widgets/ribasim_widget.py rename to ribasim_qgis/widgets/ribasim_widget.py index 6d3303952..aea0e364c 100644 --- a/qgis/widgets/ribasim_widget.py +++ b/ribasim_qgis/widgets/ribasim_widget.py @@ -9,12 +9,12 @@ from typing import Any from PyQt5.QtWidgets import QTabWidget, QVBoxLayout, QWidget +from qgis.core import QgsEditFormConfig, QgsMapLayer, QgsProject + from ribasim_qgis.widgets.dataset_widget import DatasetWidget from ribasim_qgis.widgets.nodes_widget import NodesWidget from ribasim_qgis.widgets.results_widget import ResultsWidget -from qgis.core import QgsEditFormConfig, QgsMapLayer, QgsProject - PYQT_DELETED_ERROR = "wrapped C/C++ object of type QgsLayerTreeGroup has been deleted" @@ -33,7 +33,7 @@ def __init__(self, parent, iface): self.layout = QVBoxLayout() self.tabwidget = QTabWidget() self.layout.addWidget(self.tabwidget) - self.tabwidget.addTab(self.dataset_widget, "Database") + self.tabwidget.addTab(self.dataset_widget, "Model") self.tabwidget.addTab(self.nodes_widget, "Nodes") self.tabwidget.addTab(self.results_widget, "Results") self.setLayout(self.layout) diff --git a/ruff.toml b/ruff.toml index 0f7924e99..0f3a6e69b 100644 --- a/ruff.toml +++ b/ruff.toml @@ -14,6 +14,9 @@ ignore = [ ] fixable = ["I"] extend-include = ["*.ipynb"] +exclude = [ + "ribasim_qgis/tomllib/*" +] [pydocstyle] convention = "numpy"