From b1470cd5542797090b58f8b292b7d5320b80abcb Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Mon, 26 Aug 2024 19:40:05 +0200 Subject: [PATCH 01/30] connectivity validation function implemented --- python/ribasim/ribasim/validation.py | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 python/ribasim/ribasim/validation.py diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py new file mode 100644 index 000000000..6b4dff6b5 --- /dev/null +++ b/python/ribasim/ribasim/validation.py @@ -0,0 +1,40 @@ +# Table for connectivity +# "basin": ["linear_resistance"] means that the upstream of basin can be linear_resistance only +connectivity = { + "basin": [ + "linear_resistance", + "manning_resistance", + "tabulated_rating_curve", + "pump", + "outlet", + "user_demand", + ], + "linear_resistance": ["basin", "level_boundary"], + "manning_resistance": ["basin"], + "tabulated_rating_curve": ["basin", "terminal", "level_boundary"], + "level_boundary": ["linear_resistance", "pump", "outlet", "tabulated_rating_curve"], + "flow_boundary": ["basin", "terminal", "level_boundary"], + "pump": ["basin", "terminal", "level_boundary"], + "outlet": ["basin", "terminal", "level_boundary"], + "terminal": [], + "discrete_control": [ + "pump", + "outlet", + "tabulated_rating_curve", + "linear_resistance", + "manning_resistance", + "pid_control", + ], + "continuous_control": ["pump", "outlet"], + "pid_control": ["pump", "outlet"], + "user_demand": ["basin", "terminal", "level_boundary"], + "level_demand": ["basin"], + "flow_demand": ["basin", "terminal", "level_boundary"], +} + + +# Function to validate connection +def can_connect(node_down, node_up): + if node_down in connectivity: + return node_up in connectivity[node_down] + return False From a3ec21fe82ba4d28baaa174c242b43f1641cd568 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Mon, 26 Aug 2024 19:55:46 +0200 Subject: [PATCH 02/30] raise error if when adding edge --- python/ribasim/ribasim/geometry/edge.py | 6 ++++++ python/ribasim/ribasim/validation.py | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index ca4468741..a68b6ff8a 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -16,6 +16,7 @@ from ribasim.input_base import SpatialTableModel from ribasim.schemas import _BaseSchema from ribasim.utils import UsedIDs +from ribasim.validation import connectivity __all__ = ("EdgeTable",) @@ -81,6 +82,11 @@ def add( the allocation algorithm, and should thus not be set for every edge in a subnetwork. **kwargs : Dict """ + if not connectivity(from_node.node_type, to_node.node_type): + raise ValueError( + f"Node {to_node.node_type} cannot connect with upstream node {from_node.node_type}" + ) + geometry_to_append = ( [LineString([from_node.geometry, to_node.geometry])] if geometry is None diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 6b4dff6b5..6262884c8 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -1,5 +1,5 @@ # Table for connectivity -# "basin": ["linear_resistance"] means that the upstream of basin can be linear_resistance only +# "basin": ["linear_resistance"] means that the downstream of basin can be linear_resistance only connectivity = { "basin": [ "linear_resistance", @@ -34,7 +34,7 @@ # Function to validate connection -def can_connect(node_down, node_up): - if node_down in connectivity: - return node_up in connectivity[node_down] +def can_connect(node_up, node_down): + if node_up in connectivity: + return node_down in connectivity[node_up] return False From d71736abd9ee8298a7e71ea593f268175f87dc4c Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 27 Aug 2024 09:49:13 +0200 Subject: [PATCH 03/30] fix build error --- python/ribasim/ribasim/geometry/edge.py | 4 ++-- python/ribasim/ribasim/validation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index a68b6ff8a..c475ce85b 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -16,7 +16,7 @@ from ribasim.input_base import SpatialTableModel from ribasim.schemas import _BaseSchema from ribasim.utils import UsedIDs -from ribasim.validation import connectivity +from ribasim.validation import can_connect __all__ = ("EdgeTable",) @@ -82,7 +82,7 @@ def add( the allocation algorithm, and should thus not be set for every edge in a subnetwork. **kwargs : Dict """ - if not connectivity(from_node.node_type, to_node.node_type): + if not can_connect(from_node.node_type, to_node.node_type): raise ValueError( f"Node {to_node.node_type} cannot connect with upstream node {from_node.node_type}" ) diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 6262884c8..47ee14025 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -34,7 +34,7 @@ # Function to validate connection -def can_connect(node_up, node_down): +def can_connect(node_up: str, node_down: str) -> bool: if node_up in connectivity: return node_down in connectivity[node_up] return False From 8618950710454dbd539fd102fa0174bf93940217 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 27 Aug 2024 10:05:17 +0200 Subject: [PATCH 04/30] fix pytest error --- python/ribasim/ribasim/validation.py | 54 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 47ee14025..00ade729e 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -1,35 +1,35 @@ # Table for connectivity # "basin": ["linear_resistance"] means that the downstream of basin can be linear_resistance only connectivity = { - "basin": [ - "linear_resistance", - "manning_resistance", - "tabulated_rating_curve", - "pump", - "outlet", - "user_demand", + "Basin": [ + "LinearResistance", + "ManningResistance", + "TabulatedRatingCurve", + "Pump", + "Outlet", + "UserDdemand", ], - "linear_resistance": ["basin", "level_boundary"], - "manning_resistance": ["basin"], - "tabulated_rating_curve": ["basin", "terminal", "level_boundary"], - "level_boundary": ["linear_resistance", "pump", "outlet", "tabulated_rating_curve"], - "flow_boundary": ["basin", "terminal", "level_boundary"], - "pump": ["basin", "terminal", "level_boundary"], - "outlet": ["basin", "terminal", "level_boundary"], - "terminal": [], - "discrete_control": [ - "pump", - "outlet", - "tabulated_rating_curve", - "linear_resistance", - "manning_resistance", - "pid_control", + "LinearResistance": ["Basin", "LevelBoundary"], + "ManningResistance": ["Basin"], + "TabulatedRatingCurve": ["Basin", "Terminal", "LevelBoundary"], + "LevelBoundary": ["LinearResistance", "Pump", "Outlet", "TabulatedRatingCurve"], + "FlowBoundary": ["Basin", "Terminal", "LevelBoundary"], + "Pump": ["Basin", "Terminal", "LevelBoundary"], + "Outlet": ["Basin", "Terminal", "LevelBoundary"], + "Terminal": [], + "DiscreteControl": [ + "Pump", + "Outlet", + "TabulatedRatingCurve", + "LinearResistance", + "ManningResistance", + "PidControl", ], - "continuous_control": ["pump", "outlet"], - "pid_control": ["pump", "outlet"], - "user_demand": ["basin", "terminal", "level_boundary"], - "level_demand": ["basin"], - "flow_demand": ["basin", "terminal", "level_boundary"], + "ContinuousControl": ["Pump", "Outlet"], + "PidControl": ["Pump", "Outlet"], + "UserDemand": ["Basin", "Terminal", "LevelBoundary"], + "LevelDemand": ["Basin"], + "FLowDemand": ["Basin", "Terminal", "LevelBoundary"], } From fed110257c5c389f159c0eb9a7bc0c0f9f6de297 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 27 Aug 2024 10:09:09 +0200 Subject: [PATCH 05/30] fix pytest error 2 --- python/ribasim/ribasim/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 00ade729e..338df9388 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -7,7 +7,7 @@ "TabulatedRatingCurve", "Pump", "Outlet", - "UserDdemand", + "UserDemand", ], "LinearResistance": ["Basin", "LevelBoundary"], "ManningResistance": ["Basin"], From e408fd706ab79dbf9b79d3e7bcd00a032ee06221 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 27 Aug 2024 10:15:51 +0200 Subject: [PATCH 06/30] fix build error 2 --- python/ribasim/ribasim/validation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 338df9388..be771bf5b 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -29,7 +29,13 @@ "PidControl": ["Pump", "Outlet"], "UserDemand": ["Basin", "Terminal", "LevelBoundary"], "LevelDemand": ["Basin"], - "FLowDemand": ["Basin", "Terminal", "LevelBoundary"], + "FLowDemand": [ + "LinearResistance", + "ManningResistance", + "TabulatedRatingCurve", + "Pump", + "Outlet", + ], } From 979b993fdcf43013da3ed8eedadba130af0dba97 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 27 Aug 2024 10:25:39 +0200 Subject: [PATCH 07/30] fix build error 3 --- python/ribasim/ribasim/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index be771bf5b..0be769b35 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -29,7 +29,7 @@ "PidControl": ["Pump", "Outlet"], "UserDemand": ["Basin", "Terminal", "LevelBoundary"], "LevelDemand": ["Basin"], - "FLowDemand": [ + "FlowDemand": [ "LinearResistance", "ManningResistance", "TabulatedRatingCurve", From d4c97e6eebad08ddbb7ca55c01690f6bb5c0fe79 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 27 Aug 2024 11:10:01 +0200 Subject: [PATCH 08/30] add unit test for validation --- python/ribasim/tests/test_model.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 81e6800cd..95951883b 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -156,6 +156,15 @@ def test_duplicate_edge(trivial): ) +def test_connectivity(trivial): + model = trivial + with pytest.raises( + ValueError, + match=re.escape("Node Terminal cannot connect with upstream node Basin"), + ): + model.edge.add(model.basin[6], model.terminal[2147483647]) + + def test_indexing(basic): model = basic From a0e3214574ce712fde45fc4457f2587a070d5720 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Wed, 28 Aug 2024 08:49:44 +0200 Subject: [PATCH 09/30] adding neighbor limit --- python/ribasim/ribasim/geometry/edge.py | 21 ++++++++++++++++++++- python/ribasim/ribasim/validation.py | 25 +++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index c475ce85b..fcbf7a78c 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -16,7 +16,7 @@ from ribasim.input_base import SpatialTableModel from ribasim.schemas import _BaseSchema from ribasim.utils import UsedIDs -from ribasim.validation import can_connect +from ribasim.validation import can_connect, flow_edge_amount __all__ = ("EdgeTable",) @@ -117,6 +117,25 @@ def add( index=pd.Index([edge_id], name="edge_id"), ) + upstream_flow_neighbor = self.df.loc[ + (self.df["to_node_id"] == to_node.node_id) + & (self.df["edge_type"] == "flow") + ].count() + downstream_flow_neighbor = self.df.loc[ + (self.df["from_node_id"] == from_node.node_id) + & (self.df["edge_type"] == "flow") + ].count() + # validation on neighbor amount + if ( + upstream_flow_neighbor >= flow_edge_amount[to_node.node_type][0] + ): # too many upstream neighbor, flow edge + raise ValueError + if ( + downstream_flow_neighbor >= flow_edge_amount[to_node.node_type][2] + ): # too many downstream neighbor, flow edge + raise ValueError + # if #too many + self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): raise ValueError( diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 0be769b35..d028c5ba6 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -1,6 +1,8 @@ +import numpy as np + # Table for connectivity -# "basin": ["linear_resistance"] means that the downstream of basin can be linear_resistance only -connectivity = { +# "Basin": ["LinearResistance"] means that the downstream of basin can be LinearResistance only +connectivity: dict[str, list[str]] = { "Basin": [ "LinearResistance", "ManningResistance", @@ -44,3 +46,22 @@ def can_connect(node_up: str, node_down: str) -> bool: if node_up in connectivity: return node_down in connectivity[node_up] return False + + +flow_edge_amount: dict[str, list[int]] = { + "Basin": [0, np.inf, 0, np.inf], + "LinearResistance": [1, 1, 1, 1], + "ManningResistance": [1, 1, 1, 1], + "TabulatedRatingCurve": [1, 1, 1, np.inf], + "LevelBoundary": [0, np.inf, 0, np.inf], + "FlowBoundary": [0, 0, 1, np.inf], + "Pump": [1, 1, 1, np.inf], + "Outlet": [1, 1, 1, 1], + "Terminal": [1, np.inf, 0, 0], + "DiscreteControl": [0, 0, 0, 0], + "ContinuousControl": [0, 0, 0, 0], + "PidControl": [0, 0, 0, 0], + "UserDemand": [1, 1, 1, 1], + "LevelDemand": [0, 0, 0, 0], + "FlowDemand": [0, 0, 0, 0], +} From df50143e03b2d4cbc786500a45903eaf3857e9b0 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Wed, 28 Aug 2024 16:04:23 +0200 Subject: [PATCH 10/30] check minimum for flow edge --- python/ribasim/ribasim/geometry/edge.py | 16 +++---- python/ribasim/ribasim/model.py | 57 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index fcbf7a78c..b341de435 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -84,7 +84,7 @@ def add( """ if not can_connect(from_node.node_type, to_node.node_type): raise ValueError( - f"Node {to_node.node_type} cannot connect with upstream node {from_node.node_type}" + f"Node of type {from_node.node_type} cannot be upstream of node of type {to_node.node_type}" ) geometry_to_append = ( @@ -117,24 +117,24 @@ def add( index=pd.Index([edge_id], name="edge_id"), ) - upstream_flow_neighbor = self.df.loc[ + in_flow_neighbor: int = self.df.loc[ (self.df["to_node_id"] == to_node.node_id) & (self.df["edge_type"] == "flow") - ].count() - downstream_flow_neighbor = self.df.loc[ + ].shape[0] + + out_flow_neighbor: int = self.df.loc[ (self.df["from_node_id"] == from_node.node_id) & (self.df["edge_type"] == "flow") - ].count() + ].shape[0] # validation on neighbor amount if ( - upstream_flow_neighbor >= flow_edge_amount[to_node.node_type][0] + in_flow_neighbor >= flow_edge_amount[to_node.node_type][1] ): # too many upstream neighbor, flow edge raise ValueError if ( - downstream_flow_neighbor >= flow_edge_amount[to_node.node_type][2] + out_flow_neighbor >= flow_edge_amount[to_node.node_type][3] ): # too many downstream neighbor, flow edge raise ValueError - # if #too many self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 11c50da96..e076529f8 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -58,6 +58,7 @@ _node_lookup_numpy, _time_in_ns, ) +from ribasim.validation import flow_edge_amount try: import xugrid @@ -280,6 +281,62 @@ def write(self, filepath: str | PathLike[str]) -> Path: context_file_writing.set({}) return fn + def validate_model(self): + df_edge = self.edge.df + df_chunks = [node.node.df.set_crs(self.crs) for node in self._nodes()] # type:ignore + df_node = pd.concat(df_chunks) + + df_graph = df_edge + # Join df_edge with df_node to get to_node_type + df_graph = df_graph.join( + df_node[["node_type"]], on="from_node_id", how="left", rsuffix="_from" + ) + df_graph = df_graph.rename(columns={"node_type": "from_node_type"}) + + df_graph = df_graph.join( + df_node[["node_type"]], on="to_node_id", how="left", rsuffix="_to" + ) + df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) + flow_edge_amount + self._check_neighbors() + + def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount) -> bool: + is_valid = True + # Count flow edge neighbor + df_graph_flow = df_graph.loc[df_graph["edge_type"] == "flow"] + + # check from node's neighbor + from_node_count = ( + df_graph_flow.groupby("from_node_id") + .size() + .reset_index(name="from_node_count") + ) + df_result = ( + df_graph_flow[["from_node_id", "from_node_type"]] + .drop_duplicates() + .merge(from_node_count, on="from_node_id", how="left") + ) + df_result = df_result[["from_node_id", "from_node_count", "from_node_type"]] + for _, row in df_result.iterrows(): + # from node's outneighbor + if row["from_node_count"] < flow_edge_amount[row["from_node_type"][2]]: + is_valid = False + + # check to node's neighbor + to_node_count = ( + df_graph_flow.groupby("to_node_id").size().reset_index(name="to_node_count") + ) + df_result = ( + df_graph_flow[["to_node_id", "to_node_type"]] + .drop_duplicates() + .merge(to_node_count, on="to_node_id", how="left") + ) + df_result = df_result[["to_node_id", "to_node_count", "to_node_type"]] + for _, row in df_result.iterrows(): + if row["to_node_count"] < flow_edge_amount[row["to_node_type"][0]]: + is_valid = False + return is_valid + @classmethod def _load(cls, filepath: Path | None) -> dict[str, Any]: context_file_loading.set({}) From f326e39b882d283373ed84c7eb3610f8736e37db Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Wed, 28 Aug 2024 16:53:47 +0200 Subject: [PATCH 11/30] pass pytest --- python/ribasim/ribasim/geometry/edge.py | 12 ++++++++---- python/ribasim/ribasim/model.py | 22 +++++++++++++++++----- python/ribasim/tests/test_model.py | 4 +++- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index b341de435..72706f47a 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -128,13 +128,17 @@ def add( ].shape[0] # validation on neighbor amount if ( - in_flow_neighbor >= flow_edge_amount[to_node.node_type][1] + in_flow_neighbor > flow_edge_amount[to_node.node_type][1] ): # too many upstream neighbor, flow edge - raise ValueError + raise ValueError( + f"Node {to_node.node_id} can have at most {flow_edge_amount[to_node.node_type][1]} flow edge inneighbor(s) (got {in_flow_neighbor})" + ) if ( - out_flow_neighbor >= flow_edge_amount[to_node.node_type][3] + out_flow_neighbor > flow_edge_amount[from_node.node_type][3] ): # too many downstream neighbor, flow edge - raise ValueError + raise ValueError( + f"Node {from_node.node_id} can have at most {flow_edge_amount[from_node.node_type][3]} flow edge inneighbor(s) (got {out_flow_neighbor})" + ) self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index e076529f8..de2f753bc 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -1,4 +1,5 @@ import datetime +import logging from collections.abc import Generator from os import PathLike from pathlib import Path @@ -319,9 +320,14 @@ def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount) -> bool: df_result = df_result[["from_node_id", "from_node_count", "from_node_type"]] for _, row in df_result.iterrows(): # from node's outneighbor - if row["from_node_count"] < flow_edge_amount[row["from_node_type"][2]]: - is_valid = False - + try: + if row["from_node_count"] < flow_edge_amount[row["from_node_type"][2]]: + is_valid = False + raise ValueError( + f"Node {row['from_node_id']} must have at least {flow_edge_amount[row["from_node_type"][2]]} outnrighbor(s) (got {row["from_node_count"]})" + ) + except ValueError as e: + logging.error(e) # check to node's neighbor to_node_count = ( df_graph_flow.groupby("to_node_id").size().reset_index(name="to_node_count") @@ -333,8 +339,14 @@ def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount) -> bool: ) df_result = df_result[["to_node_id", "to_node_count", "to_node_type"]] for _, row in df_result.iterrows(): - if row["to_node_count"] < flow_edge_amount[row["to_node_type"][0]]: - is_valid = False + try: + if row["to_node_count"] < flow_edge_amount[row["to_node_type"][0]]: + is_valid = False + raise ValueError( + f"Node {row['to_node_id']} must have at least {flow_edge_amount[row["to_node_type"][0]]} outnrighbor(s) (got {row["to_node_count"]})" + ) + except ValueError as e: + logging.error(e) return is_valid @classmethod diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 95951883b..1cfe0a6df 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -160,7 +160,9 @@ def test_connectivity(trivial): model = trivial with pytest.raises( ValueError, - match=re.escape("Node Terminal cannot connect with upstream node Basin"), + match=re.escape( + "Node of type Basin cannot be upstream of node of type Terminal" + ), ): model.edge.add(model.basin[6], model.terminal[2147483647]) From 26bc35dd42367e13a7744b94bc404b2e967f3f24 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Wed, 28 Aug 2024 18:58:20 +0200 Subject: [PATCH 12/30] added unit test for flow edge neighbor bound --- python/ribasim/ribasim/geometry/edge.py | 8 ++--- python/ribasim/ribasim/model.py | 7 +++-- python/ribasim/tests/conftest.py | 5 ++++ python/ribasim/tests/test_model.py | 40 ++++++++++++++++++++----- python/ribasim/tests/test_validation.py | 1 + 5 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 python/ribasim/tests/test_validation.py diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index 72706f47a..3dcaef8f4 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -127,14 +127,14 @@ def add( & (self.df["edge_type"] == "flow") ].shape[0] # validation on neighbor amount - if ( - in_flow_neighbor > flow_edge_amount[to_node.node_type][1] + if (in_flow_neighbor >= flow_edge_amount[to_node.node_type][1]) & ( + edge_type == "flow" ): # too many upstream neighbor, flow edge raise ValueError( f"Node {to_node.node_id} can have at most {flow_edge_amount[to_node.node_type][1]} flow edge inneighbor(s) (got {in_flow_neighbor})" ) - if ( - out_flow_neighbor > flow_edge_amount[from_node.node_type][3] + if (out_flow_neighbor >= flow_edge_amount[from_node.node_type][3]) & ( + edge_type == "flow" ): # too many downstream neighbor, flow edge raise ValueError( f"Node {from_node.node_id} can have at most {flow_edge_amount[from_node.node_type][3]} flow edge inneighbor(s) (got {out_flow_neighbor})" diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index de2f753bc..9dc39b6a4 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -298,10 +298,11 @@ def validate_model(self): df_node[["node_type"]], on="to_node_id", how="left", rsuffix="_to" ) df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) - flow_edge_amount - self._check_neighbors() - def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount) -> bool: + if not self._check_neighbors(df_graph, flow_edge_amount): + raise ValueError("Minimum inneighbor or outneighbor unsatisfied") + + def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount: dict) -> bool: is_valid = True # Count flow edge neighbor df_graph_flow = df_graph.loc[df_graph["edge_type"] == "flow"] diff --git a/python/ribasim/tests/conftest.py b/python/ribasim/tests/conftest.py index d77d287e1..d5886bb1c 100644 --- a/python/ribasim/tests/conftest.py +++ b/python/ribasim/tests/conftest.py @@ -29,6 +29,11 @@ def tabulated_rating_curve() -> ribasim.Model: return ribasim_testmodels.tabulated_rating_curve_model() +@pytest.fixture() +def outlet() -> ribasim.Model: + return ribasim_testmodels.outlet_model() + + @pytest.fixture() def backwater() -> ribasim.Model: return ribasim_testmodels.backwater_model() diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 1cfe0a6df..9716f00a5 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -12,7 +12,11 @@ from ribasim.geometry.edge import NodeData from ribasim.input_base import esc_id from ribasim.model import Model -from ribasim_testmodels import basic_model, trivial_model +from ribasim.nodes import ( + basin, + level_boundary, +) +from ribasim_testmodels import basic_model, outlet_model, trivial_model from shapely import Point @@ -141,17 +145,17 @@ def test_edge_table(basic): assert df.crs == CRS.from_epsg(28992) -def test_duplicate_edge(trivial): - model = trivial +def test_duplicate_edge(basic): + model = basic with pytest.raises( ValueError, match=re.escape( - "Edges have to be unique, but edge with from_node_id 6 to_node_id 0 already exists." + "Edges have to be unique, but edge with from_node_id 16 to_node_id 1 already exists." ), ): model.edge.add( - model.basin[6], - model.tabulated_rating_curve[0], + model.flow_boundary[16], + model.basin[1], name="duplicate", ) @@ -167,6 +171,28 @@ def test_connectivity(trivial): model.edge.add(model.basin[6], model.terminal[2147483647]) +def test_maximum_neighbor(outlet): + model = outlet + with pytest.raises( + ValueError, + match=re.escape("Node 2 can have at most 1 flow edge inneighbor(s) (got 1)"), + ): + model.basin.add( + Node(4, Point(1.0, 1.0)), + [ + basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), + basin.State(level=[0.0]), + ], + ) + + model.edge.add(model.outlet[2], model.basin[4]) + # Set up the level boundary + model.level_boundary.add( + Node(5, Point(0.0, 1.0)), + [level_boundary.Static(level=[3.0])], + ) + + def test_indexing(basic): model = basic @@ -198,7 +224,7 @@ def test_indexing(basic): model.basin.time[1] -@pytest.mark.parametrize("model", [basic_model(), trivial_model()]) +@pytest.mark.parametrize("model", [basic_model(), outlet_model(), trivial_model()]) def test_xugrid(model, tmp_path): uds = model.to_xugrid(add_flow=False) assert isinstance(uds, xugrid.UgridDataset) diff --git a/python/ribasim/tests/test_validation.py b/python/ribasim/tests/test_validation.py new file mode 100644 index 000000000..afc724138 --- /dev/null +++ b/python/ribasim/tests/test_validation.py @@ -0,0 +1 @@ +# after everything sets in test_model.py, move relevant unit tests here From aa0a173dc36d8481ced0e2f9c17994e7bd9b11e9 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Wed, 28 Aug 2024 19:29:06 +0200 Subject: [PATCH 13/30] add another tests for outneighbor --- python/ribasim/ribasim/geometry/edge.py | 2 +- python/ribasim/ribasim/validation.py | 32 ++++++++++++++++++------- python/ribasim/tests/test_model.py | 10 +++++--- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index 3dcaef8f4..2ca462401 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -137,7 +137,7 @@ def add( edge_type == "flow" ): # too many downstream neighbor, flow edge raise ValueError( - f"Node {from_node.node_id} can have at most {flow_edge_amount[from_node.node_type][3]} flow edge inneighbor(s) (got {out_flow_neighbor})" + f"Node {from_node.node_id} can have at most {flow_edge_amount[from_node.node_type][3]} flow edge outneighbor(s) (got {out_flow_neighbor})" ) self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index d028c5ba6..178978eb3 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -1,5 +1,3 @@ -import numpy as np - # Table for connectivity # "Basin": ["LinearResistance"] means that the downstream of basin can be LinearResistance only connectivity: dict[str, list[str]] = { @@ -49,15 +47,15 @@ def can_connect(node_up: str, node_down: str) -> bool: flow_edge_amount: dict[str, list[int]] = { - "Basin": [0, np.inf, 0, np.inf], + "Basin": [0, int(1e9), 0, int(1e9)], "LinearResistance": [1, 1, 1, 1], "ManningResistance": [1, 1, 1, 1], - "TabulatedRatingCurve": [1, 1, 1, np.inf], - "LevelBoundary": [0, np.inf, 0, np.inf], - "FlowBoundary": [0, 0, 1, np.inf], - "Pump": [1, 1, 1, np.inf], + "TabulatedRatingCurve": [1, 1, 1, int(1e9)], + "LevelBoundary": [0, int(1e9), 0, int(1e9)], + "FlowBoundary": [0, 0, 1, int(1e9)], + "Pump": [1, 1, 1, int(1e9)], "Outlet": [1, 1, 1, 1], - "Terminal": [1, np.inf, 0, 0], + "Terminal": [1, int(1e9), 0, 0], "DiscreteControl": [0, 0, 0, 0], "ContinuousControl": [0, 0, 0, 0], "PidControl": [0, 0, 0, 0], @@ -65,3 +63,21 @@ def can_connect(node_up: str, node_down: str) -> bool: "LevelDemand": [0, 0, 0, 0], "FlowDemand": [0, 0, 0, 0], } + +control_edge_amount: dict[str, list[int]] = { + "Basin": [0, 1, 0, 0], + "LinearResistance": [0, 1, 0, 0], + "ManningResistance": [0, 1, 0, 0], + "TabulatedRatingCurve": [0, 1, 0, 0], + "LevelBoundary": [0, 0, 0, 0], + "FlowBoundary": [0, 0, 0, 0], + "Pump": [0, 1, 0, 0], + "Outlet": [0, 1, 0, 0], + "Terminal": [0, 0, 0, 0], + "DiscreteControl": [0, 0, 1, int(1e9)], + "ContinuousControl": [0, 0, 1, int(1e9)], + "PidControl": [0, 1, 1, 1], + "UserDemand": [0, 0, 0, 0], + "LevelDemand": [0, 0, 1, int(1e9)], + "FlowDemand": [0, 0, 1, 1], +} diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 9716f00a5..5631e12aa 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -175,7 +175,7 @@ def test_maximum_neighbor(outlet): model = outlet with pytest.raises( ValueError, - match=re.escape("Node 2 can have at most 1 flow edge inneighbor(s) (got 1)"), + match=re.escape("Node 2 can have at most 1 flow edge outneighbor(s) (got 1)"), ): model.basin.add( Node(4, Point(1.0, 1.0)), @@ -184,13 +184,17 @@ def test_maximum_neighbor(outlet): basin.State(level=[0.0]), ], ) - model.edge.add(model.outlet[2], model.basin[4]) - # Set up the level boundary + + with pytest.raises( + ValueError, + match=re.escape("Node 2 can have at most 1 flow edge inneighbor(s) (got 1)"), + ): model.level_boundary.add( Node(5, Point(0.0, 1.0)), [level_boundary.Static(level=[3.0])], ) + model.edge.add(model.level_boundary[5], model.outlet[2]) def test_indexing(basic): From b330e958db4a38348793ad5acd12575979c2e4e7 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 10:06:25 +0200 Subject: [PATCH 14/30] unit tests for minimum in-/outneighbor --- python/ribasim/ribasim/model.py | 42 +++++++++++++++++++++++------- python/ribasim/tests/test_model.py | 30 +++++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 9dc39b6a4..959af89ca 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -268,7 +268,7 @@ def write(self, filepath: str | PathLike[str]) -> Path: A file path with .toml extension. """ # TODO - # self.validate_model() + self._validate_model() filepath = Path(filepath) self.filepath = filepath if not filepath.suffix == ".toml": @@ -282,7 +282,7 @@ def write(self, filepath: str | PathLike[str]) -> Path: context_file_writing.set({}) return fn - def validate_model(self): + def _validate_model(self): df_edge = self.edge.df df_chunks = [node.node.df.set_crs(self.crs) for node in self._nodes()] # type:ignore df_node = pd.concat(df_chunks) @@ -299,10 +299,12 @@ def validate_model(self): ) df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) - if not self._check_neighbors(df_graph, flow_edge_amount): + if not self._check_neighbors(df_graph, flow_edge_amount, df_node["node_type"]): raise ValueError("Minimum inneighbor or outneighbor unsatisfied") - def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount: dict) -> bool: + def _check_neighbors( + self, df_graph: pd.DataFrame, flow_edge_amount: dict, nodes: pd.Series + ) -> bool: is_valid = True # Count flow edge neighbor df_graph_flow = df_graph.loc[df_graph["edge_type"] == "flow"] @@ -318,14 +320,24 @@ def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount: dict) -> bo .drop_duplicates() .merge(from_node_count, on="from_node_id", how="left") ) + df_result = df_result[["from_node_id", "from_node_count", "from_node_type"]] + # print(df_result) + for index, node in enumerate(nodes): + if nodes.index[index] not in df_result["from_node_id"].to_numpy(): + new_row = { + "from_node_id": nodes.index[index], + "from_node_count": 0, + "from_node_type": node, + } + df_result.loc[len(df_result)] = new_row for _, row in df_result.iterrows(): - # from node's outneighbor + # from node's outneighbor) try: - if row["from_node_count"] < flow_edge_amount[row["from_node_type"][2]]: + if row["from_node_count"] < flow_edge_amount[row["from_node_type"]][2]: is_valid = False raise ValueError( - f"Node {row['from_node_id']} must have at least {flow_edge_amount[row["from_node_type"][2]]} outnrighbor(s) (got {row["from_node_count"]})" + f"Node {row['from_node_id']} must have at least {flow_edge_amount[row["from_node_type"]][2]} outneighbor(s) (got {row["from_node_count"]})" ) except ValueError as e: logging.error(e) @@ -339,12 +351,24 @@ def _check_neighbors(self, df_graph: pd.DataFrame, flow_edge_amount: dict) -> bo .merge(to_node_count, on="to_node_id", how="left") ) df_result = df_result[["to_node_id", "to_node_count", "to_node_type"]] + print(df_result) + for index, node in enumerate(nodes): + if nodes.index[index] not in df_result["to_node_id"].to_numpy(): + new_row = { + "to_node_id": nodes.index[index], + "to_node_count": 0, + "to_node_type": node, + } + df_result.loc[len(df_result)] = new_row + for _, row in df_result.iterrows(): try: - if row["to_node_count"] < flow_edge_amount[row["to_node_type"][0]]: + print(row["to_node_count"]) + print(flow_edge_amount[row["to_node_type"]][0]) + if row["to_node_count"] < flow_edge_amount[row["to_node_type"]][0]: is_valid = False raise ValueError( - f"Node {row['to_node_id']} must have at least {flow_edge_amount[row["to_node_type"][0]]} outnrighbor(s) (got {row["to_node_count"]})" + f"Node {row['to_node_id']} must have at least {flow_edge_amount[row["to_node_type"]][0]} inneighbor(s) (got {row["to_node_count"]})" ) except ValueError as e: logging.error(e) diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 5631e12aa..95bda9603 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -15,6 +15,7 @@ from ribasim.nodes import ( basin, level_boundary, + outlet, ) from ribasim_testmodels import basic_model, outlet_model, trivial_model from shapely import Point @@ -197,6 +198,35 @@ def test_maximum_neighbor(outlet): model.edge.add(model.level_boundary[5], model.outlet[2]) +def test_minimum_neighbor(): + model = Model( + starttime="2020-01-01", + endtime="2021-01-01", + crs="EPSG:28992", + solver=Solver(), + ) + + model.basin.add( + Node(3, Point(2.0, 0.0)), + [ + basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), + basin.State(level=[0.0]), + ], + ) + model.outlet.add( + Node(2, Point(1.0, 0.0)), + [outlet.Static(flow_rate=[1e-3], min_crest_level=[2.0])], + ) + model.terminal.add(Node(4, Point(3.0, -2.0))) + + with pytest.raises( + ValueError, + match=re.escape("Minimum inneighbor or outneighbor unsatisfied"), + ): + model.edge.add(model.basin[3], model.outlet[2]) + model.write("test.toml") + + def test_indexing(basic): model = basic From 1d26183e4e6796add2be1531fcccedcfb434f465 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 11:19:15 +0200 Subject: [PATCH 15/30] wrote unit test for maximum control edge neighbor --- python/ribasim/ribasim/geometry/edge.py | 28 +++++++++++-- python/ribasim/tests/conftest.py | 5 +++ python/ribasim/tests/test_model.py | 53 +++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index 2ca462401..7e0849773 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -16,7 +16,7 @@ from ribasim.input_base import SpatialTableModel from ribasim.schemas import _BaseSchema from ribasim.utils import UsedIDs -from ribasim.validation import can_connect, flow_edge_amount +from ribasim.validation import can_connect, control_edge_amount, flow_edge_amount __all__ = ("EdgeTable",) @@ -129,17 +129,39 @@ def add( # validation on neighbor amount if (in_flow_neighbor >= flow_edge_amount[to_node.node_type][1]) & ( edge_type == "flow" - ): # too many upstream neighbor, flow edge + ): raise ValueError( f"Node {to_node.node_id} can have at most {flow_edge_amount[to_node.node_type][1]} flow edge inneighbor(s) (got {in_flow_neighbor})" ) if (out_flow_neighbor >= flow_edge_amount[from_node.node_type][3]) & ( edge_type == "flow" - ): # too many downstream neighbor, flow edge + ): raise ValueError( f"Node {from_node.node_id} can have at most {flow_edge_amount[from_node.node_type][3]} flow edge outneighbor(s) (got {out_flow_neighbor})" ) + in_control_neighbor: int = self.df.loc[ + (self.df["to_node_id"] == to_node.node_id) + & (self.df["edge_type"] == "control") + ].shape[0] + out_control_neighbor: int = self.df.loc[ + (self.df["from_node_id"] == from_node.node_id) + & (self.df["edge_type"] == "control") + ].shape[0] + + if (in_control_neighbor >= control_edge_amount[to_node.node_type][1]) & ( + edge_type == "control" + ): + raise ValueError( + f"Node {to_node.node_id} can have at most {control_edge_amount[to_node.node_type][1]} control edge inneighbor(s) (got {in_control_neighbor})" + ) + if (out_control_neighbor >= control_edge_amount[from_node.node_type][3]) & ( + edge_type == "control" + ): + raise ValueError( + f"Node {from_node.node_id} can have at most {control_edge_amount[from_node.node_type][3]} control edge outneighbor(s) (got {out_control_neighbor})" + ) + self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): raise ValueError( diff --git a/python/ribasim/tests/conftest.py b/python/ribasim/tests/conftest.py index d5886bb1c..d1584aab7 100644 --- a/python/ribasim/tests/conftest.py +++ b/python/ribasim/tests/conftest.py @@ -24,6 +24,11 @@ def bucket() -> ribasim.Model: return ribasim_testmodels.bucket_model() +@pytest.fixture() +def pid_control_equation() -> ribasim.Model: + return ribasim_testmodels.pid_control_equation_model() + + @pytest.fixture() def tabulated_rating_curve() -> ribasim.Model: return ribasim_testmodels.tabulated_rating_curve_model() diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 95bda9603..93136653b 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -16,8 +16,15 @@ basin, level_boundary, outlet, + pid_control, + pump, +) +from ribasim_testmodels import ( + basic_model, + outlet_model, + pid_control_equation_model, + trivial_model, ) -from ribasim_testmodels import basic_model, outlet_model, trivial_model from shapely import Point @@ -172,7 +179,7 @@ def test_connectivity(trivial): model.edge.add(model.basin[6], model.terminal[2147483647]) -def test_maximum_neighbor(outlet): +def test_maximum_flow_neighbor(outlet): model = outlet with pytest.raises( ValueError, @@ -198,7 +205,42 @@ def test_maximum_neighbor(outlet): model.edge.add(model.level_boundary[5], model.outlet[2]) -def test_minimum_neighbor(): +def test_maximum_control_neighbor(pid_control_equation): + model = pid_control_equation + with pytest.raises( + ValueError, + match=re.escape("Node 2 can have at most 1 control edge inneighbor(s) (got 1)"), + ): + model.pid_control.add( + Node(5, Point(0.5, -1.0)), + [ + pid_control.Static( + listen_node_id=[1], + target=10.0, + proportional=-2.5, + integral=-0.001, + derivative=10.0, + ) + ], + ) + model.edge.add( + model.pid_control[5], + model.pump[2], + ) + with pytest.raises( + ValueError, + match=re.escape( + "Node 4 can have at most 1 control edge outneighbor(s) (got 1)" + ), + ): + model.pump.add(Node(6, Point(-1.0, 0)), [pump.Static(flow_rate=[0.0])]) + model.edge.add( + model.pid_control[4], + model.pump[6], + ) + + +def test_minimum_flow_neighbor(): model = Model( starttime="2020-01-01", endtime="2021-01-01", @@ -258,7 +300,10 @@ def test_indexing(basic): model.basin.time[1] -@pytest.mark.parametrize("model", [basic_model(), outlet_model(), trivial_model()]) +@pytest.mark.parametrize( + "model", + [basic_model(), outlet_model(), pid_control_equation_model(), trivial_model()], +) def test_xugrid(model, tmp_path): uds = model.to_xugrid(add_flow=False) assert isinstance(uds, xugrid.UgridDataset) From 103958d7b9fb7d255f84bba7128ba1567c4576ee Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 11:25:34 +0200 Subject: [PATCH 16/30] delete useless print line --- python/ribasim/ribasim/model.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 959af89ca..c59385692 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -322,7 +322,6 @@ def _check_neighbors( ) df_result = df_result[["from_node_id", "from_node_count", "from_node_type"]] - # print(df_result) for index, node in enumerate(nodes): if nodes.index[index] not in df_result["from_node_id"].to_numpy(): new_row = { @@ -332,7 +331,7 @@ def _check_neighbors( } df_result.loc[len(df_result)] = new_row for _, row in df_result.iterrows(): - # from node's outneighbor) + # from node's outneighbor try: if row["from_node_count"] < flow_edge_amount[row["from_node_type"]][2]: is_valid = False @@ -351,7 +350,6 @@ def _check_neighbors( .merge(to_node_count, on="to_node_id", how="left") ) df_result = df_result[["to_node_id", "to_node_count", "to_node_type"]] - print(df_result) for index, node in enumerate(nodes): if nodes.index[index] not in df_result["to_node_id"].to_numpy(): new_row = { @@ -363,8 +361,6 @@ def _check_neighbors( for _, row in df_result.iterrows(): try: - print(row["to_node_count"]) - print(flow_edge_amount[row["to_node_type"]][0]) if row["to_node_count"] < flow_edge_amount[row["to_node_type"]][0]: is_valid = False raise ValueError( From 66d5a1e83e48cf232d8272dbf509687b71894b65 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 13:57:44 +0200 Subject: [PATCH 17/30] add keyword of validation in Model class --- python/ribasim/ribasim/model.py | 9 ++++++--- python/ribasim/tests/test_io.py | 10 +++++++++- .../ribasim_testmodels/ribasim_testmodels/invalid.py | 6 +++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index c59385692..1b0369082 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -102,6 +102,7 @@ class Model(FileModel): user_demand: UserDemand = Field(default_factory=UserDemand) edge: EdgeTable = Field(default_factory=EdgeTable) + validation: bool = Field(default=True) _used_node_ids: UsedIDs = PrivateAttr(default_factory=UsedIDs) @@ -267,8 +268,10 @@ def write(self, filepath: str | PathLike[str]) -> Path: filepath : str | PathLike[str] A file path with .toml extension. """ - # TODO - self._validate_model() + # Skip validation if the model name starts with "invalid" + if self.validation: + self._validate_model() + filepath = Path(filepath) self.filepath = filepath if not filepath.suffix == ".toml": @@ -364,7 +367,7 @@ def _check_neighbors( if row["to_node_count"] < flow_edge_amount[row["to_node_type"]][0]: is_valid = False raise ValueError( - f"Node {row['to_node_id']} must have at least {flow_edge_amount[row["to_node_type"]][0]} inneighbor(s) (got {row["to_node_count"]})" + f"Node {row["to_node_id"]} must have at least {flow_edge_amount[row["to_node_type"]][0]} inneighbor(s) (got {row["to_node_count"]})" ) except ValueError as e: logging.error(e) diff --git a/python/ribasim/tests/test_io.py b/python/ribasim/tests/test_io.py index 60d977316..6aab6c0f4 100644 --- a/python/ribasim/tests/test_io.py +++ b/python/ribasim/tests/test_io.py @@ -130,7 +130,15 @@ def test_extra_spatial_columns(): [basin.Profile(area=1000.0, level=[0.0, 1.0]), basin.State(level=[1.0])], ) with pytest.raises(ValidationError): - model.edge.add(model.basin[1], model.user_demand[2], foo=1) + model.user_demand.add( + Node(4, Point(1, -0.5), meta_id=3), + [ + user_demand.Static( + demand=[1e-4], return_factor=0.9, min_level=0.9, priority=1 + ) + ], + ) + model.edge.add(model.basin[1], model.user_demand[4], foo=1) def test_edge_autoincrement(basic): diff --git a/python/ribasim_testmodels/ribasim_testmodels/invalid.py b/python/ribasim_testmodels/ribasim_testmodels/invalid.py index 7cb984f5a..45a699bf8 100644 --- a/python/ribasim_testmodels/ribasim_testmodels/invalid.py +++ b/python/ribasim_testmodels/ribasim_testmodels/invalid.py @@ -28,6 +28,7 @@ def invalid_qh_model() -> Model: starttime="2020-01-01", endtime="2020-12-01", crs="EPSG:28992", + validation=False, ) model.tabulated_rating_curve.add( @@ -43,6 +44,7 @@ def invalid_discrete_control_model() -> Model: starttime="2020-01-01", endtime="2020-12-01", crs="EPSG:28992", + validation=False, ) basin_shared: list[TableModel[Any]] = [ @@ -116,6 +118,7 @@ def invalid_edge_types_model() -> Model: starttime="2020-01-01", endtime="2020-12-01", crs="EPSG:28992", + validation=False, ) basin_shared: list[TableModel[Any]] = [ @@ -150,6 +153,7 @@ def invalid_unstable_model() -> Model: endtime="2021-01-01", crs="EPSG:28992", solver=Solver(dtmin=60.0), + validation=False, ) id_shift = 10 for i in range(6): @@ -176,6 +180,7 @@ def invalid_priorities_model() -> Model: endtime="2021-01-01 00:00:00", crs="EPSG:28992", allocation=Allocation(use_allocation=True, timestep=1e5), + validation=False, ) model.tabulated_rating_curve.add( @@ -215,7 +220,6 @@ def invalid_priorities_model() -> Model: ) model.edge.add(model.tabulated_rating_curve[2], model.basin[3]) model.edge.add(model.basin[3], model.user_demand[4]) - model.edge.add(model.basin[3], model.level_demand[6]) model.edge.add(model.flow_demand[5], model.tabulated_rating_curve[2]) return model From 7c52bd865311a6329a8b28bd8308a1529a5fc4ec Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 14:30:12 +0200 Subject: [PATCH 18/30] fix mypy error --- python/ribasim/ribasim/model.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 1b0369082..92fe5a142 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -287,7 +287,7 @@ def write(self, filepath: str | PathLike[str]) -> Path: def _validate_model(self): df_edge = self.edge.df - df_chunks = [node.node.df.set_crs(self.crs) for node in self._nodes()] # type:ignore + df_chunks = [node.node.df.set_crs(self.crs) for node in self._nodes()] df_node = pd.concat(df_chunks) df_graph = df_edge @@ -306,7 +306,10 @@ def _validate_model(self): raise ValueError("Minimum inneighbor or outneighbor unsatisfied") def _check_neighbors( - self, df_graph: pd.DataFrame, flow_edge_amount: dict, nodes: pd.Series + self, + df_graph: pd.DataFrame, + flow_edge_amount: dict[str, list[int]], + nodes, ) -> bool: is_valid = True # Count flow edge neighbor @@ -332,7 +335,9 @@ def _check_neighbors( "from_node_count": 0, "from_node_type": node, } - df_result.loc[len(df_result)] = new_row + df_result = pd.concat( + [df_result, pd.DataFrame([new_row])], ignore_index=True + ) for _, row in df_result.iterrows(): # from node's outneighbor try: @@ -360,7 +365,9 @@ def _check_neighbors( "to_node_count": 0, "to_node_type": node, } - df_result.loc[len(df_result)] = new_row + df_result = pd.concat( + [df_result, pd.DataFrame([new_row])], ignore_index=True + ) for _, row in df_result.iterrows(): try: From 7688be2355d1969445a6ec61a98a0c941684f325 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 15:01:23 +0200 Subject: [PATCH 19/30] fix py311 py310 test 1 --- python/ribasim/ribasim/model.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 92fe5a142..0cc9b89c4 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -319,8 +319,9 @@ def _check_neighbors( from_node_count = ( df_graph_flow.groupby("from_node_id") .size() - .reset_index(name="from_node_count") + .reset_index(name="from_node_count") # type: ignore ) + df_result = ( df_graph_flow[["from_node_id", "from_node_type"]] .drop_duplicates() @@ -344,13 +345,13 @@ def _check_neighbors( if row["from_node_count"] < flow_edge_amount[row["from_node_type"]][2]: is_valid = False raise ValueError( - f"Node {row['from_node_id']} must have at least {flow_edge_amount[row["from_node_type"]][2]} outneighbor(s) (got {row["from_node_count"]})" + f"Node {row['from_node_id']} must have at least {flow_edge_amount[row['from_node_type']][2]} outneighbor(s) (got {row['from_node_count']})" ) except ValueError as e: logging.error(e) # check to node's neighbor to_node_count = ( - df_graph_flow.groupby("to_node_id").size().reset_index(name="to_node_count") + df_graph_flow.groupby("to_node_id").size().reset_index(name="to_node_count") # type: ignore ) df_result = ( df_graph_flow[["to_node_id", "to_node_type"]] @@ -374,7 +375,7 @@ def _check_neighbors( if row["to_node_count"] < flow_edge_amount[row["to_node_type"]][0]: is_valid = False raise ValueError( - f"Node {row["to_node_id"]} must have at least {flow_edge_amount[row["to_node_type"]][0]} inneighbor(s) (got {row["to_node_count"]})" + f"Node {row['to_node_id']} must have at least {flow_edge_amount[row['to_node_type']][0]} inneighbor(s) (got {row['to_node_count']})" ) except ValueError as e: logging.error(e) From 59c52333a615c8e995e0ca3a59314100f89bee80 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 15:32:05 +0200 Subject: [PATCH 20/30] fix julia unit test --- python/ribasim_testmodels/ribasim_testmodels/invalid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ribasim_testmodels/ribasim_testmodels/invalid.py b/python/ribasim_testmodels/ribasim_testmodels/invalid.py index 45a699bf8..40702b445 100644 --- a/python/ribasim_testmodels/ribasim_testmodels/invalid.py +++ b/python/ribasim_testmodels/ribasim_testmodels/invalid.py @@ -220,6 +220,7 @@ def invalid_priorities_model() -> Model: ) model.edge.add(model.tabulated_rating_curve[2], model.basin[3]) model.edge.add(model.basin[3], model.user_demand[4]) + model.edge.add(model.level_demand[6], model.basin[3]) model.edge.add(model.flow_demand[5], model.tabulated_rating_curve[2]) return model From 58911ead8d12b238499d777b2327aeecf096e33b Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 16:12:29 +0200 Subject: [PATCH 21/30] move validation unit tests to a different file --- python/ribasim/ribasim/model.py | 37 ++++--- python/ribasim/tests/test_model.py | 123 ---------------------- python/ribasim/tests/test_validation.py | 131 +++++++++++++++++++++++- 3 files changed, 151 insertions(+), 140 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 0cc9b89c4..60ae6b268 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -59,7 +59,7 @@ _node_lookup_numpy, _time_in_ns, ) -from ribasim.validation import flow_edge_amount +from ribasim.validation import control_edge_amount, flow_edge_amount try: import xugrid @@ -302,28 +302,33 @@ def _validate_model(self): ) df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) - if not self._check_neighbors(df_graph, flow_edge_amount, df_node["node_type"]): - raise ValueError("Minimum inneighbor or outneighbor unsatisfied") + if not self._check_neighbors( + df_graph, flow_edge_amount, "flow", df_node["node_type"] + ): + raise ValueError("Minimum flow inneighbor or outneighbor unsatisfied") + if not self._check_neighbors( + df_graph, control_edge_amount, "control", df_node["node_type"] + ): + raise ValueError("Minimum control inneighbor or outneighbor unsatisfied") def _check_neighbors( self, df_graph: pd.DataFrame, - flow_edge_amount: dict[str, list[int]], + edge_amount: dict[str, list[int]], + edge_type: str, nodes, ) -> bool: is_valid = True - # Count flow edge neighbor - df_graph_flow = df_graph.loc[df_graph["edge_type"] == "flow"] + # Count edge neighbor + df_graph = df_graph.loc[df_graph["edge_type"] == edge_type] # check from node's neighbor from_node_count = ( - df_graph_flow.groupby("from_node_id") - .size() - .reset_index(name="from_node_count") # type: ignore + df_graph.groupby("from_node_id").size().reset_index(name="from_node_count") # type: ignore ) df_result = ( - df_graph_flow[["from_node_id", "from_node_type"]] + df_graph[["from_node_id", "from_node_type"]] .drop_duplicates() .merge(from_node_count, on="from_node_id", how="left") ) @@ -342,19 +347,19 @@ def _check_neighbors( for _, row in df_result.iterrows(): # from node's outneighbor try: - if row["from_node_count"] < flow_edge_amount[row["from_node_type"]][2]: + if row["from_node_count"] < edge_amount[row["from_node_type"]][2]: is_valid = False raise ValueError( - f"Node {row['from_node_id']} must have at least {flow_edge_amount[row['from_node_type']][2]} outneighbor(s) (got {row['from_node_count']})" + f"Node {row['from_node_id']} must have at least {edge_amount[row['from_node_type']][2]} outneighbor(s) (got {row['from_node_count']})" ) except ValueError as e: logging.error(e) # check to node's neighbor to_node_count = ( - df_graph_flow.groupby("to_node_id").size().reset_index(name="to_node_count") # type: ignore + df_graph.groupby("to_node_id").size().reset_index(name="to_node_count") # type: ignore ) df_result = ( - df_graph_flow[["to_node_id", "to_node_type"]] + df_graph[["to_node_id", "to_node_type"]] .drop_duplicates() .merge(to_node_count, on="to_node_id", how="left") ) @@ -372,10 +377,10 @@ def _check_neighbors( for _, row in df_result.iterrows(): try: - if row["to_node_count"] < flow_edge_amount[row["to_node_type"]][0]: + if row["to_node_count"] < edge_amount[row["to_node_type"]][0]: is_valid = False raise ValueError( - f"Node {row['to_node_id']} must have at least {flow_edge_amount[row['to_node_type']][0]} inneighbor(s) (got {row['to_node_count']})" + f"Node {row['to_node_id']} must have at least {edge_amount[row['to_node_type']][0]} inneighbor(s) (got {row['to_node_count']})" ) except ValueError as e: logging.error(e) diff --git a/python/ribasim/tests/test_model.py b/python/ribasim/tests/test_model.py index 54ed79edf..a8ebd8d24 100644 --- a/python/ribasim/tests/test_model.py +++ b/python/ribasim/tests/test_model.py @@ -12,13 +12,6 @@ from ribasim.geometry.edge import NodeData from ribasim.input_base import esc_id from ribasim.model import Model -from ribasim.nodes import ( - basin, - level_boundary, - outlet, - pid_control, - pump, -) from ribasim_testmodels import ( basic_model, outlet_model, @@ -153,122 +146,6 @@ def test_edge_table(basic): assert df.crs == CRS.from_epsg(28992) -def test_duplicate_edge(basic): - model = basic - with pytest.raises( - ValueError, - match=re.escape( - "Edges have to be unique, but edge with from_node_id 16 to_node_id 1 already exists." - ), - ): - model.edge.add( - model.flow_boundary[16], - model.basin[1], - name="duplicate", - ) - - -def test_connectivity(trivial): - model = trivial - with pytest.raises( - ValueError, - match=re.escape( - "Node of type Basin cannot be upstream of node of type Terminal" - ), - ): - model.edge.add(model.basin[6], model.terminal[2147483647]) - - -def test_maximum_flow_neighbor(outlet): - model = outlet - with pytest.raises( - ValueError, - match=re.escape("Node 2 can have at most 1 flow edge outneighbor(s) (got 1)"), - ): - model.basin.add( - Node(4, Point(1.0, 1.0)), - [ - basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), - basin.State(level=[0.0]), - ], - ) - model.edge.add(model.outlet[2], model.basin[4]) - - with pytest.raises( - ValueError, - match=re.escape("Node 2 can have at most 1 flow edge inneighbor(s) (got 1)"), - ): - model.level_boundary.add( - Node(5, Point(0.0, 1.0)), - [level_boundary.Static(level=[3.0])], - ) - model.edge.add(model.level_boundary[5], model.outlet[2]) - - -def test_maximum_control_neighbor(pid_control_equation): - model = pid_control_equation - with pytest.raises( - ValueError, - match=re.escape("Node 2 can have at most 1 control edge inneighbor(s) (got 1)"), - ): - model.pid_control.add( - Node(5, Point(0.5, -1.0)), - [ - pid_control.Static( - listen_node_id=[1], - target=10.0, - proportional=-2.5, - integral=-0.001, - derivative=10.0, - ) - ], - ) - model.edge.add( - model.pid_control[5], - model.pump[2], - ) - with pytest.raises( - ValueError, - match=re.escape( - "Node 4 can have at most 1 control edge outneighbor(s) (got 1)" - ), - ): - model.pump.add(Node(6, Point(-1.0, 0)), [pump.Static(flow_rate=[0.0])]) - model.edge.add( - model.pid_control[4], - model.pump[6], - ) - - -def test_minimum_flow_neighbor(): - model = Model( - starttime="2020-01-01", - endtime="2021-01-01", - crs="EPSG:28992", - solver=Solver(), - ) - - model.basin.add( - Node(3, Point(2.0, 0.0)), - [ - basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), - basin.State(level=[0.0]), - ], - ) - model.outlet.add( - Node(2, Point(1.0, 0.0)), - [outlet.Static(flow_rate=[1e-3], min_crest_level=[2.0])], - ) - model.terminal.add(Node(4, Point(3.0, -2.0))) - - with pytest.raises( - ValueError, - match=re.escape("Minimum inneighbor or outneighbor unsatisfied"), - ): - model.edge.add(model.basin[3], model.outlet[2]) - model.write("test.toml") - - def test_indexing(basic): model = basic diff --git a/python/ribasim/tests/test_validation.py b/python/ribasim/tests/test_validation.py index afc724138..07ae24434 100644 --- a/python/ribasim/tests/test_validation.py +++ b/python/ribasim/tests/test_validation.py @@ -1 +1,130 @@ -# after everything sets in test_model.py, move relevant unit tests here +import re + +import pytest +from ribasim import Node +from ribasim.config import Solver +from ribasim.model import Model +from ribasim.nodes import ( + basin, + level_boundary, + outlet, + pid_control, + pump, +) +from shapely import Point + + +def test_duplicate_edge(basic): + model = basic + with pytest.raises( + ValueError, + match=re.escape( + "Edges have to be unique, but edge with from_node_id 16 to_node_id 1 already exists." + ), + ): + model.edge.add( + model.flow_boundary[16], + model.basin[1], + name="duplicate", + ) + + +def test_connectivity(trivial): + model = trivial + with pytest.raises( + ValueError, + match=re.escape( + "Node of type Basin cannot be upstream of node of type Terminal" + ), + ): + model.edge.add(model.basin[6], model.terminal[2147483647]) + + +def test_maximum_flow_neighbor(outlet): + model = outlet + with pytest.raises( + ValueError, + match=re.escape("Node 2 can have at most 1 flow edge outneighbor(s) (got 1)"), + ): + model.basin.add( + Node(4, Point(1.0, 1.0)), + [ + basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), + basin.State(level=[0.0]), + ], + ) + model.edge.add(model.outlet[2], model.basin[4]) + + with pytest.raises( + ValueError, + match=re.escape("Node 2 can have at most 1 flow edge inneighbor(s) (got 1)"), + ): + model.level_boundary.add( + Node(5, Point(0.0, 1.0)), + [level_boundary.Static(level=[3.0])], + ) + model.edge.add(model.level_boundary[5], model.outlet[2]) + + +def test_maximum_control_neighbor(pid_control_equation): + model = pid_control_equation + with pytest.raises( + ValueError, + match=re.escape("Node 2 can have at most 1 control edge inneighbor(s) (got 1)"), + ): + model.pid_control.add( + Node(5, Point(0.5, -1.0)), + [ + pid_control.Static( + listen_node_id=[1], + target=10.0, + proportional=-2.5, + integral=-0.001, + derivative=10.0, + ) + ], + ) + model.edge.add( + model.pid_control[5], + model.pump[2], + ) + with pytest.raises( + ValueError, + match=re.escape( + "Node 4 can have at most 1 control edge outneighbor(s) (got 1)" + ), + ): + model.pump.add(Node(6, Point(-1.0, 0)), [pump.Static(flow_rate=[0.0])]) + model.edge.add( + model.pid_control[4], + model.pump[6], + ) + + +def test_minimum_flow_neighbor(): + model = Model( + starttime="2020-01-01", + endtime="2021-01-01", + crs="EPSG:28992", + solver=Solver(), + ) + + model.basin.add( + Node(3, Point(2.0, 0.0)), + [ + basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), + basin.State(level=[0.0]), + ], + ) + model.outlet.add( + Node(2, Point(1.0, 0.0)), + [outlet.Static(flow_rate=[1e-3], min_crest_level=[2.0])], + ) + model.terminal.add(Node(4, Point(3.0, -2.0))) + + with pytest.raises( + ValueError, + match=re.escape("Minimum flow inneighbor or outneighbor unsatisfied"), + ): + model.edge.add(model.basin[3], model.outlet[2]) + model.write("test.toml") From 6c9622fb2d75dbf0d163af60d14b645c5b11c0f4 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Thu, 29 Aug 2024 16:21:40 +0200 Subject: [PATCH 22/30] adding unit test for minimum control edge amount --- python/ribasim/tests/test_validation.py | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/python/ribasim/tests/test_validation.py b/python/ribasim/tests/test_validation.py index 07ae24434..6cd76dc0a 100644 --- a/python/ribasim/tests/test_validation.py +++ b/python/ribasim/tests/test_validation.py @@ -128,3 +128,46 @@ def test_minimum_flow_neighbor(): ): model.edge.add(model.basin[3], model.outlet[2]) model.write("test.toml") + + +def test_minimum_control_neighbor(): + model = Model( + starttime="2020-01-01", + endtime="2021-01-01", + crs="EPSG:28992", + solver=Solver(), + ) + + model.basin.add( + Node(3, Point(2.0, 0.0)), + [ + basin.Profile(area=[1000.0, 1000.0], level=[0.0, 1.0]), + basin.State(level=[0.0]), + ], + ) + model.outlet.add( + Node(2, Point(1.0, 0.0)), + [outlet.Static(flow_rate=[1e-3], min_crest_level=[2.0])], + ) + model.terminal.add(Node(4, Point(3.0, -2.0))) + + model.edge.add(model.basin[3], model.outlet[2]) + model.edge.add(model.outlet[2], model.terminal[4]) + + with pytest.raises( + ValueError, + match=re.escape("Minimum control inneighbor or outneighbor unsatisfied"), + ): + model.pid_control.add( + Node(5, Point(0.5, 1)), + [ + pid_control.Static( + listen_node_id=[3], + target=10.0, + proportional=-2.5, + integral=-0.001, + derivative=10.0, + ) + ], + ) + model.write("test.toml") From be13e64f550cee9917f0ee4f3957bfcde34a6adc Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Fri, 30 Aug 2024 08:36:17 +0200 Subject: [PATCH 23/30] move validation of edge.add to a separate function --- python/ribasim/ribasim/geometry/edge.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index d04121913..9e6a86a5e 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -118,6 +118,17 @@ def add( index=pd.Index([edge_id], name="edge_id"), ) + self._validate_edge(to_node, from_node, edge_type) + + self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) + if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): + raise ValueError( + f"Edges have to be unique, but edge with from_node_id {from_node.node_id} to_node_id {to_node.node_id} already exists." + ) + self._used_edge_ids.add(edge_id) + + def _validate_edge(self, to_node: NodeData, from_node: NodeData, edge_type: str): + assert self.df is not None in_flow_neighbor: int = self.df.loc[ (self.df["to_node_id"] == to_node.node_id) & (self.df["edge_type"] == "flow") @@ -163,13 +174,6 @@ def add( f"Node {from_node.node_id} can have at most {control_edge_amount[from_node.node_type][3]} control edge outneighbor(s) (got {out_control_neighbor})" ) - self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) - if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): - raise ValueError( - f"Edges have to be unique, but edge with from_node_id {from_node.node_id} to_node_id {to_node.node_id} already exists." - ) - self._used_edge_ids.add(edge_id) - def _get_where_edge_type(self, edge_type: str) -> NDArray[np.bool_]: assert self.df is not None return (self.df.edge_type == edge_type).to_numpy() From 4177665f38c584d1c992a027a65f293a2695c5ad Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Fri, 30 Aug 2024 09:33:04 +0200 Subject: [PATCH 24/30] suggest possible downstream notetype in error message --- python/ribasim/ribasim/geometry/edge.py | 9 +++++++-- python/ribasim/tests/test_validation.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index 9e6a86a5e..807647053 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -15,7 +15,12 @@ from ribasim.input_base import SpatialTableModel from ribasim.utils import UsedIDs -from ribasim.validation import can_connect, control_edge_amount, flow_edge_amount +from ribasim.validation import ( + can_connect, + connectivity, + control_edge_amount, + flow_edge_amount, +) from .base import _GeoBaseSchema @@ -85,7 +90,7 @@ def add( """ if not can_connect(from_node.node_type, to_node.node_type): raise ValueError( - f"Node of type {from_node.node_type} cannot be upstream of node of type {to_node.node_type}" + f"Node of type {to_node.node_type} cannot be downstream of node of type {from_node.node_type}. Possible downstream node: {connectivity[from_node.node_type]}." ) geometry_to_append = ( diff --git a/python/ribasim/tests/test_validation.py b/python/ribasim/tests/test_validation.py index 6cd76dc0a..afb78a409 100644 --- a/python/ribasim/tests/test_validation.py +++ b/python/ribasim/tests/test_validation.py @@ -34,7 +34,7 @@ def test_connectivity(trivial): with pytest.raises( ValueError, match=re.escape( - "Node of type Basin cannot be upstream of node of type Terminal" + "Node of type Terminal cannot be downstream of node of type Basin. Possible downstream node: ['LinearResistance', 'ManningResistance', 'TabulatedRatingCurve', 'Pump', 'Outlet', 'UserDemand']" ), ): model.edge.add(model.basin[6], model.terminal[2147483647]) From 894d8c784ed5e168c3e049267efb254df32bf782 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Mon, 2 Sep 2024 14:59:45 +0200 Subject: [PATCH 25/30] address all comments --- python/ribasim/ribasim/geometry/edge.py | 68 +++++++------------ python/ribasim/ribasim/model.py | 59 +++++++++------- python/ribasim/ribasim/validation.py | 18 ++--- .../ribasim_testmodels/invalid.py | 10 +-- 4 files changed, 74 insertions(+), 81 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index 807647053..6185e99a7 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -17,9 +17,9 @@ from ribasim.utils import UsedIDs from ribasim.validation import ( can_connect, - connectivity, - control_edge_amount, - flow_edge_amount, + control_edge_neighbor_amount, + flow_edge_neighbor_amount, + node_type_connectivity, ) from .base import _GeoBaseSchema @@ -90,7 +90,7 @@ def add( """ if not can_connect(from_node.node_type, to_node.node_type): raise ValueError( - f"Node of type {to_node.node_type} cannot be downstream of node of type {from_node.node_type}. Possible downstream node: {connectivity[from_node.node_type]}." + f"Node of type {to_node.node_type} cannot be downstream of node of type {from_node.node_type}. Possible downstream node: {node_type_connectivity[from_node.node_type]}." ) geometry_to_append = ( @@ -134,50 +134,34 @@ def add( def _validate_edge(self, to_node: NodeData, from_node: NodeData, edge_type: str): assert self.df is not None - in_flow_neighbor: int = self.df.loc[ + in_neighbor: int = self.df.loc[ (self.df["to_node_id"] == to_node.node_id) - & (self.df["edge_type"] == "flow") + & (self.df["edge_type"] == edge_type) ].shape[0] - out_flow_neighbor: int = self.df.loc[ + out_neighbor: int = self.df.loc[ (self.df["from_node_id"] == from_node.node_id) - & (self.df["edge_type"] == "flow") + & (self.df["edge_type"] == edge_type) ].shape[0] # validation on neighbor amount - if (in_flow_neighbor >= flow_edge_amount[to_node.node_type][1]) & ( - edge_type == "flow" - ): - raise ValueError( - f"Node {to_node.node_id} can have at most {flow_edge_amount[to_node.node_type][1]} flow edge inneighbor(s) (got {in_flow_neighbor})" - ) - if (out_flow_neighbor >= flow_edge_amount[from_node.node_type][3]) & ( - edge_type == "flow" - ): - raise ValueError( - f"Node {from_node.node_id} can have at most {flow_edge_amount[from_node.node_type][3]} flow edge outneighbor(s) (got {out_flow_neighbor})" - ) - - in_control_neighbor: int = self.df.loc[ - (self.df["to_node_id"] == to_node.node_id) - & (self.df["edge_type"] == "control") - ].shape[0] - out_control_neighbor: int = self.df.loc[ - (self.df["from_node_id"] == from_node.node_id) - & (self.df["edge_type"] == "control") - ].shape[0] - - if (in_control_neighbor >= control_edge_amount[to_node.node_type][1]) & ( - edge_type == "control" - ): - raise ValueError( - f"Node {to_node.node_id} can have at most {control_edge_amount[to_node.node_type][1]} control edge inneighbor(s) (got {in_control_neighbor})" - ) - if (out_control_neighbor >= control_edge_amount[from_node.node_type][3]) & ( - edge_type == "control" - ): - raise ValueError( - f"Node {from_node.node_id} can have at most {control_edge_amount[from_node.node_type][3]} control edge outneighbor(s) (got {out_control_neighbor})" - ) + if edge_type == "flow": + if in_neighbor >= flow_edge_neighbor_amount[to_node.node_type][1]: + raise ValueError( + f"Node {to_node.node_id} can have at most {flow_edge_neighbor_amount[to_node.node_type][1]} flow edge inneighbor(s) (got {in_neighbor})" + ) + if out_neighbor >= flow_edge_neighbor_amount[from_node.node_type][3]: + raise ValueError( + f"Node {from_node.node_id} can have at most {flow_edge_neighbor_amount[from_node.node_type][3]} flow edge outneighbor(s) (got {out_neighbor})" + ) + elif edge_type == "control": + if in_neighbor >= control_edge_neighbor_amount[to_node.node_type][1]: + raise ValueError( + f"Node {to_node.node_id} can have at most {control_edge_neighbor_amount[to_node.node_type][1]} control edge inneighbor(s) (got {in_neighbor})" + ) + if out_neighbor >= control_edge_neighbor_amount[from_node.node_type][3]: + raise ValueError( + f"Node {from_node.node_id} can have at most {control_edge_neighbor_amount[from_node.node_type][3]} control edge outneighbor(s) (got {out_neighbor})" + ) def _get_where_edge_type(self, edge_type: str) -> NDArray[np.bool_]: assert self.df is not None diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 60ae6b268..a00095e5d 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -59,7 +59,7 @@ _node_lookup_numpy, _time_in_ns, ) -from ribasim.validation import control_edge_amount, flow_edge_amount +from ribasim.validation import control_edge_neighbor_amount, flow_edge_neighbor_amount try: import xugrid @@ -102,7 +102,7 @@ class Model(FileModel): user_demand: UserDemand = Field(default_factory=UserDemand) edge: EdgeTable = Field(default_factory=EdgeTable) - validation: bool = Field(default=True) + use_validation: bool = Field(default=True) _used_node_ids: UsedIDs = PrivateAttr(default_factory=UsedIDs) @@ -268,8 +268,8 @@ def write(self, filepath: str | PathLike[str]) -> Path: filepath : str | PathLike[str] A file path with .toml extension. """ - # Skip validation if the model name starts with "invalid" - if self.validation: + + if self.use_validation: self._validate_model() filepath = Path(filepath) @@ -303,11 +303,11 @@ def _validate_model(self): df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) if not self._check_neighbors( - df_graph, flow_edge_amount, "flow", df_node["node_type"] + df_graph, flow_edge_neighbor_amount, "flow", df_node["node_type"] ): raise ValueError("Minimum flow inneighbor or outneighbor unsatisfied") if not self._check_neighbors( - df_graph, control_edge_amount, "control", df_node["node_type"] + df_graph, control_edge_neighbor_amount, "control", df_node["node_type"] ): raise ValueError("Minimum control inneighbor or outneighbor unsatisfied") @@ -318,22 +318,27 @@ def _check_neighbors( edge_type: str, nodes, ) -> bool: + """Check if the neighbor amount of the two nodes connected by the given edge meet the minimum requirements.""" + is_valid = True - # Count edge neighbor + + # filter graph by edge type df_graph = df_graph.loc[df_graph["edge_type"] == edge_type] - # check from node's neighbor + # count occurrence of "from_node" which reflects the number of outneighbors from_node_count = ( df_graph.groupby("from_node_id").size().reset_index(name="from_node_count") # type: ignore ) + # append from_node_count column to result df_result = ( df_graph[["from_node_id", "from_node_type"]] .drop_duplicates() .merge(from_node_count, on="from_node_id", how="left") ) - df_result = df_result[["from_node_id", "from_node_count", "from_node_type"]] + + # loop over nodes, add the one that is not the upstream of any other nodes for index, node in enumerate(nodes): if nodes.index[index] not in df_result["from_node_id"].to_numpy(): new_row = { @@ -344,26 +349,29 @@ def _check_neighbors( df_result = pd.concat( [df_result, pd.DataFrame([new_row])], ignore_index=True ) + # loop over all the "from_node" and check if they have enough outneighbor for _, row in df_result.iterrows(): # from node's outneighbor - try: - if row["from_node_count"] < edge_amount[row["from_node_type"]][2]: - is_valid = False - raise ValueError( - f"Node {row['from_node_id']} must have at least {edge_amount[row['from_node_type']][2]} outneighbor(s) (got {row['from_node_count']})" - ) - except ValueError as e: - logging.error(e) - # check to node's neighbor + if row["from_node_count"] < edge_amount[row["from_node_type"]][2]: + is_valid = False + logging.error( + f"Node {row['from_node_id']} must have at least {edge_amount[row['from_node_type']][2]} outneighbor(s) (got {row['from_node_count']})" + ) + + # count occurrence of "to_node" which reflects the number of inneighbors to_node_count = ( df_graph.groupby("to_node_id").size().reset_index(name="to_node_count") # type: ignore ) + + # append to_node_count column to result df_result = ( df_graph[["to_node_id", "to_node_type"]] .drop_duplicates() .merge(to_node_count, on="to_node_id", how="left") ) df_result = df_result[["to_node_id", "to_node_count", "to_node_type"]] + + # loop over nodes, add the one that is not the downstream of any other nodes for index, node in enumerate(nodes): if nodes.index[index] not in df_result["to_node_id"].to_numpy(): new_row = { @@ -375,15 +383,14 @@ def _check_neighbors( [df_result, pd.DataFrame([new_row])], ignore_index=True ) + # loop over all the "to_node" and check if they have enough inneighbor for _, row in df_result.iterrows(): - try: - if row["to_node_count"] < edge_amount[row["to_node_type"]][0]: - is_valid = False - raise ValueError( - f"Node {row['to_node_id']} must have at least {edge_amount[row['to_node_type']][0]} inneighbor(s) (got {row['to_node_count']})" - ) - except ValueError as e: - logging.error(e) + if row["to_node_count"] < edge_amount[row["to_node_type"]][0]: + is_valid = False + logging.error( + f"Node {row['to_node_id']} must have at least {edge_amount[row['to_node_type']][0]} inneighbor(s) (got {row['to_node_count']})" + ) + return is_valid @classmethod diff --git a/python/ribasim/ribasim/validation.py b/python/ribasim/ribasim/validation.py index 178978eb3..808a355e9 100644 --- a/python/ribasim/ribasim/validation.py +++ b/python/ribasim/ribasim/validation.py @@ -1,6 +1,6 @@ -# Table for connectivity +# Table for connectivity between different node types # "Basin": ["LinearResistance"] means that the downstream of basin can be LinearResistance only -connectivity: dict[str, list[str]] = { +node_type_connectivity: dict[str, list[str]] = { "Basin": [ "LinearResistance", "ManningResistance", @@ -39,14 +39,15 @@ } -# Function to validate connection -def can_connect(node_up: str, node_down: str) -> bool: - if node_up in connectivity: - return node_down in connectivity[node_up] +# Function to validate connectivity between two node types +def can_connect(node_type_up: str, node_type_down: str) -> bool: + if node_type_up in node_type_connectivity: + return node_type_down in node_type_connectivity[node_type_up] return False -flow_edge_amount: dict[str, list[int]] = { +flow_edge_neighbor_amount: dict[str, list[int]] = { + # list[int] = [in_min, in_max, out_min, out_max] "Basin": [0, int(1e9), 0, int(1e9)], "LinearResistance": [1, 1, 1, 1], "ManningResistance": [1, 1, 1, 1], @@ -64,7 +65,8 @@ def can_connect(node_up: str, node_down: str) -> bool: "FlowDemand": [0, 0, 0, 0], } -control_edge_amount: dict[str, list[int]] = { +control_edge_neighbor_amount: dict[str, list[int]] = { + # list[int] = [in_min, in_max, out_min, out_max] "Basin": [0, 1, 0, 0], "LinearResistance": [0, 1, 0, 0], "ManningResistance": [0, 1, 0, 0], diff --git a/python/ribasim_testmodels/ribasim_testmodels/invalid.py b/python/ribasim_testmodels/ribasim_testmodels/invalid.py index 40702b445..995d30993 100644 --- a/python/ribasim_testmodels/ribasim_testmodels/invalid.py +++ b/python/ribasim_testmodels/ribasim_testmodels/invalid.py @@ -28,7 +28,7 @@ def invalid_qh_model() -> Model: starttime="2020-01-01", endtime="2020-12-01", crs="EPSG:28992", - validation=False, + use_validation=False, ) model.tabulated_rating_curve.add( @@ -44,7 +44,7 @@ def invalid_discrete_control_model() -> Model: starttime="2020-01-01", endtime="2020-12-01", crs="EPSG:28992", - validation=False, + use_validation=False, ) basin_shared: list[TableModel[Any]] = [ @@ -118,7 +118,7 @@ def invalid_edge_types_model() -> Model: starttime="2020-01-01", endtime="2020-12-01", crs="EPSG:28992", - validation=False, + use_validation=False, ) basin_shared: list[TableModel[Any]] = [ @@ -153,7 +153,7 @@ def invalid_unstable_model() -> Model: endtime="2021-01-01", crs="EPSG:28992", solver=Solver(dtmin=60.0), - validation=False, + use_validation=False, ) id_shift = 10 for i in range(6): @@ -180,7 +180,7 @@ def invalid_priorities_model() -> Model: endtime="2021-01-01 00:00:00", crs="EPSG:28992", allocation=Allocation(use_allocation=True, timestep=1e5), - validation=False, + use_validation=False, ) model.tabulated_rating_curve.add( From d9d5394b92e6f6420944dd57cc9437fa91e4e1d8 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Mon, 2 Sep 2024 16:20:22 +0200 Subject: [PATCH 26/30] refactor and split check_neighbor_amount function --- python/ribasim/ribasim/model.py | 91 +++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index a00095e5d..918cb2810 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -302,16 +302,16 @@ def _validate_model(self): ) df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) - if not self._check_neighbors( + if not self._check_neighbor_amount( df_graph, flow_edge_neighbor_amount, "flow", df_node["node_type"] ): raise ValueError("Minimum flow inneighbor or outneighbor unsatisfied") - if not self._check_neighbors( + if not self._check_neighbor_amount( df_graph, control_edge_neighbor_amount, "control", df_node["node_type"] ): raise ValueError("Minimum control inneighbor or outneighbor unsatisfied") - def _check_neighbors( + def _check_neighbor_amount( self, df_graph: pd.DataFrame, edge_amount: dict[str, list[int]], @@ -330,27 +330,21 @@ def _check_neighbors( df_graph.groupby("from_node_id").size().reset_index(name="from_node_count") # type: ignore ) - # append from_node_count column to result - df_result = ( + # append from_node_count column to from_node_id and from_node_type + from_node_info = ( df_graph[["from_node_id", "from_node_type"]] .drop_duplicates() .merge(from_node_count, on="from_node_id", how="left") ) - df_result = df_result[["from_node_id", "from_node_count", "from_node_type"]] - - # loop over nodes, add the one that is not the upstream of any other nodes - for index, node in enumerate(nodes): - if nodes.index[index] not in df_result["from_node_id"].to_numpy(): - new_row = { - "from_node_id": nodes.index[index], - "from_node_count": 0, - "from_node_type": node, - } - df_result = pd.concat( - [df_result, pd.DataFrame([new_row])], ignore_index=True - ) + from_node_info = from_node_info[ + ["from_node_id", "from_node_count", "from_node_type"] + ] + + # add the node that is not the upstream of any other nodes + from_node_info = self._add_source_sink_node(nodes, from_node_info, "from") + # loop over all the "from_node" and check if they have enough outneighbor - for _, row in df_result.iterrows(): + for _, row in from_node_info.iterrows(): # from node's outneighbor if row["from_node_count"] < edge_amount[row["from_node_type"]][2]: is_valid = False @@ -364,27 +358,18 @@ def _check_neighbors( ) # append to_node_count column to result - df_result = ( + to_node_info = ( df_graph[["to_node_id", "to_node_type"]] .drop_duplicates() .merge(to_node_count, on="to_node_id", how="left") ) - df_result = df_result[["to_node_id", "to_node_count", "to_node_type"]] - - # loop over nodes, add the one that is not the downstream of any other nodes - for index, node in enumerate(nodes): - if nodes.index[index] not in df_result["to_node_id"].to_numpy(): - new_row = { - "to_node_id": nodes.index[index], - "to_node_count": 0, - "to_node_type": node, - } - df_result = pd.concat( - [df_result, pd.DataFrame([new_row])], ignore_index=True - ) + to_node_info = to_node_info[["to_node_id", "to_node_count", "to_node_type"]] + + # add the node that is not the downstream of any other nodes + to_node_info = self._add_source_sink_node(nodes, to_node_info, "to") # loop over all the "to_node" and check if they have enough inneighbor - for _, row in df_result.iterrows(): + for _, row in to_node_info.iterrows(): if row["to_node_count"] < edge_amount[row["to_node_type"]][0]: is_valid = False logging.error( @@ -393,6 +378,44 @@ def _check_neighbors( return is_valid + def _add_source_sink_node( + self, nodes, node_info: pd.DataFrame, direction: str + ) -> pd.DataFrame: + """Loop over node table. + + Add the nodes whose id are missing in the from_node and to_node column in the edge table because they are not the upstream or downstrem of other nodes. + + Specify that their occurrence in from_node table or to_node table is 0. + """ + + if direction == "from": + # loop over nodes, add the one that is not the downstream of any other nodes + for index, node in enumerate(nodes): + if nodes.index[index] not in node_info["from_node_id"].to_numpy(): + new_row = { + "from_node_id": nodes.index[index], + "from_node_count": 0, + "from_node_type": node, + } + node_info = pd.concat( + [node_info, pd.DataFrame([new_row])], ignore_index=True + ) + + elif direction == "to": + # loop over nodes, add the one that is not the downstream of any other nodes + for index, node in enumerate(nodes): + if nodes.index[index] not in node_info["to_node_id"].to_numpy(): + new_row = { + "to_node_id": nodes.index[index], + "to_node_count": 0, + "to_node_type": node, + } + node_info = pd.concat( + [node_info, pd.DataFrame([new_row])], ignore_index=True + ) + + return node_info + @classmethod def _load(cls, filepath: Path | None) -> dict[str, Any]: context_file_loading.set({}) From 088b869ac46d0bb1947a4d7bf707b11c8885d8eb Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Mon, 2 Sep 2024 21:11:43 +0200 Subject: [PATCH 27/30] fix outlets after merge --- python/ribasim/tests/test_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ribasim/tests/test_validation.py b/python/ribasim/tests/test_validation.py index afb78a409..96dce8227 100644 --- a/python/ribasim/tests/test_validation.py +++ b/python/ribasim/tests/test_validation.py @@ -118,7 +118,7 @@ def test_minimum_flow_neighbor(): ) model.outlet.add( Node(2, Point(1.0, 0.0)), - [outlet.Static(flow_rate=[1e-3], min_crest_level=[2.0])], + [outlet.Static(flow_rate=[1e-3], min_upstream_level=[2.0])], ) model.terminal.add(Node(4, Point(3.0, -2.0))) @@ -147,7 +147,7 @@ def test_minimum_control_neighbor(): ) model.outlet.add( Node(2, Point(1.0, 0.0)), - [outlet.Static(flow_rate=[1e-3], min_crest_level=[2.0])], + [outlet.Static(flow_rate=[1e-3], min_upstream_level=[2.0])], ) model.terminal.add(Node(4, Point(3.0, -2.0))) From bae5d6f8679af780b9be00e90a0ca266cf538834 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 3 Sep 2024 10:03:55 +0200 Subject: [PATCH 28/30] 2nd round of refactoring --- python/ribasim/ribasim/model.py | 42 +++++++++++---------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/python/ribasim/ribasim/model.py b/python/ribasim/ribasim/model.py index 577a4c676..2b3588ea7 100644 --- a/python/ribasim/ribasim/model.py +++ b/python/ribasim/ribasim/model.py @@ -309,16 +309,16 @@ def _validate_model(self): ) df_graph = df_graph.rename(columns={"node_type": "to_node_type"}) - if not self._check_neighbor_amount( + if not self._has_valid_neighbor_amount( df_graph, flow_edge_neighbor_amount, "flow", df_node["node_type"] ): raise ValueError("Minimum flow inneighbor or outneighbor unsatisfied") - if not self._check_neighbor_amount( + if not self._has_valid_neighbor_amount( df_graph, control_edge_neighbor_amount, "control", df_node["node_type"] ): raise ValueError("Minimum control inneighbor or outneighbor unsatisfied") - def _check_neighbor_amount( + def _has_valid_neighbor_amount( self, df_graph: pd.DataFrame, edge_amount: dict[str, list[int]], @@ -395,31 +395,17 @@ def _add_source_sink_node( Specify that their occurrence in from_node table or to_node table is 0. """ - if direction == "from": - # loop over nodes, add the one that is not the downstream of any other nodes - for index, node in enumerate(nodes): - if nodes.index[index] not in node_info["from_node_id"].to_numpy(): - new_row = { - "from_node_id": nodes.index[index], - "from_node_count": 0, - "from_node_type": node, - } - node_info = pd.concat( - [node_info, pd.DataFrame([new_row])], ignore_index=True - ) - - elif direction == "to": - # loop over nodes, add the one that is not the downstream of any other nodes - for index, node in enumerate(nodes): - if nodes.index[index] not in node_info["to_node_id"].to_numpy(): - new_row = { - "to_node_id": nodes.index[index], - "to_node_count": 0, - "to_node_type": node, - } - node_info = pd.concat( - [node_info, pd.DataFrame([new_row])], ignore_index=True - ) + # loop over nodes, add the one that is not the downstream (from) or upstream (to) of any other nodes + for index, node in enumerate(nodes): + if nodes.index[index] not in node_info[f"{direction}_node_id"].to_numpy(): + new_row = { + f"{direction}_node_id": nodes.index[index], + f"{direction}_node_count": 0, + f"{direction}_node_type": node, + } + node_info = pd.concat( + [node_info, pd.DataFrame([new_row])], ignore_index=True + ) return node_info From 450d81c9d5848ec4eb37cf7577a5fed100598000 Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 3 Sep 2024 16:41:19 +0200 Subject: [PATCH 29/30] move validation function up a bit --- python/ribasim/ribasim/geometry/edge.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index 6185e99a7..c78213571 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -101,6 +101,7 @@ def add( edge_type = ( "control" if from_node.node_type in SPATIALCONTROLNODETYPES else "flow" ) + self._validate_edge(to_node, from_node, edge_type) assert self.df is not None if edge_id is None: edge_id = self._used_edge_ids.new_id() @@ -123,8 +124,6 @@ def add( index=pd.Index([edge_id], name="edge_id"), ) - self._validate_edge(to_node, from_node, edge_type) - self.df = GeoDataFrame[EdgeSchema](pd.concat([self.df, table_to_append])) if self.df.duplicated(subset=["from_node_id", "to_node_id"]).any(): raise ValueError( From 27610435ed98322d40d8f21ea304948ee547343d Mon Sep 17 00:00:00 2001 From: Jingru Feng Date: Tue, 3 Sep 2024 17:20:31 +0200 Subject: [PATCH 30/30] 3rd round of refactor --- python/ribasim/ribasim/geometry/edge.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/python/ribasim/ribasim/geometry/edge.py b/python/ribasim/ribasim/geometry/edge.py index c78213571..b2a3b62bd 100644 --- a/python/ribasim/ribasim/geometry/edge.py +++ b/python/ribasim/ribasim/geometry/edge.py @@ -143,23 +143,27 @@ def _validate_edge(self, to_node: NodeData, from_node: NodeData, edge_type: str) & (self.df["edge_type"] == edge_type) ].shape[0] # validation on neighbor amount + max_in_flow: int = flow_edge_neighbor_amount[to_node.node_type][1] + max_out_flow: int = flow_edge_neighbor_amount[from_node.node_type][3] + max_in_control: int = control_edge_neighbor_amount[to_node.node_type][1] + max_out_control: int = control_edge_neighbor_amount[from_node.node_type][3] if edge_type == "flow": - if in_neighbor >= flow_edge_neighbor_amount[to_node.node_type][1]: + if in_neighbor >= max_in_flow: raise ValueError( - f"Node {to_node.node_id} can have at most {flow_edge_neighbor_amount[to_node.node_type][1]} flow edge inneighbor(s) (got {in_neighbor})" + f"Node {to_node.node_id} can have at most {max_in_flow} flow edge inneighbor(s) (got {in_neighbor})" ) - if out_neighbor >= flow_edge_neighbor_amount[from_node.node_type][3]: + if out_neighbor >= max_out_flow: raise ValueError( - f"Node {from_node.node_id} can have at most {flow_edge_neighbor_amount[from_node.node_type][3]} flow edge outneighbor(s) (got {out_neighbor})" + f"Node {from_node.node_id} can have at most {max_out_flow} flow edge outneighbor(s) (got {out_neighbor})" ) elif edge_type == "control": - if in_neighbor >= control_edge_neighbor_amount[to_node.node_type][1]: + if in_neighbor >= max_in_control: raise ValueError( - f"Node {to_node.node_id} can have at most {control_edge_neighbor_amount[to_node.node_type][1]} control edge inneighbor(s) (got {in_neighbor})" + f"Node {to_node.node_id} can have at most {max_in_control} control edge inneighbor(s) (got {in_neighbor})" ) - if out_neighbor >= control_edge_neighbor_amount[from_node.node_type][3]: + if out_neighbor >= max_out_control: raise ValueError( - f"Node {from_node.node_id} can have at most {control_edge_neighbor_amount[from_node.node_type][3]} control edge outneighbor(s) (got {out_neighbor})" + f"Node {from_node.node_id} can have at most {max_out_control} control edge outneighbor(s) (got {out_neighbor})" ) def _get_where_edge_type(self, edge_type: str) -> NDArray[np.bool_]: