From e89549822bcadcc288cf06be3309e3296259a7a6 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Thu, 14 Sep 2023 12:56:26 +0200 Subject: [PATCH 1/6] Sort DataFrames inplace. --- .pre-commit-config.yaml | 2 +- python/ribasim/ribasim/input_base.py | 2 +- python/ribasim/ribasim/node_types/basin.py | 10 +++++----- python/ribasim/ribasim/node_types/fractional_flow.py | 2 +- .../ribasim/node_types/tabulated_rating_curve.py | 9 ++++++--- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ca7d3efd..b41824a4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: rev: v0.0.286 hooks: - id: ruff - args: [--fix, --exit-non-zero-on-fix] + args: [--fix, --exit-non-zero-on-fix, --ignore=PD002] - repo: https://github.com/psf/black rev: 23.7.0 hooks: diff --git a/python/ribasim/ribasim/input_base.py b/python/ribasim/ribasim/input_base.py index f4acc07d6..17a5f808c 100644 --- a/python/ribasim/ribasim/input_base.py +++ b/python/ribasim/ribasim/input_base.py @@ -197,5 +197,5 @@ def sort(self): if dataframe is None: continue else: - dataframe = dataframe.sort_values("node_id", ignore_index=True) + dataframe.sort_values("node_id", ignore_index=True, inplace=True) return diff --git a/python/ribasim/ribasim/node_types/basin.py b/python/ribasim/ribasim/node_types/basin.py index 900b407c3..18737e008 100644 --- a/python/ribasim/ribasim/node_types/basin.py +++ b/python/ribasim/ribasim/node_types/basin.py @@ -35,12 +35,12 @@ class Basin(TableModel): state: Optional[DataFrame[BasinStateSchema]] = None def sort(self): - self.profile = self.profile.sort_values(["node_id", "level"], ignore_index=True) + self.profile.sort_values(["node_id", "level"], ignore_index=True, inplace=True) if self.static is not None: - self.static = self.static.sort_values("node_id", ignore_index=True) + self.static.sort_values("node_id", ignore_index=True, inplace=True) if self.forcing is not None: - self.forcing = self.forcing.sort_values( - ["time", "node_id"], ignore_index=True + self.forcing.sort_values( + ["time", "node_id"], ignore_index=True, inplace=True ) if self.state is not None: - self.state = self.state.sort_values("node_id", ignore_index=True) + self.state.sort_values("node_id", ignore_index=True, inplace=True) diff --git a/python/ribasim/ribasim/node_types/fractional_flow.py b/python/ribasim/ribasim/node_types/fractional_flow.py index 8e6648701..4fa3ecf16 100644 --- a/python/ribasim/ribasim/node_types/fractional_flow.py +++ b/python/ribasim/ribasim/node_types/fractional_flow.py @@ -19,4 +19,4 @@ class FractionalFlow(TableModel): static: DataFrame[FractionalFlowStaticSchema] def sort(self): - self.static = self.static.sort_values("node_id", ignore_index=True) + self.static.sort_values("node_id", ignore_index=True, inplace=True) diff --git a/python/ribasim/ribasim/node_types/tabulated_rating_curve.py b/python/ribasim/ribasim/node_types/tabulated_rating_curve.py index 765a45965..9bbb7ec7b 100644 --- a/python/ribasim/ribasim/node_types/tabulated_rating_curve.py +++ b/python/ribasim/ribasim/node_types/tabulated_rating_curve.py @@ -27,8 +27,11 @@ class TabulatedRatingCurve(TableModel): time: Optional[DataFrame[TabulatedRatingCurveTimeSchema]] = None def sort(self): - self.static = self.static.sort_values(["node_id", "level"], ignore_index=True) + if self.static: + self.static.sort_values( + ["node_id", "level"], ignore_index=True, inplace=True + ) if self.time is not None: - self.time = self.time.sort_values( - ["time", "node_id", "level"], ignore_index=True + self.time.sort_values( + ["time", "node_id", "level"], ignore_index=True, inplace=True ) From 9cdce72a500504cdd77b91062c548dc79b838eb5 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Thu, 14 Sep 2023 13:03:26 +0200 Subject: [PATCH 2/6] Fix static check. --- python/ribasim/ribasim/node_types/tabulated_rating_curve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ribasim/ribasim/node_types/tabulated_rating_curve.py b/python/ribasim/ribasim/node_types/tabulated_rating_curve.py index 9bbb7ec7b..cb88f888d 100644 --- a/python/ribasim/ribasim/node_types/tabulated_rating_curve.py +++ b/python/ribasim/ribasim/node_types/tabulated_rating_curve.py @@ -27,7 +27,7 @@ class TabulatedRatingCurve(TableModel): time: Optional[DataFrame[TabulatedRatingCurveTimeSchema]] = None def sort(self): - if self.static: + if self.static is not None: self.static.sort_values( ["node_id", "level"], ignore_index=True, inplace=True ) From cc44c49378b12f407c2d862b9bca0061ead9534b Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Thu, 14 Sep 2023 14:58:27 +0200 Subject: [PATCH 3/6] address review comment --- .pre-commit-config.yaml | 2 +- ruff.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b41824a4c..3ca7d3efd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: rev: v0.0.286 hooks: - id: ruff - args: [--fix, --exit-non-zero-on-fix, --ignore=PD002] + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black rev: 23.7.0 hooks: diff --git a/ruff.toml b/ruff.toml index bc0f750e8..20e78ea84 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,5 @@ select = ["D", "E", "F", "NPY", "PD", "C4", "I"] -ignore = ["D1", "D202", "D205", "D400", "D404", "E501", "PD901"] +ignore = ["D1", "D202", "D205", "D400", "D404", "E501", "PD002", "PD901"] fixable = ["I"] [pydocstyle] From 03d2eb7928a3e791e0e5b3ae2ee2172e184fb1be Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Thu, 14 Sep 2023 18:07:33 +0200 Subject: [PATCH 4/6] restructure headings of usage page So the tables are subheadings of the node types. --- docs/core/usage.qmd | 162 +++++++++++++++++++------------------------- 1 file changed, 69 insertions(+), 93 deletions(-) diff --git a/docs/core/usage.qmd b/docs/core/usage.qmd index 526103d24..daedbe09d 100644 --- a/docs/core/usage.qmd +++ b/docs/core/usage.qmd @@ -20,10 +20,7 @@ This will give the following message: Usage: ribasim 'path/to/config.toml' ``` - -## Input and output files - -### Configuration file +# Configuration file Ribasim has a single configuration file, which is written in the [TOML](https://toml.io/) format. It contains settings, as well as paths to other input and output files. @@ -31,7 +28,7 @@ format. It contains settings, as well as paths to other input and output files. ```{.toml include="../../core/test/docs.toml"} ``` -#### Solver settings +## Solver settings The solver section in the configuration file is entirely optional, since we aim to use defaults that will generally work well. Common reasons to modify the solver settings are to adjust the calculation or output stepsizes: `adaptive`, `dt`, and `saveat`. @@ -66,7 +63,7 @@ The total maximum number of iterations `maxiters = 1e9`, can normally stay as-is The absolute and relative tolerance for adaptive timestepping can be set with `abstol` and `reltol`. For more information on these and other solver options, see the [DifferentialEquations.jl docs](https://docs.sciml.ai/DiffEqDocs/latest/basics/common_solver_opts/#solver_options). -### GeoPackage and Arrow tables +# GeoPackage and Arrow tables The input and output tables described below all share that they are tabular files. The Node and Edge tables always have to be in the [GeoPackage](https://www.geopackage.org/) file, and @@ -108,7 +105,7 @@ Tables are also allowed to have rows for timestamps that are not part of the sim these will be ignored. That makes it easy to prepare data for a larger period, and test models on a shorted period. -### Node +# Node Node is a table that specifies the ID and type of each node of a model. The ID must be unique among all nodes, and the type must be one of the available node types listed below. @@ -166,7 +163,7 @@ Adding a geometry to the node table can be helpful to examine models in [QGIS](https://qgis.org/en/site/), as it will show the location of the nodes on the map. The geometry is not used by Ribasim. -### Edge +# Edge Edges define connections between nodes. The only thing that defines an edge is the nodes it connects, and in what direction. There are currently 2 possible edge types: @@ -194,20 +191,7 @@ geom | geometry | (optional) Similarly to the node table, you can use a geometry to visualize the connections between the nodes in QGIS. For instance, you can draw a line connecting the two node coordinates. -### Basin / state - -The state table aims to capture the full state of the Basin, such that it can be used as an -initial condition, potentially the outcome of an earlier simulation. Currently only the -Basin node types have state. - -column | type | unit | restriction ---------- | ------- | ------------ | ----------- -node_id | Int | - | sorted -level | Float64 | $m$ | $\ge$ basin bottom - -Each Basin ID needs to be in the table. - -### Basin +# Basin The Basin table can be used to set the static value of variables. The forcing table has a similar schema, with the time column added. A static value for a variable is only used if @@ -227,7 +211,7 @@ Note that if variables are not set in the static table, default values are used possible. These are generally zero, e.g. no precipitation, no inflow. If it is not possible to have a reasonable and safe default, a value must be provided in the static table. -### Basin / forcing +## Basin / forcing This table is the transient form of the `Basin` table. The only difference is that a time column is added. @@ -235,7 +219,20 @@ The table must by sorted by time, and per time it must be sorted by `node_id`. A linear interpolation between the given timesteps is currently done if the solver takes timesteps between the given data points. More options will be available later. -### Basin / profile +## Basin / state + +The state table aims to capture the full state of the Basin, such that it can be used as an +initial condition, potentially the outcome of an earlier simulation. Currently only the +Basin node types have state. + +column | type | unit | restriction +--------- | ------- | ------------ | ----------- +node_id | Int | - | sorted +level | Float64 | $m$ | $\ge$ basin bottom + +Each Basin ID needs to be in the table. + +## Basin / profile The profile table defines the physical dimensions of the storage reservoir of each basin. @@ -263,7 +260,36 @@ Internally this get converted to two functions, $A(S)$ and $h(S)$, by integratin The minimum area cannot be zero to avoid numerical issues. The maximum area is used to convert the precipitation flux into an inflow. -### FractionalFlow +## Basin output + +The basin table contains outputs of the storage and level of each basin at every solver +timestep. The initial condition is also written to the file. + +column | type | unit +-------- | -------- | ---- +time | DateTime | - +node_id | Int | - +storage | Float64 | $m^3$ +level | Float64 | $m$ + +The table is sorted by time, and per time it is sorted by `node_id`. + +## Flow output + +The flow table contains outputs of the flow on every edge in the model, for each solver +timestep. + +column | type | unit +------------- | -------- | ---- +time | DateTime | - +edge_id | Int | - +from_node_id | Int | - +to_node_id | Int | - +flow | Float64 | $m^3 s^{-1}$ + +The table is sorted by time, and per time the same edge_id order is used, though not sorted. + +# FractionalFlow Lets a fraction (in [0,1]) of the incoming flow trough. @@ -273,8 +299,7 @@ node_id | Int | - | sorted fraction | Float64 | - | in the interval [0,1] control_state | String | - | (optional) - -### TabulatedRatingCurve +# TabulatedRatingCurve This table is similar in structure to the Basin profile. The TabulatedRatingCurve gives a relation between the storage of a connected Basin (via the outlet level) and its outflow. @@ -295,7 +320,7 @@ node_id | discharge | level 2 | 0.942702 | 20.095 3 | 0.0 | 2.129 -### TabulatedRatingCurve / time +## TabulatedRatingCurve / time This table is the transient form of the `TabulatedRatingCurve` table. The only difference is that a time column is added. @@ -310,8 +335,7 @@ node_id | Int | - | sorted per time level | Float64 | $m$ | - discharge | Float64 | $m^3 s^{-1}$ | non-negative - -### Pump +# Pump Pump water from a source node to a destination node. The set flow rate will be pumped unless the intake storage is less than $10~m^3$, @@ -328,7 +352,7 @@ min_flow_rate | Float64 | $m^3 s^{-1}$ | (optional, default 0.0) max_flow_rate | Float64 | $m^3 s^{-1}$ | (optional) control_state | String | - | (optional) -### Outlet +# Outlet The outlet is very similar to the pump. The outlet has two additional physical constraints: water only flows trough the outlet when the head difference is positive (i.e. water flows down by gravity), and the upstream level must be above the minimum crest level if the upstream level is defined. When PID controlled, the outlet must point towards the controlled basin in terms of edges. @@ -343,7 +367,7 @@ max_flow_rate | Float64 | $m^3 s^{-1}$ | (optional) min_crest_level | Float64 | $m$ | (optional) control_state | String | - | (optional) -### LevelBoundary +# LevelBoundary Acts like an infinitely large basin where the level does not change by flow. This can be connected to a basin via a `LinearResistance`. @@ -356,8 +380,7 @@ node_id | Int | - | sorted active | Bool | - | (optional, default true) level | Float64 | $m^3$ | - - -### LevelBoundary / time +## LevelBoundary / time This table is the transient form of the `LevelBoundary` table. The only difference is that a time column is added and activity is assumed to be true. @@ -373,9 +396,7 @@ time | DateTime | - | sorted node_id | Int | - | sorted per time level | Float64 | $m^3 s^{-1}$ | - - - -### FlowBoundary +# FlowBoundary Pump water to a destination node. We require that the edge connecting the flow boundary to the Basin should point towards the basin, @@ -390,7 +411,7 @@ node_id | Int | - | sorted active | Bool | - | (optional, default true) flow_rate | Float64 | $m^3 s^{-1}$ | non-negative -### FlowBoundary / time +## FlowBoundary / time This table is the transient form of the `FlowBoundary` table. The only differences are that a time column is added and the nodes are assumed to be active so this column is removed. @@ -406,7 +427,7 @@ time | DateTime | - | sorted node_id | Int | - | sorted per time flow_rate | Float64 | $m^3 s^{-1}$ | non-negative -### LinearResistance +# LinearResistance Flow proportional to the level difference between the connected basins. @@ -417,8 +438,7 @@ active | Bool | - | (optional, default true) resistance | Float64 | $sm^{-2}$ | - control_state | String | - | (optional) - -### ManningResistance +# ManningResistance Flow through this connection is estimated by conservation of energy and the Manning-Gauckler formula to estimate friction losses. @@ -432,7 +452,7 @@ profile_with | Float64 | $m$ | positive profile_slope | Float64 | - | - control_state | String | - | (optional) -### Terminal +# Terminal A terminal is a water sink without state or properties. Any water that flows into a terminal node is removed from the model. @@ -443,40 +463,11 @@ column | type | unit | restriction --------- | ------- | ------------ | ----------- node_id | Int | - | sorted -### Basin output - -The basin table contains outputs of the storage and level of each basin at every solver -timestep. The initial condition is also written to the file. - -column | type | unit --------- | -------- | ---- -time | DateTime | - -node_id | Int | - -storage | Float64 | $m^3$ -level | Float64 | $m$ - -The table is sorted by time, and per time it is sorted by `node_id`. - -### Flow output - -The flow table contains outputs of the flow on every edge in the model, for each solver -timestep. - -column | type | unit -------------- | -------- | ---- -time | DateTime | - -edge_id | Int | - -from_node_id | Int | - -to_node_id | Int | - -flow | Float64 | $m^3 s^{-1}$ - -The table is sorted by time, and per time the same edge_id order is used, though not sorted. - -### DisceteControl +# DisceteControl DiscreteControl is implemented based on [VectorContinuousCallback](https://docs.sciml.ai/DiffEqDocs/stable/features/callback_functions/#VectorContinuousCallback). -#### DiscreteControl / condition {#sec-condition} +## DiscreteControl / condition {#sec-condition} The condition schema defines conditions of the form 'the discrete_control node with this node id listens to whether the given variable of the node with the given listen feature id is grater than the given value'. If the condition variable comes from a time-series, a look ahead $\Delta t$ can be supplied. @@ -488,7 +479,7 @@ variable | String | - | must be "level" or "flow_rate" greater_than | Float64 | various | - look_ahead | Float64 | $s$ | Only on transient boundary conditions, non-negative (optional, default 0) -#### DiscreteControl / logic +## DiscreteControl / logic The logic schema defines which control states are triggered based on the truth of the conditions a discrete_control node listens to. DiscreteControl is applied in the Julia core as follows: @@ -507,8 +498,7 @@ node_id | Int | - | sorted truth_state | String | - | Consists of the characters "T" (true), "F" (false), "U" (upcrossing), "D" (downcrossing) and "*" (any) control_state | String | - | - -#### DiscreteControl output +## DiscreteControl output The control table contains a record of each change of control state: when it happened, which control node was involved, to which control state it changed and based on which truth state. @@ -519,9 +509,7 @@ control_node_id | Int truth_state | String control_state | String - - -### PidControl +# PidControl The PidControl node controls the level in a basin by continuously controlling the flow rate of a connected pump or outlet. See also [PID controller](https://en.wikipedia.org/wiki/PID_controller). When A PidControl node is made inactive, the node under its control retains the last flow rate value, and the error integral is reset to 0. @@ -540,8 +528,7 @@ integral | Float64 | $s^{-2}$ | - derivative | Float64 | - | - control_state | String | - | - - -### PidControl / time +## PidControl / time This table is the transient form of the `PidControl` table. The differences are that a time column is added and the nodes are assumed to be active so this column is removed. @@ -553,21 +540,10 @@ Note that a `node_id` can be either in this table or in the static one, but not column | type | unit | restriction -------------- | -------- | -------- | ----------- -node_id | Int | - | sorted +node_id | Int | - | sorted per time time | DateTime | - | sorted listen_node_id | Int | - | - target | Float64 | $m$ | - proportional | Float64 | $s^{-1}$ | - integral | Float64 | $s^{-2}$ | - derivative | Float64 | - | - - - -## Example input files - -From [this link](https://github.com/visr/ribasim-artifacts/releases) you can download an -existing schematization for the Netherlands that was used for testing purposes during -development. It is provided here as an example to help people get started. Based on the -description of the input files above, you can also generate your own schematization using -your tools of choice. For Python users -[ribasim-python](https://github.com/Deltares/ribasim-python) was created to make it easy to -do pre- and postprocessing. From dc0c288bf04750e64837d7afc40039f7d42f85d5 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Thu, 14 Sep 2023 18:13:10 +0200 Subject: [PATCH 5/6] update sorting methods --- python/ribasim/ribasim/node_types/flow_boundary.py | 6 ++++++ python/ribasim/ribasim/node_types/fractional_flow.py | 3 --- python/ribasim/ribasim/node_types/level_boundary.py | 6 ++++++ python/ribasim/ribasim/node_types/pid_control.py | 6 ++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/python/ribasim/ribasim/node_types/flow_boundary.py b/python/ribasim/ribasim/node_types/flow_boundary.py index f78f913ca..e53520fd7 100644 --- a/python/ribasim/ribasim/node_types/flow_boundary.py +++ b/python/ribasim/ribasim/node_types/flow_boundary.py @@ -25,3 +25,9 @@ class FlowBoundary(TableModel): static: Optional[DataFrame[FlowBoundaryStaticSchema]] = None time: Optional[DataFrame[FlowBoundaryTimeSchema]] = None + + def sort(self): + if self.static is not None: + self.static.sort_values("node_id", ignore_index=True, inplace=True) + if self.time is not None: + self.time.sort_values(["time", "node_id"], ignore_index=True, inplace=True) diff --git a/python/ribasim/ribasim/node_types/fractional_flow.py b/python/ribasim/ribasim/node_types/fractional_flow.py index 4fa3ecf16..e24eecde5 100644 --- a/python/ribasim/ribasim/node_types/fractional_flow.py +++ b/python/ribasim/ribasim/node_types/fractional_flow.py @@ -17,6 +17,3 @@ class FractionalFlow(TableModel): """ static: DataFrame[FractionalFlowStaticSchema] - - def sort(self): - self.static.sort_values("node_id", ignore_index=True, inplace=True) diff --git a/python/ribasim/ribasim/node_types/level_boundary.py b/python/ribasim/ribasim/node_types/level_boundary.py index c83b9e064..3437acf7b 100644 --- a/python/ribasim/ribasim/node_types/level_boundary.py +++ b/python/ribasim/ribasim/node_types/level_boundary.py @@ -23,3 +23,9 @@ class LevelBoundary(TableModel): static: Optional[DataFrame[LevelBoundaryStaticSchema]] = None time: Optional[DataFrame[LevelBoundaryTimeSchema]] = None + + def sort(self): + if self.static is not None: + self.static.sort_values("node_id", ignore_index=True, inplace=True) + if self.time is not None: + self.time.sort_values(["time", "node_id"], ignore_index=True, inplace=True) diff --git a/python/ribasim/ribasim/node_types/pid_control.py b/python/ribasim/ribasim/node_types/pid_control.py index a75eae42d..f70dbf3e3 100644 --- a/python/ribasim/ribasim/node_types/pid_control.py +++ b/python/ribasim/ribasim/node_types/pid_control.py @@ -26,3 +26,9 @@ class PidControl(TableModel): class Config: validate_assignment = True + + def sort(self): + if self.static is not None: + self.static.sort_values("node_id", ignore_index=True, inplace=True) + if self.time is not None: + self.time.sort_values(["time", "node_id"], ignore_index=True, inplace=True) From fe642699f099027c919f0ce70a284e664b824407 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Thu, 14 Sep 2023 18:40:28 +0200 Subject: [PATCH 6/6] sort LevelBoundaryTime and PidControlTime by time first --- core/src/validation.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/validation.jl b/core/src/validation.jl index cafabd4db..4f422a5d6 100644 --- a/core/src/validation.jl +++ b/core/src/validation.jl @@ -321,7 +321,13 @@ sort_by_function(table::StructVector{<:Legolas.AbstractRecord}) = sort_by_id sort_by_function(table::StructVector{TabulatedRatingCurveStaticV1}) = sort_by_id_state_level sort_by_function(table::StructVector{BasinProfileV1}) = sort_by_id_level -const TimeSchemas = Union{TabulatedRatingCurveTimeV1, FlowBoundaryTimeV1, BasinForcingV1} +const TimeSchemas = Union{ + BasinForcingV1, + FlowBoundaryTimeV1, + LevelBoundaryTimeV1, + PidControlTimeV1, + TabulatedRatingCurveTimeV1, +} function sort_by_function(table::StructVector{<:TimeSchemas}) return sort_by_time_id