From d7af65a253a68a7fae66f7bd079521811ab1b70d Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Mon, 6 Nov 2023 20:32:03 +0100 Subject: [PATCH 01/21] Add allocation output --- core/src/allocation.jl | 35 +++++++++++++++++++++++++++++++---- core/src/bmi.jl | 7 ++++++- core/src/config.jl | 1 + core/src/create.jl | 12 ++++++++++++ core/src/io.jl | 17 +++++++++++++++++ core/src/solve.jl | 27 ++++++++++++++++++++++++++- core/test/docs.toml | 1 + 7 files changed, 94 insertions(+), 6 deletions(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index d66ca698e..22aebde95 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -640,6 +640,7 @@ An AllocationModel object. """ function AllocationModel( + allocation_network_id::Int, p::Parameters, subnetwork_node_ids::Vector{Int}, source_edge_ids::Vector{Int}, @@ -667,6 +668,7 @@ function AllocationModel( ) return AllocationModel( + allocation_network_id, subnetwork_node_ids, node_id_mapping, node_id_mapping_inverse, @@ -707,13 +709,37 @@ end """ Assign the allocations to the users as determined by the solution of the allocation problem. """ -function assign_allocations!(allocation_model::AllocationModel, user::User)::Nothing +function assign_allocations!( + allocation_model::AllocationModel, + p::Parameters, + t::Float64, + priority_idx::Int, +)::Nothing (; problem, allocgraph_edge_ids_user_demand, node_id_mapping_inverse) = allocation_model + (; connectivity, user) = p + (; graph_flow, flow) = connectivity + (; record) = user F = problem[:F] + flow = get_tmp(flow, 0) for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand model_node_id = node_id_mapping_inverse[allocgraph_node_id][1] user_idx = findsorted(user.node_id, model_node_id) - user.allocated[user_idx] .= JuMP.value(F[allocgraph_edge_id]) + allocated = JuMP.value(F[allocgraph_edge_id]) + user.allocated[user_idx][priority_idx] = allocated + + # Save allocations to record + push!(record.time, t) + push!(record.allocation_network_id, allocation_model.allocation_network_id) + push!(record.user_node_id, model_node_id) + push!(record.priority, user.priorities[priority_idx]) + push!(record.demand, user.demand[user_idx][priority_idx](t)) + push!(record.allocated, allocated) + # Note: This is now the last abstraction before the allocation update, + # should be the average abstraction since the last allocation solve + push!( + record.abstracted, + flow[only(inneighbors(graph_flow, model_node_id)), model_node_id], + ) end return nothing end @@ -778,6 +804,7 @@ and flows, solve the allocation problem and assign the results to the users. function allocate!(p::Parameters, allocation_model::AllocationModel, t::Float64)::Nothing (; user) = p (; problem) = allocation_model + (; priorities, record) = user set_source_flows!(allocation_model, p) @@ -786,7 +813,7 @@ function allocate!(p::Parameters, allocation_model::AllocationModel, t::Float64) # in the flow_conservation constraints if the net flow is positive. # Solve this as a separate problem before the priorities below - for priority_idx in eachindex(user.priorities) + for priority_idx in eachindex(priorities) # Subtract the flows used by the allocation of the previous priority from the capacities of the edges # or set edge capacities if priority_idx = 1 adjust_edge_capacities!(allocation_model, priority_idx) @@ -801,6 +828,6 @@ function allocate!(p::Parameters, allocation_model::AllocationModel, t::Float64) end # Assign the allocations to the users for this priority - assign_allocations!(allocation_model, p.user) + assign_allocations!(allocation_model, p, t, priority_idx) end end diff --git a/core/src/bmi.jl b/core/src/bmi.jl index 1c762ecd5..84127c67a 100644 --- a/core/src/bmi.jl +++ b/core/src/bmi.jl @@ -167,6 +167,11 @@ function BMI.finalize(model::Model)::Model path = results_path(config, results.control) write_arrow(path, table, compress) + # allocation + table = allocation_table(model) + path = results_path(config, results.allocation) + write_arrow(path, table, compress) + @debug "Wrote results." return model end @@ -217,7 +222,7 @@ function create_callbacks( allocation_cb = PeriodicCallback( update_allocation!, config.allocation.timestep; - initial_affect = true, + initial_affect = false, ) push!(callbacks, allocation_cb) end diff --git a/core/src/config.jl b/core/src/config.jl index d9743b446..f09327eac 100644 --- a/core/src/config.jl +++ b/core/src/config.jl @@ -112,6 +112,7 @@ end basin::String = "results/basin.arrow" flow::String = "results/flow.arrow" control::String = "results/control.arrow" + allocation::String = "results/allocation.arrow" outstate::Union{String, Nothing} = nothing compression::Compression = "zstd" compression_level::Int = 6 diff --git a/core/src/create.jl b/core/src/create.jl index 67c8dc248..af2b121e8 100644 --- a/core/src/create.jl +++ b/core/src/create.jl @@ -282,6 +282,7 @@ function generate_allocation_models!(p::Parameters, db::DB, config::Config)::Not push!( connectivity.allocation_models, AllocationModel( + allocation_network_id, p, allocation_group_node.fid, source_edge_ids, @@ -780,6 +781,16 @@ function User(db::DB, config::Config)::User allocated = [zeros(length(priorities)) for id in node_ids] + record = ( + time = Vector{Float64}(), + allocation_network_id = Vector{Int}(), + user_node_id = Vector{Int}(), + priority = Vector{Int}(), + demand = Vector{Float64}(), + allocated = Vector{Float64}(), + abstracted = Vector{Float64}(), + ) + return User( node_ids, active, @@ -788,6 +799,7 @@ function User(db::DB, config::Config)::User return_factor, min_level, priorities, + record, ) end diff --git a/core/src/io.jl b/core/src/io.jl index 3d0e0631e..66d315cc0 100644 --- a/core/src/io.jl +++ b/core/src/io.jl @@ -210,6 +210,23 @@ function discrete_control_table(model::Model)::NamedTuple return (; time, record.control_node_id, record.truth_state, record.control_state) end +"Create an allocation result table for the saved data" +function allocation_table(model::Model)::NamedTuple + (; config) = model + (; record) = model.integrator.p.user + + time = datetime_since.(record.time, config.starttime) + return (; + time, + record.allocation_network_id, + record.user_node_id, + record.priority, + record.demand, + record.allocated, + record.abstracted, + ) +end + "Write a result table to disk as an Arrow file" function write_arrow( path::AbstractString, diff --git a/core/src/solve.jl b/core/src/solve.jl index db3c5c957..ffdbf31b6 100644 --- a/core/src/solve.jl +++ b/core/src/solve.jl @@ -7,6 +7,7 @@ const VectorInterpolation = """ Store information for a subnetwork used for allocation. +allocation_network_id: The ID of this allocation network node_id: All the IDs of the nodes that are in this subnetwork node_id_mapping: Mapping Dictionary; model_node_id => AG_node_id where such a correspondence exists (all AG node ids are in the values) @@ -19,6 +20,7 @@ problem: The JuMP.jl model for solving the allocation problem Δt_allocation: The time interval between consecutive allocation solves """ struct AllocationModel + allocation_network_id::Int node_id::Vector{Int} node_id_mapping::Dict{Int, Tuple{Int, Symbol}} node_id_mapping_inverse::Dict{Int, Tuple{Int, Symbol}} @@ -449,6 +451,26 @@ struct User <: AbstractParameterNode return_factor::Vector{Float64} min_level::Vector{Float64} priorities::Vector{Int} + record::NamedTuple{ + ( + :time, + :allocation_network_id, + :user_node_id, + :priority, + :demand, + :allocated, + :abstracted, + ), + Tuple{ + Vector{Float64}, + Vector{Int}, + Vector{Int}, + Vector{Int}, + Vector{Float64}, + Vector{Float64}, + Vector{Float64}, + }, + } end # TODO Automatically add all nodetypes here @@ -989,7 +1011,10 @@ function formulate_flow!( for (i, id) in enumerate(node_id) downstream_id = only(outneighbors(graph_flow, id)) upstream_id = only(inneighbors(graph_flow, id)) - flow[id, downstream_id] = flow[upstream_id, id] * fraction[i] + # overwrite the inflow such that flow is conserved over the FractionalFlow + outflow = flow[upstream_id, id] * fraction[i] + flow[upstream_id, id] = outflow + flow[id, downstream_id] = outflow end return nothing end diff --git a/core/test/docs.toml b/core/test/docs.toml index 6b9124cf0..0338e085d 100644 --- a/core/test/docs.toml +++ b/core/test/docs.toml @@ -40,5 +40,6 @@ timing = false # optional, whether to log debug timing statements flow = "results/flow.arrow" # optional, default "results/flow.arrow" basin = "results/basin.arrow" # optional, default "results/basin.arrow" control = "results/control.arrow" # optional, default "results/control.arrow" +allocation = "results/allocation.arrow" # optional, default "results/allocation.arrow" compression = "zstd" # optional, default "zstd", also supports "lz4" compression_level = 6 # optional, default 6 From b0c549a7676ebcd0a2cd0c9d544c18c57e454d9f Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 7 Nov 2023 13:55:45 +0100 Subject: [PATCH 02/21] Add test --- core/src/solve.jl | 5 - core/test/allocation.jl | 29 +++- .../ribasim_testmodels/__init__.py | 2 + .../ribasim_testmodels/allocation.py | 136 ++++++++++++++++++ 4 files changed, 165 insertions(+), 7 deletions(-) diff --git a/core/src/solve.jl b/core/src/solve.jl index ffdbf31b6..fecf16214 100644 --- a/core/src/solve.jl +++ b/core/src/solve.jl @@ -809,11 +809,6 @@ function formulate_flow!( continue end - # For now allocated = demand - for priority in eachindex(allocated[i]) - allocated[i][priority] = demand[i][priority](t) - end - q = sum(allocated[i]) # Smoothly let abstraction go to 0 as the source basin dries out diff --git a/core/test/allocation.jl b/core/test/allocation.jl index 4228d2372..d3946e57f 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -22,7 +22,32 @@ using PreallocationTools: get_tmp @test F ≈ [0.0, 4.0, 0.0, 0.0, 0.0, 4.5] allocated = p.user.allocated - @test allocated[1] ≈ [4.0, 4.0] - @test allocated[2] ≈ [0.0, 0.0] + @test allocated[1] ≈ [0.0, 4.0] + @test allocated[2] ≈ [4.0, 0.0] @test allocated[3] ≈ [0.0, 0.0] end + +@testset "Simulation with allocation" begin + toml_path = + normpath(@__DIR__, "../../generated_testmodels/simple_subnetwork/ribasim.toml") + @test ispath(toml_path) + model = Ribasim.run(toml_path) + record = model.integrator.p.user.record + where_5 = (record.user_node_id .== 5) + + @test all(record.demand[where_5] .== 1.0e-3) + @test all( + isapprox( + record.allocated[where_5], + collect(range(1.0e-3, 0.0, sum(where_5))); + rtol = 0.01, + ), + ) + @test all( + isapprox( + record.abstracted[where_5], + collect(range(1.0e-3, 0.0, sum(where_5))); + rtol = 0.1, + ), + ) +end diff --git a/python/ribasim_testmodels/ribasim_testmodels/__init__.py b/python/ribasim_testmodels/ribasim_testmodels/__init__.py index 383928301..064e2021f 100644 --- a/python/ribasim_testmodels/ribasim_testmodels/__init__.py +++ b/python/ribasim_testmodels/ribasim_testmodels/__init__.py @@ -6,6 +6,7 @@ import ribasim_testmodels from ribasim_testmodels.allocation import ( + simple_subnetwork_model, # looped_subnetwork_model, subnetwork_model, user_model, @@ -74,6 +75,7 @@ "outlet_model", "user_model", "subnetwork_model", + "simple_subnetwork_model", # Disable until this issue is resolved: # https://github.com/Deltares/Ribasim/issues/692 # "looped_subnetwork_model", diff --git a/python/ribasim_testmodels/ribasim_testmodels/allocation.py b/python/ribasim_testmodels/ribasim_testmodels/allocation.py index 5ea605a3e..5c17fc5e2 100644 --- a/python/ribasim_testmodels/ribasim_testmodels/allocation.py +++ b/python/ribasim_testmodels/ribasim_testmodels/allocation.py @@ -530,3 +530,139 @@ def looped_subnetwork_model(): ) return model + + +def simple_subnetwork_model(): + xy = np.array( + [ + (0.0, 0.0), # 1: FlowBoundary + (0.0, 1.0), # 2: Basin + (0.0, 2.0), # 3: Pump + (0.0, 3.0), # 4: Basin + (-1.0, 4.0), # 5: User + (1.0, 4.0), # 6: User + ] + ) + node_xy = gpd.points_from_xy(x=xy[:, 0], y=xy[:, 1]) + + node_type = ["FlowBoundary", "Basin", "Pump", "Basin", "User", "User"] + + # Make sure the feature id starts at 1: explicitly give an index. + node = ribasim.Node( + static=gpd.GeoDataFrame( + data={"type": node_type, "allocation_network_id": 1}, + index=pd.Index(np.arange(len(xy)) + 1, name="fid"), + geometry=node_xy, + crs="EPSG:28992", + ) + ) + + # Setup the edges: + from_id = np.array( + [1, 2, 3, 4, 4, 5, 6], + dtype=np.int64, + ) + to_id = np.array( + [2, 3, 4, 5, 6, 4, 4], + dtype=np.int64, + ) + allocation_network_id = len(from_id) * [None] + allocation_network_id[0] = 1 + lines = ribasim.utils.geometry_from_connectivity(node, from_id, to_id) + edge = ribasim.Edge( + static=gpd.GeoDataFrame( + data={ + "from_node_id": from_id, + "to_node_id": to_id, + "edge_type": len(from_id) * ["flow"], + "allocation_network_id": allocation_network_id, + }, + geometry=lines, + crs="EPSG:28992", + ) + ) + + # Setup the basins: + profile = pd.DataFrame( + data={ + "node_id": [2, 2, 4, 4], + "area": 1000.0, + "level": 2 * [0.0, 1.0], + } + ) + + static = pd.DataFrame( + data={ + "node_id": [2, 4], + "drainage": 0.0, + "potential_evaporation": 0.0, + "infiltration": 0.0, + "precipitation": 0.0, + "urban_runoff": 0.0, + } + ) + + state = pd.DataFrame(data={"node_id": [2, 4], "level": 1.0}) + + basin = ribasim.Basin(profile=profile, static=static, state=state) + + # Setup the flow boundary: + flow_boundary = ribasim.FlowBoundary( + static=pd.DataFrame( + data={ + "node_id": [1], + "flow_rate": 2.0e-3, + } + ) + ) + + # Setup the pump: + pump = ribasim.Pump( + static=pd.DataFrame( + data={ + "node_id": [3], + "flow_rate": [4.0e-3], + "max_flow_rate": [4.0e-3], + } + ) + ) + + # Setup the users: + user = ribasim.User( + static=pd.DataFrame( + data={ + "node_id": [5], + "demand": 1.0e-3, + "return_factor": 0.9, + "min_level": 0.9, + "priority": 1, + } + ), + time=pd.DataFrame( + data={ + "time": ["2020-01-01 00:00:00", "2021-01-01 00:00:00"], + "node_id": 6, + "demand": [1.0e-3, 2.0e-3], + "return_factor": 0.9, + "min_level": 0.9, + "priority": 1, + } + ), + ) + + # Setup allocation: + allocation = ribasim.Allocation(use_allocation=True, timestep=86400) + + model = ribasim.Model( + node=node, + edge=edge, + basin=basin, + flow_boundary=flow_boundary, + pump=pump, + user=user, + allocation=allocation, + starttime="2020-01-01 00:00:00", + endtime="2021-01-01 00:00:00", + ) + + return model From 442a52ba0bdcb48e9baae5a2819f3b6ca8d59404 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 7 Nov 2023 14:12:13 +0100 Subject: [PATCH 03/21] Tests completed --- core/test/allocation.jl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/test/allocation.jl b/core/test/allocation.jl index d3946e57f..2ef72e0d0 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -34,6 +34,7 @@ end model = Ribasim.run(toml_path) record = model.integrator.p.user.record where_5 = (record.user_node_id .== 5) + where_6 = .!where_5 @test all(record.demand[where_5] .== 1.0e-3) @test all( @@ -50,4 +51,25 @@ end rtol = 0.1, ), ) + @test all( + isapprox( + record.demand[where_6], + collect(range(1.0e-3, 2.0e-3, sum(where_5))); + rtol = 0.01, + ), + ) + @test all( + isapprox( + record.allocated[where_6], + collect(range(1.0e-3, 2.0e-3, sum(where_5))); + rtol = 0.01, + ), + ) + @test all( + isapprox( + record.abstracted[where_6][2:end], + collect(range(1.0e-3, 2.0e-3, sum(where_5)))[2:end]; + rtol = 0.01, + ), + ) end From 9f3033e3b3851f9e2c85c6c8c08eed8173327402 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 7 Nov 2023 14:22:19 +0100 Subject: [PATCH 04/21] Test adjusted --- core/test/allocation.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/test/allocation.jl b/core/test/allocation.jl index 2ef72e0d0..dd5721095 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -46,9 +46,9 @@ end ) @test all( isapprox( - record.abstracted[where_5], - collect(range(1.0e-3, 0.0, sum(where_5))); - rtol = 0.1, + record.abstracted[where_5][2:end], + collect(range(1.0e-3, 0.0, sum(where_5)))[2:end]; + rtol = 0.01, ), ) @test all( From 4d7123c48e43c9c3476fb82e88a4edfe4db432fe Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 7 Nov 2023 15:23:55 +0100 Subject: [PATCH 05/21] Add fair distribution options --- core/src/allocation.jl | 23 +++++++++++++++++------ core/src/config.jl | 1 + core/src/create.jl | 1 + core/test/docs.toml | 1 + docs/schema/Allocation.schema.json | 8 +++++++- docs/schema/Config.schema.json | 3 ++- python/ribasim/ribasim/config.py | 7 ++++++- 7 files changed, 35 insertions(+), 9 deletions(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index d66ca698e..21db48b66 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -555,14 +555,22 @@ Objective function: Sum of flows to the users. function add_objective_function!( problem::JuMP.Model, allocgraph_edge_ids_user_demand::Dict{Int, Int}, + config::Config, )::Nothing F = problem[:F] A_basin = problem[:A_basin] - JuMP.@objective( - problem, - Max, - sum(A_basin) + sum([F[i] for i in values(allocgraph_edge_ids_user_demand)]) - ) + distribution = config.allocation.distribution + if distribution in ["quadratic_absolute", "quadratic_relative"] + # Assume demand = 1.0, set later + JuMP.@objective( + problem, + Min, + sum((A_basin .- 1.0) .^ 2) + + sum([(F[i] - 1.0)^2 for i in values(allocgraph_edge_ids_user_demand)]) + ) + else + error("Invalid allocation distribution type $distribution.") + end return nothing end @@ -570,6 +578,7 @@ end Construct the allocation problem for the current subnetwork as a JuMP.jl model. """ function allocation_problem( + config::Config, node_id_mapping::Dict{Int, Tuple{Int, Symbol}}, allocgraph_node_ids_user_with_returnflow::Vector{Int}, allocgraph_edges::Vector{Edge{Int}}, @@ -613,7 +622,7 @@ function allocation_problem( # TODO: The fractional flow constraints # Add objective to problem - add_objective_function!(problem, allocgraph_edge_ids_user_demand) + add_objective_function!(problem, allocgraph_edge_ids_user_demand, config) return problem end @@ -640,6 +649,7 @@ An AllocationModel object. """ function AllocationModel( + config::Config, p::Parameters, subnetwork_node_ids::Vector{Int}, source_edge_ids::Vector{Int}, @@ -657,6 +667,7 @@ function AllocationModel( # The JuMP.jl allocation problem problem = allocation_problem( + config, node_id_mapping, allocgraph_node_ids_user_with_returnflow, allocgraph_edges, diff --git a/core/src/config.jl b/core/src/config.jl index d9743b446..99bb5a0d9 100644 --- a/core/src/config.jl +++ b/core/src/config.jl @@ -125,6 +125,7 @@ end @option struct Allocation <: TableOption timestep::Union{Float64, Nothing} = nothing use_allocation::Bool = false + distribution::String = "quadratic_relative" end @option @addnodetypes struct Config <: TableOption diff --git a/core/src/create.jl b/core/src/create.jl index 67c8dc248..f81e85527 100644 --- a/core/src/create.jl +++ b/core/src/create.jl @@ -282,6 +282,7 @@ function generate_allocation_models!(p::Parameters, db::DB, config::Config)::Not push!( connectivity.allocation_models, AllocationModel( + config, p, allocation_group_node.fid, source_edge_ids, diff --git a/core/test/docs.toml b/core/test/docs.toml index 6b9124cf0..61245613d 100644 --- a/core/test/docs.toml +++ b/core/test/docs.toml @@ -15,6 +15,7 @@ time = "basin/time.arrow" [allocation] timestep = 86400 use_allocation = true +distribution = "quadratic_relative" [solver] algorithm = "QNDF" # optional, default "QNDF" diff --git a/docs/schema/Allocation.schema.json b/docs/schema/Allocation.schema.json index d81c4423d..3ce953007 100644 --- a/docs/schema/Allocation.schema.json +++ b/docs/schema/Allocation.schema.json @@ -21,9 +21,15 @@ "format": "default", "type": "boolean", "default": false + }, + "distribution": { + "format": "default", + "type": "string", + "default": "quadratic_relative" } }, "required": [ - "use_allocation" + "use_allocation", + "distribution" ] } diff --git a/docs/schema/Config.schema.json b/docs/schema/Config.schema.json index 446f9c99a..2e68760af 100644 --- a/docs/schema/Config.schema.json +++ b/docs/schema/Config.schema.json @@ -36,7 +36,8 @@ "$ref": "https://deltares.github.io/Ribasim/schema/Allocation.schema.json", "default": { "timestep": null, - "use_allocation": false + "use_allocation": false, + "distribution": "quadratic_relative" } }, "solver": { diff --git a/python/ribasim/ribasim/config.py b/python/ribasim/ribasim/config.py index 28c02343a..ba9e14f8b 100644 --- a/python/ribasim/ribasim/config.py +++ b/python/ribasim/ribasim/config.py @@ -12,6 +12,7 @@ class Allocation(BaseModel): timestep: Optional[float] = None use_allocation: bool = False + distribution: str = "quadratic_relative" class Solver(BaseModel): @@ -113,7 +114,11 @@ class Config(BaseModel): database: str allocation: Allocation = Field( default_factory=lambda: Allocation.parse_obj( - {"timestep": None, "use_allocation": False} + { + "timestep": None, + "use_allocation": False, + "distribution": "quadratic_relative", + } ) ) solver: Solver = Field( From 3554b537fdfc3d058c33f9e34a9e6fface4f3a41 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Wed, 8 Nov 2023 11:07:29 +0100 Subject: [PATCH 06/21] Towards fair distribution options --- core/Manifest.toml | 146 ++++++++++++++++------------- core/src/Ribasim.jl | 2 +- core/src/allocation.jl | 137 ++++++++++++++++++--------- core/src/config.jl | 2 +- core/src/solve.jl | 2 + core/test/docs.toml | 2 +- docs/Manifest.toml | 2 +- docs/schema/Allocation.schema.json | 4 +- docs/schema/Config.schema.json | 2 +- python/ribasim/ribasim/config.py | 4 +- 10 files changed, 187 insertions(+), 116 deletions(-) diff --git a/core/Manifest.toml b/core/Manifest.toml index 6dfa5a16b..c6581547a 100644 --- a/core/Manifest.toml +++ b/core/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.9.3" manifest_format = "2.0" -project_hash = "c63b237aec6a6bf6314284b2b07a94b93a9f0a36" +project_hash = "b5b649db1096de5a0eae1089a6a3b319b7bef6e7" [[deps.ADTypes]] git-tree-sha1 = "5d2e21d7b0d8c22f67483ef95ebdc39c0e6b6003" @@ -36,9 +36,9 @@ version = "0.2.0" [[deps.ArrayInterface]] deps = ["Adapt", "LinearAlgebra", "Requires", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "f83ec24f76d4c8f525099b2ac475fc098138ec31" +git-tree-sha1 = "16267cf279190ca7c1b30d020758ced95db89cd0" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "7.4.11" +version = "7.5.1" [deps.ArrayInterface.extensions] ArrayInterfaceBandedMatricesExt = "BandedMatrices" @@ -56,12 +56,6 @@ version = "7.4.11" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" -[[deps.ArrayInterfaceCore]] -deps = ["LinearAlgebra", "SnoopPrecompile", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "e5f08b5689b1aad068e01751889f2f615c7db36d" -uuid = "30b0a656-2188-435a-8636-2ec0e6a096e2" -version = "0.1.29" - [[deps.Arrow]] deps = ["ArrowTypes", "BitIntegers", "CodecLz4", "CodecZstd", "ConcurrentUtilities", "DataAPI", "Dates", "EnumX", "LoggingExtras", "Mmap", "PooledArrays", "SentinelArrays", "Tables", "TimeZones", "TranscodingStreams", "UUIDs"] git-tree-sha1 = "954666e252835c4cf8819ce4ffaf31073c1b7233" @@ -219,9 +213,9 @@ version = "0.2.3" [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] -git-tree-sha1 = "5372dbbf8f0bdb8c700db5367132925c0771ef7e" +git-tree-sha1 = "8cfa272e8bdedfa88b6aefbbca7c19f1befac519" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.2.1" +version = "2.3.0" [[deps.Configurations]] deps = ["ExproniconLite", "OrderedCollections", "TOML"] @@ -249,6 +243,11 @@ git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" uuid = "adafc99b-e345-5852-983c-f28acb93d879" version = "0.3.1" +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + [[deps.DBInterface]] git-tree-sha1 = "9b0dc525a052b9269ccc5f7f04d5b3639c65bca5" uuid = "a10d1c49-ce27-4219-8d33-6db1a4562965" @@ -260,10 +259,10 @@ uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.15.0" [[deps.DataInterpolations]] -deps = ["LinearAlgebra", "RecipesBase", "RecursiveArrayTools", "Reexport", "Requires"] -git-tree-sha1 = "5d8ddbe1e7e539d3c2f6ae34d32a770e722eec07" +deps = ["LinearAlgebra", "PrettyTables", "RecipesBase", "RecursiveArrayTools", "Reexport", "Requires"] +git-tree-sha1 = "97b5c6115165fc9f99908cbdd2ea4c0efda0843e" uuid = "82cc6244-b520-54b8-b5a6-8a565e85f1d0" -version = "4.4.0" +version = "4.5.0" [deps.DataInterpolations.extensions] DataInterpolationsChainRulesCoreExt = "ChainRulesCore" @@ -299,14 +298,15 @@ uuid = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4" version = "0.3.25" [[deps.DiffEqBase]] -deps = ["ArrayInterface", "ChainRulesCore", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces", "ZygoteRules"] -git-tree-sha1 = "36a590efdbee58b38f903ffc3b378f4a5336bc3f" +deps = ["ArrayInterface", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"] +git-tree-sha1 = "de4709e30bd5490435122c4b415b90a812c23fbf" uuid = "2b5f629d-d688-5b77-993f-72d75c75574e" -version = "6.134.0" +version = "6.138.1" [deps.DiffEqBase.extensions] + DiffEqBaseChainRulesCoreExt = "ChainRulesCore" DiffEqBaseDistributionsExt = "Distributions" - DiffEqBaseEnzymeExt = "Enzyme" + DiffEqBaseEnzymeExt = ["ChainRulesCore", "Enzyme"] DiffEqBaseGeneralizedGeneratedExt = "GeneralizedGenerated" DiffEqBaseMPIExt = "MPI" DiffEqBaseMeasurementsExt = "Measurements" @@ -314,9 +314,9 @@ version = "6.134.0" DiffEqBaseReverseDiffExt = "ReverseDiff" DiffEqBaseTrackerExt = "Tracker" DiffEqBaseUnitfulExt = "Unitful" - DiffEqBaseZygoteExt = "Zygote" [deps.DiffEqBase.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" GeneralizedGenerated = "6b9d7cbe-bcb9-11e9-073f-15a7a543e2eb" @@ -326,7 +326,6 @@ version = "6.134.0" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" - Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [[deps.DiffEqCallbacks]] deps = ["DataStructures", "DiffEqBase", "ForwardDiff", "Functors", "LinearAlgebra", "Markdown", "NLsolve", "Parameters", "RecipesBase", "RecursiveArrayTools", "SciMLBase", "StaticArraysCore"] @@ -383,9 +382,9 @@ version = "1.0.4" [[deps.EnzymeCore]] deps = ["Adapt"] -git-tree-sha1 = "d8701002a745c450c03b890f10d53636d1a8a7ea" +git-tree-sha1 = "ab81396e4e7b61f5590db02fa1c17fae4f16d7ab" uuid = "f151be2c-9106-41f4-ab19-57ee4f262869" -version = "0.6.2" +version = "0.6.3" [[deps.ExponentialUtilities]] deps = ["Adapt", "ArrayInterface", "GPUArraysCore", "GenericSchur", "LinearAlgebra", "PrecompileTools", "Printf", "SparseArrays", "libblastrampoline_jll"] @@ -599,11 +598,16 @@ git-tree-sha1 = "17e462054b42dcdda73e9a9ba0c67754170c88ae" uuid = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" version = "0.9.4" +[[deps.LaTeXStrings]] +git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.1" + [[deps.LayoutPointers]] deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"] -git-tree-sha1 = "88b8f66b604da079a627b6fb2860d3704a6729a1" +git-tree-sha1 = "62edfee3211981241b57ff1cedf4d74d79519277" uuid = "10f19ff3-798f-405d-979b-55457f8fc047" -version = "0.1.14" +version = "0.1.15" [[deps.Lazy]] deps = ["MacroTools"] @@ -660,10 +664,10 @@ deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LinearSolve]] -deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "EnzymeCore", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "MKL_jll", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"] -git-tree-sha1 = "a563cd835c9ed5295c35a7d140e0bf0a514bb10a" +deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "EnzymeCore", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "MKL", "MKL_jll", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"] +git-tree-sha1 = "84bdad5fb1fe03a6637ad413e0e4b7e48ac22be5" uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" -version = "2.13.0" +version = "2.16.2" [deps.LinearSolve.extensions] LinearSolveBandedMatricesExt = "BandedMatrices" @@ -717,10 +721,10 @@ uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" version = "1.0.3" [[deps.LoopVectorization]] -deps = ["ArrayInterface", "ArrayInterfaceCore", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "c88a4afe1703d731b1c4fdf4e3c7e77e3b176ea2" +deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] +git-tree-sha1 = "0f5648fbae0d015e3abe5867bca2b362f67a5894" uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.165" +version = "0.12.166" weakdeps = ["ChainRulesCore", "ForwardDiff", "SpecialFunctions"] [deps.LoopVectorization.extensions] @@ -733,6 +737,12 @@ git-tree-sha1 = "6c26c5e8a4203d43b5497be3ec5d4e0c3cde240a" uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" version = "1.9.4+0" +[[deps.MKL]] +deps = ["Artifacts", "Libdl", "LinearAlgebra", "MKL_jll"] +git-tree-sha1 = "100521a1d2181cb39036ee1a6955d6b9686bb363" +uuid = "33e6dc65-8f57-5167-99aa-e5a354878fb2" +version = "0.6.1" + [[deps.MKL_jll]] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] git-tree-sha1 = "eb006abbd7041c28e0d16260e50a24f8f9104913" @@ -756,9 +766,9 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MathOptInterface]] deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "DataStructures", "ForwardDiff", "JSON", "LinearAlgebra", "MutableArithmetics", "NaNMath", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays", "SpecialFunctions", "Test", "Unicode"] -git-tree-sha1 = "13b3d40084d04e609e0509730f05215fb2a2fba4" +git-tree-sha1 = "70ea2892b8bfffecc0387ba1a6a21192814f120c" uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" -version = "1.21.0" +version = "1.22.0" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] @@ -812,16 +822,18 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" [[deps.NonlinearSolve]] -deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "EnumX", "FiniteDiff", "ForwardDiff", "LineSearches", "LinearAlgebra", "LinearSolve", "PrecompileTools", "RecursiveArrayTools", "Reexport", "SciMLBase", "SimpleNonlinearSolve", "SparseArrays", "SparseDiffTools", "StaticArraysCore", "UnPack"] -git-tree-sha1 = "9203b3333c9610664de2e8cbc23cfd726663df7d" +deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "EnumX", "FastBroadcast", "FiniteDiff", "ForwardDiff", "LineSearches", "LinearAlgebra", "LinearSolve", "PrecompileTools", "RecursiveArrayTools", "Reexport", "SciMLBase", "SimpleNonlinearSolve", "SparseArrays", "SparseDiffTools", "StaticArraysCore", "UnPack"] +git-tree-sha1 = "f400009287afedef175058e64aadf7d41f593fef" uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" -version = "2.4.0" +version = "2.8.0" [deps.NonlinearSolve.extensions] + NonlinearSolveBandedMatricesExt = "BandedMatrices" NonlinearSolveFastLevenbergMarquardtExt = "FastLevenbergMarquardt" NonlinearSolveLeastSquaresOptimExt = "LeastSquaresOptim" [deps.NonlinearSolve.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce" LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891" @@ -854,9 +866,9 @@ version = "1.6.2" [[deps.OrdinaryDiffEq]] deps = ["ADTypes", "Adapt", "ArrayInterface", "DataStructures", "DiffEqBase", "DocStringExtensions", "ExponentialUtilities", "FastBroadcast", "FastClosures", "FiniteDiff", "ForwardDiff", "FunctionWrappersWrappers", "IfElse", "InteractiveUtils", "LineSearches", "LinearAlgebra", "LinearSolve", "Logging", "LoopVectorization", "MacroTools", "MuladdMacro", "NLsolve", "NonlinearSolve", "Polyester", "PreallocationTools", "PrecompileTools", "Preferences", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLNLSolve", "SciMLOperators", "SimpleNonlinearSolve", "SimpleUnPack", "SparseArrays", "SparseDiffTools", "StaticArrayInterface", "StaticArrays", "TruncatedStacktraces"] -git-tree-sha1 = "def999a7447854f0e9ca9fdda235e04a65916b76" +git-tree-sha1 = "f0f43037c0ba045e96f32d65858eb825a211b817" uuid = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -version = "6.58.0" +version = "6.58.2" [[deps.PackageExtensionCompat]] git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518" @@ -883,9 +895,9 @@ version = "1.9.2" [[deps.Polyester]] deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StaticArrayInterface", "StrideArraysCore", "ThreadingUtilities"] -git-tree-sha1 = "398f91235beaac50445557c937ecb0145d171842" +git-tree-sha1 = "fca25670784a1ae44546bcb17288218310af2778" uuid = "f517fe37-dbe3-4b94-8317-1923a5111588" -version = "0.7.8" +version = "0.7.9" [[deps.PolyesterWeave]] deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] @@ -923,6 +935,12 @@ git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" uuid = "21216c6a-2e73-6563-6e65-726566657250" version = "1.4.1" +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "6842ce83a836fbbc0cfeca0b5a4de1a4dcbdb8d1" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.2.8" + [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" @@ -1003,9 +1021,9 @@ version = "0.1.0" [[deps.SLEEFPirates]] deps = ["IfElse", "Static", "VectorizationBase"] -git-tree-sha1 = "4b8586aece42bee682399c4c4aee95446aa5cd19" +git-tree-sha1 = "3aac6d68c5e57449f5b9b865c9ba50ac2970c4cf" uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" -version = "0.6.39" +version = "0.6.42" [[deps.SQLite]] deps = ["DBInterface", "Random", "SQLite_jll", "Serialization", "Tables", "WeakRefStrings"] @@ -1020,12 +1038,13 @@ uuid = "76ed43ae-9a5d-5a62-8c75-30186b810ce8" version = "3.43.0+0" [[deps.SciMLBase]] -deps = ["ADTypes", "ArrayInterface", "ChainRulesCore", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FillArrays", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables", "TruncatedStacktraces", "ZygoteRules"] -git-tree-sha1 = "1c2a4e245744dd76b2eb677d3535ffad16d8b989" +deps = ["ADTypes", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FillArrays", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables", "TruncatedStacktraces"] +git-tree-sha1 = "8d1bb6b4c0aa7c1bcd3bcd81e2b8c3b4f4b0733e" uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" -version = "2.5.0" +version = "2.7.3" [deps.SciMLBase.extensions] + SciMLBaseChainRulesCoreExt = "ChainRulesCore" SciMLBasePartialFunctionsExt = "PartialFunctions" SciMLBasePyCallExt = "PyCall" SciMLBasePythonCallExt = "PythonCall" @@ -1033,6 +1052,7 @@ version = "2.5.0" SciMLBaseZygoteExt = "Zygote" [deps.SciMLBase.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" PartialFunctions = "570af359-4316-4cb7-8c74-252c00c2016b" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" @@ -1047,21 +1067,21 @@ version = "0.1.9" [[deps.SciMLOperators]] deps = ["ArrayInterface", "DocStringExtensions", "Lazy", "LinearAlgebra", "Setfield", "SparseArrays", "StaticArraysCore", "Tricks"] -git-tree-sha1 = "65c2e6ced6f62ea796af251eb292a0e131a3613b" +git-tree-sha1 = "51ae235ff058a64815e0a2c34b1db7578a06813d" uuid = "c0aeaf25-5076-4817-a8d5-81caf7dfa961" -version = "0.3.6" +version = "0.3.7" [[deps.Scratch]] deps = ["Dates"] -git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.0" +version = "1.2.1" [[deps.SentinelArrays]] deps = ["Dates", "Random"] -git-tree-sha1 = "04bdff0b09c65ff3e06a05e3eb7b120223da3d39" +git-tree-sha1 = "0e7508ff27ba32f26cd459474ca2ede1bc10991f" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.0" +version = "1.4.1" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -1077,10 +1097,10 @@ deps = ["Distributed", "Mmap", "Random", "Serialization"] uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" [[deps.SimpleNonlinearSolve]] -deps = ["ArrayInterface", "DiffEqBase", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "PackageExtensionCompat", "PrecompileTools", "Reexport", "SciMLBase", "StaticArraysCore"] -git-tree-sha1 = "15ff97fa4881133caa324dacafe28b5ac47ad8a2" +deps = ["ArrayInterface", "DiffEqBase", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "PrecompileTools", "Reexport", "SciMLBase", "StaticArraysCore"] +git-tree-sha1 = "69b1a53374dd14d7c165d98cb646aeb5f36f8d07" uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7" -version = "0.1.23" +version = "0.1.25" [deps.SimpleNonlinearSolve.extensions] SimpleNonlinearSolveNNlibExt = "NNlib" @@ -1114,9 +1134,9 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.SparseDiffTools]] deps = ["ADTypes", "Adapt", "ArrayInterface", "Compat", "DataStructures", "FiniteDiff", "ForwardDiff", "Graphs", "LinearAlgebra", "PackageExtensionCompat", "Random", "Reexport", "SciMLOperators", "Setfield", "SparseArrays", "StaticArrayInterface", "StaticArrays", "Tricks", "UnPack", "VertexSafeGraphs"] -git-tree-sha1 = "336fd944a1bbb8873bfa8171387608ca93317d68" +git-tree-sha1 = "e162b74fd1ce6d371ff5c584b53e34538edb9212" uuid = "47a9eef4-7e08-11e9-0b38-333d64bd3804" -version = "2.8.0" +version = "2.11.0" [deps.SparseDiffTools.extensions] SparseDiffToolsEnzymeExt = "Enzyme" @@ -1193,6 +1213,12 @@ git-tree-sha1 = "f02eb61eb5c97b48c153861c72fbbfdddc607e06" uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da" version = "0.4.17" +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.3.4" + [[deps.StructArrays]] deps = ["Adapt", "ConstructionBase", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] git-tree-sha1 = "0a3db38e4cce3c54fe7a71f831cd7b6194a54213" @@ -1282,9 +1308,9 @@ version = "0.9.13" [[deps.TriangularSolve]] deps = ["CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "LoopVectorization", "Polyester", "Static", "VectorizationBase"] -git-tree-sha1 = "31eedbc0b6d07c08a700e26d31298ac27ef330eb" +git-tree-sha1 = "fadebab77bf3ae041f77346dd1c290173da5a443" uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" -version = "0.1.19" +version = "0.1.20" [[deps.Tricks]] git-tree-sha1 = "eae1bb484cd63b36999ee58be2de6c178105112f" @@ -1338,12 +1364,6 @@ git-tree-sha1 = "49ce682769cd5de6c72dcf1b94ed7790cd08974c" uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.5+0" -[[deps.ZygoteRules]] -deps = ["ChainRulesCore", "MacroTools"] -git-tree-sha1 = "9d749cd449fb448aeca4feee9a2f4186dbb5d184" -uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.2.4" - [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" diff --git a/core/src/Ribasim.jl b/core/src/Ribasim.jl index 7f68deec4..c214d4bbb 100644 --- a/core/src/Ribasim.jl +++ b/core/src/Ribasim.jl @@ -61,9 +61,9 @@ TimerOutputs.complement!() include("validation.jl") include("solve.jl") -include("allocation.jl") include("config.jl") using .config +include("allocation.jl") include("utils.jl") include("lib.jl") include("io.jl") diff --git a/core/src/allocation.jl b/core/src/allocation.jl index 21db48b66..79dcc4a5f 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -548,31 +548,31 @@ function add_constraints_user_returnflow!( return nothing end -""" -Add the objective function to be maximized to the allocation problem. -Objective function: Sum of flows to the users. -""" -function add_objective_function!( - problem::JuMP.Model, - allocgraph_edge_ids_user_demand::Dict{Int, Int}, - config::Config, -)::Nothing - F = problem[:F] - A_basin = problem[:A_basin] - distribution = config.allocation.distribution - if distribution in ["quadratic_absolute", "quadratic_relative"] - # Assume demand = 1.0, set later - JuMP.@objective( - problem, - Min, - sum((A_basin .- 1.0) .^ 2) + - sum([(F[i] - 1.0)^2 for i in values(allocgraph_edge_ids_user_demand)]) - ) - else - error("Invalid allocation distribution type $distribution.") - end - return nothing -end +# """ +# Add the objective function to be maximized to the allocation problem. +# Objective function: Sum of flows to the users. +# """ +# function add_objective_function!( +# problem::JuMP.Model, +# allocgraph_edge_ids_user_demand::Dict{Int, Int}, +# config::Config, +# )::Nothing +# F = problem[:F] +# A_basin = problem[:A_basin] +# distribution = config.allocation.distribution +# if distribution in ["quadratic_absolute", "quadratic_relative"] +# # Assume demand = 1.0, set later +# JuMP.@objective( +# problem, +# Min, +# sum((A_basin .- 1.0) .^ 2) + +# sum([(F[i] - 1.0)^2 for i in values(allocgraph_edge_ids_user_demand)]) +# ) +# else +# error("Invalid allocation distribution type $distribution.") +# end +# return nothing +# end """ Construct the allocation problem for the current subnetwork as a JuMP.jl model. @@ -621,8 +621,8 @@ function allocation_problem( add_constraints_user_returnflow!(problem, allocgraph_node_ids_user_with_returnflow) # TODO: The fractional flow constraints - # Add objective to problem - add_objective_function!(problem, allocgraph_edge_ids_user_demand, config) + # # Add objective to problem + # add_objective_function!(problem, allocgraph_edge_ids_user_demand, config) return problem end @@ -678,6 +678,7 @@ function AllocationModel( ) return AllocationModel( + Symbol(config.allocation.objective_type), subnetwork_node_ids, node_id_mapping, node_id_mapping_inverse, @@ -690,28 +691,71 @@ function AllocationModel( ) end -""" -Set the demands of the users of the current time and priority -in the allocation problem. -""" -function set_demands_priority!( +# """ +# Set the demands of the users of the current time and priority +# in the allocation problem. +# """ +# function set_demands_priority!( +# allocation_model::AllocationModel, +# user::User, +# priority_idx::Int, +# t::Float64, +# )::Nothing +# (; problem, allocgraph_edge_ids_user_demand, node_id_mapping_inverse) = allocation_model +# (; demand, node_id) = user +# constraints_demand = problem[:demand_user] + +# for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand +# model_user_id = node_id_mapping_inverse[allocgraph_node_id][1] +# user_idx = findsorted(node_id, model_user_id) +# JuMP.set_normalized_rhs( +# constraints_demand[allocgraph_edge_id], +# demand[user_idx][priority_idx](t), +# ) +# end +# return nothing +# end + +""" + +""" +function set_objective!( allocation_model::AllocationModel, user::User, - priority_idx::Int, t::Float64, + priority_idx::Int, )::Nothing - (; problem, allocgraph_edge_ids_user_demand, node_id_mapping_inverse) = allocation_model + (; objective_type, problem, allocgraph_edge_ids_user_demand, node_id_mapping_inverse) = + allocation_model (; demand, node_id) = user - constraints_demand = problem[:demand_user] - - for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand - model_user_id = node_id_mapping_inverse[allocgraph_node_id][1] - user_idx = findsorted(node_id, model_user_id) - JuMP.set_normalized_rhs( - constraints_demand[allocgraph_edge_id], - demand[user_idx][priority_idx](t), - ) + F = problem[:F] + ex = JuMP.QuadExpr() + if objective_type == :quadratic_absolute + for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand + user_idx = findsorted(node_id, node_id_mapping_inverse[allocgraph_node_id][1]) + d = demand[user_idx][priority_idx](t) + F_ij = F[allocgraph_edge_id] + JuMP.add_to_expression!(ex, 1, F_ij, F_ij) + JuMP.add_to_expression!(ex, -2 * d, F_ij) + JuMP.add_to_expression!(ex, d^2) + end + elseif objective_type == :quadratic_relative + for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand + user_idx = findsorted(node_id, node_id_mapping_inverse[allocgraph_node_id][1]) + d = demand[user_idx][priority_idx](t) + if d ≈ 0 + continue + end + F_ij = F[allocgraph_edge_id] + JuMP.add_to_expression!(ex, 1.0 / d^2, F_ij, F_ij) + JuMP.add_to_expression!(ex, -2.0 / d, F_ij) + JuMP.add_to_expression!(ex, 1.0) + end + else + error("Invalid allocation objective type $objective_type.") end + new_objective = JuMP.@expression(problem, ex) + JuMP.set_objective_function(problem, new_objective) return nothing end @@ -802,7 +846,12 @@ function allocate!(p::Parameters, allocation_model::AllocationModel, t::Float64) # or set edge capacities if priority_idx = 1 adjust_edge_capacities!(allocation_model, priority_idx) - set_demands_priority!(allocation_model, user, priority_idx, t) + # Set the objective depending on the demands + # A new objective function is set instead of modifying the coefficients + # of an existing objective function because this is not supported for + # quadratic terms: + # https://jump.dev/JuMP.jl/stable/manual/objective/#Modify-an-objective-coefficient + set_objective!(allocation_model, user, t, priority_idx) # Solve the allocation problem for this priority JuMP.optimize!(problem) diff --git a/core/src/config.jl b/core/src/config.jl index 99bb5a0d9..24b694d92 100644 --- a/core/src/config.jl +++ b/core/src/config.jl @@ -125,7 +125,7 @@ end @option struct Allocation <: TableOption timestep::Union{Float64, Nothing} = nothing use_allocation::Bool = false - distribution::String = "quadratic_relative" + objective_type::String = "quadratic_relative" end @option @addnodetypes struct Config <: TableOption diff --git a/core/src/solve.jl b/core/src/solve.jl index 90d2cc212..33979c0b8 100644 --- a/core/src/solve.jl +++ b/core/src/solve.jl @@ -7,6 +7,7 @@ const VectorInterpolation = """ Store information for a subnetwork used for allocation. +objective_type: The name of the type of objective used node_id: All the IDs of the nodes that are in this subnetwork node_id_mapping: Mapping Dictionary; model_node_id => AG_node_id where such a correspondence exists (all AG node ids are in the values) @@ -19,6 +20,7 @@ problem: The JuMP.jl model for solving the allocation problem Δt_allocation: The time interval between consecutive allocation solves """ struct AllocationModel + objective_type::Symbol node_id::Vector{Int} node_id_mapping::Dict{Int, Tuple{Int, Symbol}} node_id_mapping_inverse::Dict{Int, Tuple{Int, Symbol}} diff --git a/core/test/docs.toml b/core/test/docs.toml index 61245613d..0d31cb4e6 100644 --- a/core/test/docs.toml +++ b/core/test/docs.toml @@ -15,7 +15,7 @@ time = "basin/time.arrow" [allocation] timestep = 86400 use_allocation = true -distribution = "quadratic_relative" +objective_type = "quadratic_relative" [solver] algorithm = "QNDF" # optional, default "QNDF" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 0ea664daa..5e7212385 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.9.3" manifest_format = "2.0" -project_hash = "f9da26c97b8dd21bb861d490428435c07bbfff72" +project_hash = "be943413ce780c11a7d6da180de486f1760d780f" [[deps.ANSIColoredPrinters]] git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" diff --git a/docs/schema/Allocation.schema.json b/docs/schema/Allocation.schema.json index 3ce953007..94195d977 100644 --- a/docs/schema/Allocation.schema.json +++ b/docs/schema/Allocation.schema.json @@ -22,7 +22,7 @@ "type": "boolean", "default": false }, - "distribution": { + "objective_type": { "format": "default", "type": "string", "default": "quadratic_relative" @@ -30,6 +30,6 @@ }, "required": [ "use_allocation", - "distribution" + "objective_type" ] } diff --git a/docs/schema/Config.schema.json b/docs/schema/Config.schema.json index 2e68760af..abae55018 100644 --- a/docs/schema/Config.schema.json +++ b/docs/schema/Config.schema.json @@ -37,7 +37,7 @@ "default": { "timestep": null, "use_allocation": false, - "distribution": "quadratic_relative" + "objective_type": "quadratic_relative" } }, "solver": { diff --git a/python/ribasim/ribasim/config.py b/python/ribasim/ribasim/config.py index ba9e14f8b..74e4add7d 100644 --- a/python/ribasim/ribasim/config.py +++ b/python/ribasim/ribasim/config.py @@ -12,7 +12,7 @@ class Allocation(BaseModel): timestep: Optional[float] = None use_allocation: bool = False - distribution: str = "quadratic_relative" + objective_type: str = "quadratic_relative" class Solver(BaseModel): @@ -117,7 +117,7 @@ class Config(BaseModel): { "timestep": None, "use_allocation": False, - "distribution": "quadratic_relative", + "objective_type": "quadratic_relative", } ) ) From c7115c575692c017914d1ea60e95ece6b4d6c774 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Wed, 8 Nov 2023 11:38:39 +0100 Subject: [PATCH 07/21] Let allocation default to allocated = demand --- core/Manifest.toml | 146 +++++++++++++++++++++++------------------ core/src/allocation.jl | 7 +- core/src/create.jl | 3 + core/src/solve.jl | 15 ++++- 4 files changed, 105 insertions(+), 66 deletions(-) diff --git a/core/Manifest.toml b/core/Manifest.toml index 6dfa5a16b..9b644c2e1 100644 --- a/core/Manifest.toml +++ b/core/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.9.3" manifest_format = "2.0" -project_hash = "c63b237aec6a6bf6314284b2b07a94b93a9f0a36" +project_hash = "b5b649db1096de5a0eae1089a6a3b319b7bef6e7" [[deps.ADTypes]] git-tree-sha1 = "5d2e21d7b0d8c22f67483ef95ebdc39c0e6b6003" @@ -36,9 +36,9 @@ version = "0.2.0" [[deps.ArrayInterface]] deps = ["Adapt", "LinearAlgebra", "Requires", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "f83ec24f76d4c8f525099b2ac475fc098138ec31" +git-tree-sha1 = "16267cf279190ca7c1b30d020758ced95db89cd0" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "7.4.11" +version = "7.5.1" [deps.ArrayInterface.extensions] ArrayInterfaceBandedMatricesExt = "BandedMatrices" @@ -56,12 +56,6 @@ version = "7.4.11" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" -[[deps.ArrayInterfaceCore]] -deps = ["LinearAlgebra", "SnoopPrecompile", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "e5f08b5689b1aad068e01751889f2f615c7db36d" -uuid = "30b0a656-2188-435a-8636-2ec0e6a096e2" -version = "0.1.29" - [[deps.Arrow]] deps = ["ArrowTypes", "BitIntegers", "CodecLz4", "CodecZstd", "ConcurrentUtilities", "DataAPI", "Dates", "EnumX", "LoggingExtras", "Mmap", "PooledArrays", "SentinelArrays", "Tables", "TimeZones", "TranscodingStreams", "UUIDs"] git-tree-sha1 = "954666e252835c4cf8819ce4ffaf31073c1b7233" @@ -219,9 +213,9 @@ version = "0.2.3" [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] -git-tree-sha1 = "5372dbbf8f0bdb8c700db5367132925c0771ef7e" +git-tree-sha1 = "8cfa272e8bdedfa88b6aefbbca7c19f1befac519" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.2.1" +version = "2.3.0" [[deps.Configurations]] deps = ["ExproniconLite", "OrderedCollections", "TOML"] @@ -249,6 +243,11 @@ git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" uuid = "adafc99b-e345-5852-983c-f28acb93d879" version = "0.3.1" +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + [[deps.DBInterface]] git-tree-sha1 = "9b0dc525a052b9269ccc5f7f04d5b3639c65bca5" uuid = "a10d1c49-ce27-4219-8d33-6db1a4562965" @@ -260,10 +259,10 @@ uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.15.0" [[deps.DataInterpolations]] -deps = ["LinearAlgebra", "RecipesBase", "RecursiveArrayTools", "Reexport", "Requires"] -git-tree-sha1 = "5d8ddbe1e7e539d3c2f6ae34d32a770e722eec07" +deps = ["LinearAlgebra", "PrettyTables", "RecipesBase", "RecursiveArrayTools", "Reexport", "Requires"] +git-tree-sha1 = "97b5c6115165fc9f99908cbdd2ea4c0efda0843e" uuid = "82cc6244-b520-54b8-b5a6-8a565e85f1d0" -version = "4.4.0" +version = "4.5.0" [deps.DataInterpolations.extensions] DataInterpolationsChainRulesCoreExt = "ChainRulesCore" @@ -299,14 +298,15 @@ uuid = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4" version = "0.3.25" [[deps.DiffEqBase]] -deps = ["ArrayInterface", "ChainRulesCore", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces", "ZygoteRules"] -git-tree-sha1 = "36a590efdbee58b38f903ffc3b378f4a5336bc3f" +deps = ["ArrayInterface", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"] +git-tree-sha1 = "de4709e30bd5490435122c4b415b90a812c23fbf" uuid = "2b5f629d-d688-5b77-993f-72d75c75574e" -version = "6.134.0" +version = "6.138.1" [deps.DiffEqBase.extensions] + DiffEqBaseChainRulesCoreExt = "ChainRulesCore" DiffEqBaseDistributionsExt = "Distributions" - DiffEqBaseEnzymeExt = "Enzyme" + DiffEqBaseEnzymeExt = ["ChainRulesCore", "Enzyme"] DiffEqBaseGeneralizedGeneratedExt = "GeneralizedGenerated" DiffEqBaseMPIExt = "MPI" DiffEqBaseMeasurementsExt = "Measurements" @@ -314,9 +314,9 @@ version = "6.134.0" DiffEqBaseReverseDiffExt = "ReverseDiff" DiffEqBaseTrackerExt = "Tracker" DiffEqBaseUnitfulExt = "Unitful" - DiffEqBaseZygoteExt = "Zygote" [deps.DiffEqBase.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" GeneralizedGenerated = "6b9d7cbe-bcb9-11e9-073f-15a7a543e2eb" @@ -326,7 +326,6 @@ version = "6.134.0" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" - Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [[deps.DiffEqCallbacks]] deps = ["DataStructures", "DiffEqBase", "ForwardDiff", "Functors", "LinearAlgebra", "Markdown", "NLsolve", "Parameters", "RecipesBase", "RecursiveArrayTools", "SciMLBase", "StaticArraysCore"] @@ -383,9 +382,9 @@ version = "1.0.4" [[deps.EnzymeCore]] deps = ["Adapt"] -git-tree-sha1 = "d8701002a745c450c03b890f10d53636d1a8a7ea" +git-tree-sha1 = "ab81396e4e7b61f5590db02fa1c17fae4f16d7ab" uuid = "f151be2c-9106-41f4-ab19-57ee4f262869" -version = "0.6.2" +version = "0.6.3" [[deps.ExponentialUtilities]] deps = ["Adapt", "ArrayInterface", "GPUArraysCore", "GenericSchur", "LinearAlgebra", "PrecompileTools", "Printf", "SparseArrays", "libblastrampoline_jll"] @@ -599,11 +598,16 @@ git-tree-sha1 = "17e462054b42dcdda73e9a9ba0c67754170c88ae" uuid = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" version = "0.9.4" +[[deps.LaTeXStrings]] +git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.1" + [[deps.LayoutPointers]] deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"] -git-tree-sha1 = "88b8f66b604da079a627b6fb2860d3704a6729a1" +git-tree-sha1 = "62edfee3211981241b57ff1cedf4d74d79519277" uuid = "10f19ff3-798f-405d-979b-55457f8fc047" -version = "0.1.14" +version = "0.1.15" [[deps.Lazy]] deps = ["MacroTools"] @@ -660,10 +664,10 @@ deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LinearSolve]] -deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "EnzymeCore", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "MKL_jll", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"] -git-tree-sha1 = "a563cd835c9ed5295c35a7d140e0bf0a514bb10a" +deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "EnzymeCore", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "MKL", "MKL_jll", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "Requires", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "SuiteSparse", "UnPack"] +git-tree-sha1 = "34082ca62d07e9d4ae04659f616b3674a321f740" uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" -version = "2.13.0" +version = "2.17.1" [deps.LinearSolve.extensions] LinearSolveBandedMatricesExt = "BandedMatrices" @@ -717,10 +721,10 @@ uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" version = "1.0.3" [[deps.LoopVectorization]] -deps = ["ArrayInterface", "ArrayInterfaceCore", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "c88a4afe1703d731b1c4fdf4e3c7e77e3b176ea2" +deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] +git-tree-sha1 = "0f5648fbae0d015e3abe5867bca2b362f67a5894" uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.165" +version = "0.12.166" weakdeps = ["ChainRulesCore", "ForwardDiff", "SpecialFunctions"] [deps.LoopVectorization.extensions] @@ -733,6 +737,12 @@ git-tree-sha1 = "6c26c5e8a4203d43b5497be3ec5d4e0c3cde240a" uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" version = "1.9.4+0" +[[deps.MKL]] +deps = ["Artifacts", "Libdl", "LinearAlgebra", "MKL_jll"] +git-tree-sha1 = "100521a1d2181cb39036ee1a6955d6b9686bb363" +uuid = "33e6dc65-8f57-5167-99aa-e5a354878fb2" +version = "0.6.1" + [[deps.MKL_jll]] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] git-tree-sha1 = "eb006abbd7041c28e0d16260e50a24f8f9104913" @@ -756,9 +766,9 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MathOptInterface]] deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "DataStructures", "ForwardDiff", "JSON", "LinearAlgebra", "MutableArithmetics", "NaNMath", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays", "SpecialFunctions", "Test", "Unicode"] -git-tree-sha1 = "13b3d40084d04e609e0509730f05215fb2a2fba4" +git-tree-sha1 = "70ea2892b8bfffecc0387ba1a6a21192814f120c" uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" -version = "1.21.0" +version = "1.22.0" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] @@ -812,16 +822,18 @@ uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" [[deps.NonlinearSolve]] -deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "EnumX", "FiniteDiff", "ForwardDiff", "LineSearches", "LinearAlgebra", "LinearSolve", "PrecompileTools", "RecursiveArrayTools", "Reexport", "SciMLBase", "SimpleNonlinearSolve", "SparseArrays", "SparseDiffTools", "StaticArraysCore", "UnPack"] -git-tree-sha1 = "9203b3333c9610664de2e8cbc23cfd726663df7d" +deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "EnumX", "FastBroadcast", "FiniteDiff", "ForwardDiff", "LineSearches", "LinearAlgebra", "LinearSolve", "PrecompileTools", "RecursiveArrayTools", "Reexport", "SciMLBase", "SimpleNonlinearSolve", "SparseArrays", "SparseDiffTools", "StaticArraysCore", "UnPack"] +git-tree-sha1 = "f400009287afedef175058e64aadf7d41f593fef" uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" -version = "2.4.0" +version = "2.8.0" [deps.NonlinearSolve.extensions] + NonlinearSolveBandedMatricesExt = "BandedMatrices" NonlinearSolveFastLevenbergMarquardtExt = "FastLevenbergMarquardt" NonlinearSolveLeastSquaresOptimExt = "LeastSquaresOptim" [deps.NonlinearSolve.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce" LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891" @@ -854,9 +866,9 @@ version = "1.6.2" [[deps.OrdinaryDiffEq]] deps = ["ADTypes", "Adapt", "ArrayInterface", "DataStructures", "DiffEqBase", "DocStringExtensions", "ExponentialUtilities", "FastBroadcast", "FastClosures", "FiniteDiff", "ForwardDiff", "FunctionWrappersWrappers", "IfElse", "InteractiveUtils", "LineSearches", "LinearAlgebra", "LinearSolve", "Logging", "LoopVectorization", "MacroTools", "MuladdMacro", "NLsolve", "NonlinearSolve", "Polyester", "PreallocationTools", "PrecompileTools", "Preferences", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLNLSolve", "SciMLOperators", "SimpleNonlinearSolve", "SimpleUnPack", "SparseArrays", "SparseDiffTools", "StaticArrayInterface", "StaticArrays", "TruncatedStacktraces"] -git-tree-sha1 = "def999a7447854f0e9ca9fdda235e04a65916b76" +git-tree-sha1 = "f0f43037c0ba045e96f32d65858eb825a211b817" uuid = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" -version = "6.58.0" +version = "6.58.2" [[deps.PackageExtensionCompat]] git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518" @@ -883,9 +895,9 @@ version = "1.9.2" [[deps.Polyester]] deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StaticArrayInterface", "StrideArraysCore", "ThreadingUtilities"] -git-tree-sha1 = "398f91235beaac50445557c937ecb0145d171842" +git-tree-sha1 = "fca25670784a1ae44546bcb17288218310af2778" uuid = "f517fe37-dbe3-4b94-8317-1923a5111588" -version = "0.7.8" +version = "0.7.9" [[deps.PolyesterWeave]] deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] @@ -923,6 +935,12 @@ git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" uuid = "21216c6a-2e73-6563-6e65-726566657250" version = "1.4.1" +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "6842ce83a836fbbc0cfeca0b5a4de1a4dcbdb8d1" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.2.8" + [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" @@ -1003,9 +1021,9 @@ version = "0.1.0" [[deps.SLEEFPirates]] deps = ["IfElse", "Static", "VectorizationBase"] -git-tree-sha1 = "4b8586aece42bee682399c4c4aee95446aa5cd19" +git-tree-sha1 = "3aac6d68c5e57449f5b9b865c9ba50ac2970c4cf" uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" -version = "0.6.39" +version = "0.6.42" [[deps.SQLite]] deps = ["DBInterface", "Random", "SQLite_jll", "Serialization", "Tables", "WeakRefStrings"] @@ -1020,12 +1038,13 @@ uuid = "76ed43ae-9a5d-5a62-8c75-30186b810ce8" version = "3.43.0+0" [[deps.SciMLBase]] -deps = ["ADTypes", "ArrayInterface", "ChainRulesCore", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FillArrays", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables", "TruncatedStacktraces", "ZygoteRules"] -git-tree-sha1 = "1c2a4e245744dd76b2eb677d3535ffad16d8b989" +deps = ["ADTypes", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FillArrays", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables", "TruncatedStacktraces"] +git-tree-sha1 = "8d1bb6b4c0aa7c1bcd3bcd81e2b8c3b4f4b0733e" uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" -version = "2.5.0" +version = "2.7.3" [deps.SciMLBase.extensions] + SciMLBaseChainRulesCoreExt = "ChainRulesCore" SciMLBasePartialFunctionsExt = "PartialFunctions" SciMLBasePyCallExt = "PyCall" SciMLBasePythonCallExt = "PythonCall" @@ -1033,6 +1052,7 @@ version = "2.5.0" SciMLBaseZygoteExt = "Zygote" [deps.SciMLBase.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" PartialFunctions = "570af359-4316-4cb7-8c74-252c00c2016b" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" @@ -1047,21 +1067,21 @@ version = "0.1.9" [[deps.SciMLOperators]] deps = ["ArrayInterface", "DocStringExtensions", "Lazy", "LinearAlgebra", "Setfield", "SparseArrays", "StaticArraysCore", "Tricks"] -git-tree-sha1 = "65c2e6ced6f62ea796af251eb292a0e131a3613b" +git-tree-sha1 = "51ae235ff058a64815e0a2c34b1db7578a06813d" uuid = "c0aeaf25-5076-4817-a8d5-81caf7dfa961" -version = "0.3.6" +version = "0.3.7" [[deps.Scratch]] deps = ["Dates"] -git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a" +git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.0" +version = "1.2.1" [[deps.SentinelArrays]] deps = ["Dates", "Random"] -git-tree-sha1 = "04bdff0b09c65ff3e06a05e3eb7b120223da3d39" +git-tree-sha1 = "0e7508ff27ba32f26cd459474ca2ede1bc10991f" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.0" +version = "1.4.1" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -1077,10 +1097,10 @@ deps = ["Distributed", "Mmap", "Random", "Serialization"] uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" [[deps.SimpleNonlinearSolve]] -deps = ["ArrayInterface", "DiffEqBase", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "PackageExtensionCompat", "PrecompileTools", "Reexport", "SciMLBase", "StaticArraysCore"] -git-tree-sha1 = "15ff97fa4881133caa324dacafe28b5ac47ad8a2" +deps = ["ArrayInterface", "DiffEqBase", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "PrecompileTools", "Reexport", "SciMLBase", "StaticArraysCore"] +git-tree-sha1 = "69b1a53374dd14d7c165d98cb646aeb5f36f8d07" uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7" -version = "0.1.23" +version = "0.1.25" [deps.SimpleNonlinearSolve.extensions] SimpleNonlinearSolveNNlibExt = "NNlib" @@ -1114,9 +1134,9 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.SparseDiffTools]] deps = ["ADTypes", "Adapt", "ArrayInterface", "Compat", "DataStructures", "FiniteDiff", "ForwardDiff", "Graphs", "LinearAlgebra", "PackageExtensionCompat", "Random", "Reexport", "SciMLOperators", "Setfield", "SparseArrays", "StaticArrayInterface", "StaticArrays", "Tricks", "UnPack", "VertexSafeGraphs"] -git-tree-sha1 = "336fd944a1bbb8873bfa8171387608ca93317d68" +git-tree-sha1 = "e162b74fd1ce6d371ff5c584b53e34538edb9212" uuid = "47a9eef4-7e08-11e9-0b38-333d64bd3804" -version = "2.8.0" +version = "2.11.0" [deps.SparseDiffTools.extensions] SparseDiffToolsEnzymeExt = "Enzyme" @@ -1193,6 +1213,12 @@ git-tree-sha1 = "f02eb61eb5c97b48c153861c72fbbfdddc607e06" uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da" version = "0.4.17" +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.3.4" + [[deps.StructArrays]] deps = ["Adapt", "ConstructionBase", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] git-tree-sha1 = "0a3db38e4cce3c54fe7a71f831cd7b6194a54213" @@ -1282,9 +1308,9 @@ version = "0.9.13" [[deps.TriangularSolve]] deps = ["CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "LoopVectorization", "Polyester", "Static", "VectorizationBase"] -git-tree-sha1 = "31eedbc0b6d07c08a700e26d31298ac27ef330eb" +git-tree-sha1 = "fadebab77bf3ae041f77346dd1c290173da5a443" uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" -version = "0.1.19" +version = "0.1.20" [[deps.Tricks]] git-tree-sha1 = "eae1bb484cd63b36999ee58be2de6c178105112f" @@ -1338,12 +1364,6 @@ git-tree-sha1 = "49ce682769cd5de6c72dcf1b94ed7790cd08974c" uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.5+0" -[[deps.ZygoteRules]] -deps = ["ChainRulesCore", "MacroTools"] -git-tree-sha1 = "9d749cd449fb448aeca4feee9a2f4186dbb5d184" -uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.2.4" - [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" diff --git a/core/src/allocation.jl b/core/src/allocation.jl index 22aebde95..446cd1e29 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -8,7 +8,7 @@ function get_node_id_mapping( subnetwork_node_ids::Vector{Int}, source_edge_ids::Vector{Int}, ) - (; lookup, connectivity) = p + (; lookup, connectivity, user) = p (; graph_flow, edge_ids_flow_inv) = connectivity # Mapping node_id => (allocgraph_node_id, type) where such a correspondence exists; @@ -24,6 +24,11 @@ function get_node_id_mapping( if node_type in [:user, :basin] add_allocgraph_node = true + if node_type == :user + user_idx = findsorted(user.node_id, subnetwork_node_id) + user.allocation_optimized[user_idx] = true + end + elseif length(all_neighbors(graph_flow, subnetwork_node_id)) > 2 # Each junction (that is, a node with more than 2 neighbors) # in the subnetwork gets an allocgraph node diff --git a/core/src/create.jl b/core/src/create.jl index af2b121e8..9cf566951 100644 --- a/core/src/create.jl +++ b/core/src/create.jl @@ -791,6 +791,8 @@ function User(db::DB, config::Config)::User abstracted = Vector{Float64}(), ) + allocation_optimized = BitVector(zeros(UInt8, length(node_ids))) + return User( node_ids, active, @@ -799,6 +801,7 @@ function User(db::DB, config::Config)::User return_factor, min_level, priorities, + allocation_optimized, record, ) end diff --git a/core/src/solve.jl b/core/src/solve.jl index fecf16214..46cbb31fe 100644 --- a/core/src/solve.jl +++ b/core/src/solve.jl @@ -441,7 +441,10 @@ allocated: water flux currently allocated to user per priority return_factor: the factor in [0,1] of how much of the abstracted water is given back to the system min_level: The level of the source basin below which the user does not abstract priorities: All used priority values. Each user has a demand for all these priorities, -which is always 0.0 if it is not provided explicitly. + which is 0.0 if it is not provided explicitly. +allocation_optimized: Whether the allocated abstraction rate of the user is set by allocation optimization. + If not, allocated is equal to demand. +record: Collected data of allocation optimizations for output file. """ struct User <: AbstractParameterNode node_id::Vector{Int} @@ -451,6 +454,7 @@ struct User <: AbstractParameterNode return_factor::Vector{Float64} min_level::Vector{Float64} priorities::Vector{Int} + allocation_optimized::BitVector record::NamedTuple{ ( :time, @@ -797,7 +801,8 @@ function formulate_flow!( )::Nothing (; connectivity, basin) = p (; graph_flow, flow) = connectivity - (; node_id, allocated, demand, active, return_factor, min_level) = user + (; node_id, allocated, demand, active, return_factor, min_level, allocation_optimized) = + user flow = get_tmp(flow, storage) @@ -809,6 +814,12 @@ function formulate_flow!( continue end + if !allocation_optimized[i] + for priority_idx in eachindex(allocated[i]) + allocated[i][priority_idx] = demand[i][priority_idx](t) + end + end + q = sum(allocated[i]) # Smoothly let abstraction go to 0 as the source basin dries out From 6e3252c9a615230c664992e367136be48493b2bc Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Wed, 8 Nov 2023 12:17:35 +0100 Subject: [PATCH 08/21] Take minimum of current demand and optimized allocation --- core/src/solve.jl | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/src/solve.jl b/core/src/solve.jl index 46cbb31fe..847634739 100644 --- a/core/src/solve.jl +++ b/core/src/solve.jl @@ -814,14 +814,24 @@ function formulate_flow!( continue end - if !allocation_optimized[i] + q = 0.0 + + if allocation_optimized[i] + # If allocation has been optimized for this user, + # use the minimum of that allocation and the current demand of the user for priority_idx in eachindex(allocated[i]) - allocated[i][priority_idx] = demand[i][priority_idx](t) + alloc = min(allocated[i][priority_idx], demand[i][priority_idx](t)) + q += alloc + end + else + # If allocation has not been optimized for this user, + # use the demand as allocated directly + for priority_idx in eachindex(allocated[i]) + alloc = demand[i][priority_idx](t) + q += alloc end end - q = sum(allocated[i]) - # Smoothly let abstraction go to 0 as the source basin dries out _, basin_idx = id_index(basin.node_id, src_id) factor_basin = reduction_factor(storage[basin_idx], 10.0) From bcfc493e5904e6c86dbc260ed40fbc446623cc3b Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Wed, 8 Nov 2023 12:36:01 +0100 Subject: [PATCH 09/21] docs update --- docs/core/allocation.qmd | 4 ++-- docs/core/equations.qmd | 25 ++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/core/allocation.qmd b/docs/core/allocation.qmd index 5f3509d89..35f09279a 100644 --- a/docs/core/allocation.qmd +++ b/docs/core/allocation.qmd @@ -62,8 +62,8 @@ Both fractional flow nodes and user nodes dictate proportional relationships bet A new graph is created from the subnetwork, which we call the allocation graph. To indicate the difference between subnetwork data and allocation graph data, the allocation graph data is denoted with a hat. The allocation graph consists of: -- Nodes $\hat{V_S}$, where each basin, source and user in the subnetwork get a node in the allocation graph. Also nodes that have fractional flow outneighbors get a node in the allocation graph. The implementation makes heavy use of the node id mapping $m_S : i \mapsto \hat{i}$ to translate from subnetwork node IDs to allocation graph node IDs. Unless specified otherwise, we assume this relationship between index symbols that appear both with and without a hat. -- Edges $\hat{E_S}$, where the edges in the allocation graph are given by one or more edges in the subnetwork, where those edges connect nodes in the subnetwork that have an equivalent in the allocation graph. The direction of the edges in the allocation graph is given by the direction constraints in the subnetwork. +- Nodes $\hat{V}_s$, where each basin, source and user in the subnetwork get a node in the allocation graph. Also nodes that have fractional flow outneighbors get a node in the allocation graph. The implementation makes heavy use of the node id mapping $m_S : i \mapsto \hat{i}$ to translate from subnetwork node IDs to allocation graph node IDs. Unless specified otherwise, we assume this relationship between index symbols that appear both with and without a hat. +- Edges $\hat{E}_S$, where the edges in the allocation graph are given by one or more edges in the subnetwork, where those edges connect nodes in the subnetwork that have an equivalent in the allocation graph. The direction of the edges in the allocation graph is given by the direction constraints in the subnetwork. For notational convenience, we use the notation diff --git a/docs/core/equations.qmd b/docs/core/equations.qmd index 5b1d2e606..aa7dcaeb8 100644 --- a/docs/core/equations.qmd +++ b/docs/core/equations.qmd @@ -448,6 +448,29 @@ $$ B = w + 2 d \sqrt{\left(\frac{\Delta y}{\Delta z}\right)^2 + 1} $$ +# User allocation +Users have an allocated flow rate $F^p$ per priority $p=1,2,\ldots, p_{\max}$, which is either determined by [allocation optimization](allocation.qmd) or simply equal to the demand at time $t$; $F^p = d^p(t)$. The actual abstraction rate of a user is given by +$$ + Q_\text{user, in} = \phi(u, 10.0)\phi(h-l_{\min}, 0.1)\sum_{p=1}^{p_{\max}} \min\left(F^p, d^p(t)\right). +$$ + +From left to right: + +- The first reduction factor lets the user abstraction go smoothly to $0$ as the source basin dries out; +- The second reduction factor lets the user abstraction go smoothly ro $0$ as the source basin level approaches the minimum source basin level (from above); +- The last term is the sum of the allocations over the priorities. If the current demand happens to be lower than the allocation at some priority, the demand is taken instead. + +::: {.callout-note} +Currently $l_{\min}$ is a priority of a user, in the future this will become a property of a basin. +::: + +Users also have a return factor $0 \le r \le 1$, which determines the return flow (outflow) of the user: + +$$ +Q_\text{user, out} = r \cdot Q_\text{user, in}. +$$ + +Note that this means that the user has a consumption rate of $(1-r)Q_\text{user, in}$. # PID controller {#sec-PID} @@ -523,7 +546,7 @@ Ribasim uses OrdinaryDiffEq.jl to provide a numerical solution to the water balance equations. Changes to forcings or parameters such as precipitation, but also the allocated water abstraction is managed through the use of CallBack functions [@callbacks]. In a coupled run, the exchanges with MODFLOW 6 are also -managed via the use of a callback function. +managed via the use of a callback function. For more a more in-depth discussion of numerical computations see [Numerical considerations](numerics.qmd). # Performance From 558e42182aa1144fa54a84eb02986a4e1920d503 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Wed, 8 Nov 2023 14:47:29 +0100 Subject: [PATCH 10/21] Use JuMP.@objective --- core/src/allocation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index 79dcc4a5f..f4f090e78 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -755,7 +755,7 @@ function set_objective!( error("Invalid allocation objective type $objective_type.") end new_objective = JuMP.@expression(problem, ex) - JuMP.set_objective_function(problem, new_objective) + JuMP.@objective(problem, Min, new_objective) return nothing end From 8a8c63b41b299cbd9c89c4672b1d94e827ac8988 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Fri, 10 Nov 2023 11:30:16 +0100 Subject: [PATCH 11/21] Towards testing all objective types --- core/src/allocation.jl | 102 ++++-------------- core/test/allocation.jl | 95 +++++++++------- .../ribasim_testmodels/allocation.py | 8 +- 3 files changed, 82 insertions(+), 123 deletions(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index c54c44148..d8005d293 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -391,35 +391,28 @@ function add_variables_flow!( return nothing end -""" -Add the basin allocation variables A_basin to the allocation problem. -The variable indices are the allocation graph basin node IDs. -Non-negativivity constraints are also immediately added to the basin allocation variables. -""" -function add_variables_allocation_basin!( - problem::JuMP.Model, - node_id_mapping::Dict{Int, Tuple{Int, Symbol}}, - allocgraph_node_ids_basin::Vector{Int}, -)::Nothing - JuMP.@variable(problem, A_basin[i = allocgraph_node_ids_basin] >= 0.0) - return nothing -end +# """ +# Add the basin allocation variables A_basin to the allocation problem. +# The variable indices are the allocation graph basin node IDs. +# Non-negativivity constraints are also immediately added to the basin allocation variables. +# """ +# function add_variables_allocation_basin!( +# problem::JuMP.Model, +# allocgraph_node_ids_basin::Vector{Int}, +# )::Nothing +# JuMP.@variable(problem, A_basin[i = allocgraph_node_ids_basin] >= 0.0) +# return nothing +# end -""" -Add the user allocation constraints to the allocation problem: -The flow to a user is bounded from above by the demand of the user. -""" -function add_constraints_user_allocation!( +function add_variables_absolute_value!( problem::JuMP.Model, allocgraph_edge_ids_user_demand::Dict{Int, Int}, + config::Config, )::Nothing - F = problem[:F] - # Allocation flows are bounded from above by demands - problem[:demand_user] = JuMP.@constraint( - problem, - [i = values(allocgraph_edge_ids_user_demand)], - F[i] <= 0.0 - ) + if startswith(config.allocation.objective_type, "linear") + problem[:F_abs] = + JuMP.@variable(problem, F_abs[values(allocgraph_edge_ids_user_demand)]) + end return nothing end @@ -553,32 +546,6 @@ function add_constraints_user_returnflow!( return nothing end -# """ -# Add the objective function to be maximized to the allocation problem. -# Objective function: Sum of flows to the users. -# """ -# function add_objective_function!( -# problem::JuMP.Model, -# allocgraph_edge_ids_user_demand::Dict{Int, Int}, -# config::Config, -# )::Nothing -# F = problem[:F] -# A_basin = problem[:A_basin] -# distribution = config.allocation.distribution -# if distribution in ["quadratic_absolute", "quadratic_relative"] -# # Assume demand = 1.0, set later -# JuMP.@objective( -# problem, -# Min, -# sum((A_basin .- 1.0) .^ 2) + -# sum([(F[i] - 1.0)^2 for i in values(allocgraph_edge_ids_user_demand)]) -# ) -# else -# error("Invalid allocation distribution type $distribution.") -# end -# return nothing -# end - """ Construct the allocation problem for the current subnetwork as a JuMP.jl model. """ @@ -605,11 +572,11 @@ function allocation_problem( # Add variables to problem add_variables_flow!(problem, allocgraph_edges) - add_variables_allocation_basin!(problem, node_id_mapping, allocgraph_node_ids_basin) + # add_variables_allocation_basin!(problem, allocgraph_node_ids_basin) + add_variables_absolute_value!(problem, allocgraph_edge_ids_user_demand, config) # Add constraints to problem - add_constraints_user_allocation!(problem, allocgraph_edge_ids_user_demand) - add_constraints_basin_allocation!(problem, allocgraph_node_ids_basin) + # add_constraints_basin_allocation!(problem, allocgraph_node_ids_basin) add_constraints_capacity!(problem, capacity, allocgraph_edges) add_constraints_source!( problem, @@ -698,31 +665,6 @@ function AllocationModel( ) end -# """ -# Set the demands of the users of the current time and priority -# in the allocation problem. -# """ -# function set_demands_priority!( -# allocation_model::AllocationModel, -# user::User, -# priority_idx::Int, -# t::Float64, -# )::Nothing -# (; problem, allocgraph_edge_ids_user_demand, node_id_mapping_inverse) = allocation_model -# (; demand, node_id) = user -# constraints_demand = problem[:demand_user] - -# for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand -# model_user_id = node_id_mapping_inverse[allocgraph_node_id][1] -# user_idx = findsorted(node_id, model_user_id) -# JuMP.set_normalized_rhs( -# constraints_demand[allocgraph_edge_id], -# demand[user_idx][priority_idx](t), -# ) -# end -# return nothing -# end - """ """ @@ -758,6 +700,8 @@ function set_objective!( JuMP.add_to_expression!(ex, -2.0 / d, F_ij) JuMP.add_to_expression!(ex, 1.0) end + elseif objective in [:linear_relative, :linear_absolute] + pass else error("Invalid allocation objective type $objective_type.") end diff --git a/core/test/allocation.jl b/core/test/allocation.jl index dd5721095..b05d69470 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -27,49 +27,62 @@ using PreallocationTools: get_tmp @test allocated[3] ≈ [0.0, 0.0] end -@testset "Simulation with allocation" begin +@testset "Allocation objective types" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/simple_subnetwork/ribasim.toml") @test ispath(toml_path) - model = Ribasim.run(toml_path) - record = model.integrator.p.user.record - where_5 = (record.user_node_id .== 5) - where_6 = .!where_5 - @test all(record.demand[where_5] .== 1.0e-3) - @test all( - isapprox( - record.allocated[where_5], - collect(range(1.0e-3, 0.0, sum(where_5))); - rtol = 0.01, - ), - ) - @test all( - isapprox( - record.abstracted[where_5][2:end], - collect(range(1.0e-3, 0.0, sum(where_5)))[2:end]; - rtol = 0.01, - ), - ) - @test all( - isapprox( - record.demand[where_6], - collect(range(1.0e-3, 2.0e-3, sum(where_5))); - rtol = 0.01, - ), - ) - @test all( - isapprox( - record.allocated[where_6], - collect(range(1.0e-3, 2.0e-3, sum(where_5))); - rtol = 0.01, - ), - ) - @test all( - isapprox( - record.abstracted[where_6][2:end], - collect(range(1.0e-3, 2.0e-3, sum(where_5)))[2:end]; - rtol = 0.01, - ), - ) + config = Ribasim.Config(toml_path; allocation_objective_type = "quadratic_absolute") + Ribasim.run(config) + + config = Ribasim.Config(toml_path; allocation_objective_type = "quadratic_relative") + Ribasim.run(config) + + config = Ribasim.Config(toml_path; allocation_objective_type = "linear_absolute") + Ribasim.run(config) + + config = Ribasim.Config(toml_path; allocation_objective_type = "linear_relative") + Ribasim.run(config) + + # model = Ribasim.run(toml_path) + # record = model.integrator.p.user.record + # where_5 = (record.user_node_id .== 5) + # where_6 = .!where_5 + + # @test all(record.demand[where_5] .== 1.0e-3) + # @test all( + # isapprox( + # record.allocated[where_5], + # collect(range(1.0e-3, 0.0, sum(where_5))); + # rtol = 0.01, + # ), + # ) + # @test all( + # isapprox( + # record.abstracted[where_5][2:end], + # collect(range(1.0e-3, 0.0, sum(where_5)))[2:end]; + # rtol = 0.01, + # ), + # ) + # @test all( + # isapprox( + # record.demand[where_6], + # collect(range(1.0e-3, 2.0e-3, sum(where_5))); + # rtol = 0.01, + # ), + # ) + # @test all( + # isapprox( + # record.allocated[where_6], + # collect(range(1.0e-3, 2.0e-3, sum(where_5))); + # rtol = 0.01, + # ), + # ) + # @test all( + # isapprox( + # record.abstracted[where_6][2:end], + # collect(range(1.0e-3, 2.0e-3, sum(where_5)))[2:end]; + # rtol = 0.01, + # ), + # ) end diff --git a/python/ribasim_testmodels/ribasim_testmodels/allocation.py b/python/ribasim_testmodels/ribasim_testmodels/allocation.py index 5c17fc5e2..f8563ee41 100644 --- a/python/ribasim_testmodels/ribasim_testmodels/allocation.py +++ b/python/ribasim_testmodels/ribasim_testmodels/allocation.py @@ -621,8 +621,8 @@ def simple_subnetwork_model(): static=pd.DataFrame( data={ "node_id": [3], - "flow_rate": [4.0e-3], - "max_flow_rate": [4.0e-3], + "flow_rate": [1.5e-3], + "max_flow_rate": [1.5e-3], } ) ) @@ -651,7 +651,9 @@ def simple_subnetwork_model(): ) # Setup allocation: - allocation = ribasim.Allocation(use_allocation=True, timestep=86400) + allocation = ribasim.Allocation( + use_allocation=True, timestep=86400, objective_type="quadratic_absolute" + ) model = ribasim.Model( node=node, From dc1c09f570deeaf6d5243d578aaf4bb458f75397 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Mon, 13 Nov 2023 13:49:58 +0100 Subject: [PATCH 12/21] Manifest fix --- .vscode/settings.json | 5 ----- Manifest.toml | 6 ------ 2 files changed, 11 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0a301f9dd..cd7219c98 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,11 +2,6 @@ "[julia]": { "editor.formatOnSave": true }, - "notebook.formatOnSave.enabled": true, - "notebook.codeActionsOnSave": { - "source.fixAll.ruff": true, - "source.organizeImports.ruff": true - }, "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, diff --git a/Manifest.toml b/Manifest.toml index 6bda639c4..9b0e5d18a 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1434,12 +1434,6 @@ git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" version = "0.3.4" -[[deps.StringManipulation]] -deps = ["PrecompileTools"] -git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" -uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.3.4" - [[deps.StructArrays]] deps = ["Adapt", "ConstructionBase", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] git-tree-sha1 = "0a3db38e4cce3c54fe7a71f831cd7b6194a54213" From 29c2b09c59bdd781c708200d6b43f75962d0614d Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 10:47:48 +0100 Subject: [PATCH 13/21] Make all allocation objective functions run without error --- .vscode/settings.json | 5 ++ core/src/allocation.jl | 102 ++++++++++++++++++++++++++++++++-------- core/src/solve.jl | 3 +- core/test/allocation.jl | 3 +- 4 files changed, 91 insertions(+), 22 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index cd7219c98..0a301f9dd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,11 @@ "[julia]": { "editor.formatOnSave": true }, + "notebook.formatOnSave.enabled": true, + "notebook.codeActionsOnSave": { + "source.fixAll.ruff": true, + "source.organizeImports.ruff": true + }, "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.formatOnSave": true, diff --git a/core/src/allocation.jl b/core/src/allocation.jl index fb6eea119..afff70efb 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -24,11 +24,6 @@ function get_node_id_mapping( if node_type in [:user, :basin] add_allocgraph_node = true - if node_type == :user - user_idx = findsorted(user.node_id, subnetwork_node_id) - user.allocation_optimized[user_idx] = true - end - elseif length(all_neighbors(graph_flow, subnetwork_node_id)) > 2 # Each junction (that is, a node with more than 2 neighbors) # in the subnetwork gets an allocgraph node @@ -546,6 +541,53 @@ function add_constraints_user_returnflow!( return nothing end +function add_constraints_absolute_value!( + problem::JuMP.Model, + allocgraph_edge_ids_user_demand::Dict{Int, Int}, + config::Config, +)::Nothing + objective_type = config.allocation.objective_type + if startswith(objective_type, "linear") + allocgraph_edge_ids_user_demand = collect(values(allocgraph_edge_ids_user_demand)) + F = problem[:F] + F_abs = problem[:F_abs] + d = 2.0 + + if config.allocation.objective_type == "linear_absolute" + # These constraints together make sure that F_abs acts as the absolute + # value F_abs = |x| where x = F-d (here for example d = 2) + problem[:abs_positive] = JuMP.@constraint( + problem, + [i = allocgraph_edge_ids_user_demand], + F_abs[i] >= (F[i] - d), + base_name = "abs_positive" + ) + problem[:abs_negative] = JuMP.@constraint( + problem, + [i = allocgraph_edge_ids_user_demand], + F_abs[i] >= -(F[i] - d), + base_name = "abs_negative" + ) + elseif config.allocation.objective_type == "linear_relative" + # These constraints together make sure that F_abs acts as the absolute + # value F_abs = |x| where x = 1-F/d (here for example d = 2) + problem[:abs_positive] = JuMP.@constraint( + problem, + [i = allocgraph_edge_ids_user_demand], + F_abs[i] >= (1 - F[i] / d), + base_name = "abs_positive" + ) + problem[:abs_negative] = JuMP.@constraint( + problem, + [i = allocgraph_edge_ids_user_demand], + F_abs[i] >= -(1 - F[i] / d), + base_name = "abs_negative" + ) + end + end + return nothing +end + """ Construct the allocation problem for the current subnetwork as a JuMP.jl model. """ @@ -591,6 +633,7 @@ function allocation_problem( allocgraph_node_outedge_ids, ) add_constraints_user_returnflow!(problem, allocgraph_node_ids_user_with_returnflow) + add_constraints_absolute_value!(problem, allocgraph_edge_ids_user_demand, config) # TODO: The fractional flow constraints # # Add objective to problem @@ -621,6 +664,7 @@ An AllocationModel object. """ function AllocationModel( + config::Config, allocation_network_id::Int, p::Parameters, subnetwork_node_ids::Vector{Int}, @@ -677,20 +721,25 @@ function set_objective!( allocation_model (; demand, node_id) = user F = problem[:F] - ex = JuMP.QuadExpr() - if objective_type == :quadratic_absolute - for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand - user_idx = findsorted(node_id, node_id_mapping_inverse[allocgraph_node_id][1]) - d = demand[user_idx][priority_idx](t) + if objective_type in [:quadratic_absolute, :quadratic_relative] + ex = JuMP.QuadExpr() + elseif objective_type in [:linear_absolute, :linear_relative] + ex = sum(problem[:F_abs]) + end + + for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand + user_idx = findsorted(node_id, node_id_mapping_inverse[allocgraph_node_id][1]) + d = demand[user_idx][priority_idx](t) + + if objective_type == :quadratic_absolute + # Objective function ∑ (F - d)^2 F_ij = F[allocgraph_edge_id] JuMP.add_to_expression!(ex, 1, F_ij, F_ij) JuMP.add_to_expression!(ex, -2 * d, F_ij) JuMP.add_to_expression!(ex, d^2) - end - elseif objective_type == :quadratic_relative - for (allocgraph_node_id, allocgraph_edge_id) in allocgraph_edge_ids_user_demand - user_idx = findsorted(node_id, node_id_mapping_inverse[allocgraph_node_id][1]) - d = demand[user_idx][priority_idx](t) + + elseif objective_type == :quadratic_relative + # Objective function ∑ (1 - F/d)^2S if d ≈ 0 continue end @@ -698,11 +747,26 @@ function set_objective!( JuMP.add_to_expression!(ex, 1.0 / d^2, F_ij, F_ij) JuMP.add_to_expression!(ex, -2.0 / d, F_ij) JuMP.add_to_expression!(ex, 1.0) + + elseif objective_type == :linear_absolute + # Objective function ∑ |F - d| + JuMP.set_normalized_rhs(problem[:abs_positive][allocgraph_edge_id], -d) + JuMP.set_normalized_rhs(problem[:abs_negative][allocgraph_edge_id], d) + elseif objective_type == :linear_relative + # Objective function ∑ |1 - F/d| + JuMP.set_normalized_coefficient( + problem[:abs_positive][allocgraph_edge_id], + F[allocgraph_edge_id], + iszero(d) ? 0 : 1 / d, + ) + JuMP.set_normalized_coefficient( + problem[:abs_negative][allocgraph_edge_id], + F[allocgraph_edge_id], + iszero(d) ? 0 : -1 / d, + ) + else + error("Invalid allocation objective type $objective_type.") end - elseif objective in [:linear_relative, :linear_absolute] - pass - else - error("Invalid allocation objective type $objective_type.") end new_objective = JuMP.@expression(problem, ex) JuMP.@objective(problem, Min, new_objective) diff --git a/core/src/solve.jl b/core/src/solve.jl index a2ccc847b..6bd952c62 100644 --- a/core/src/solve.jl +++ b/core/src/solve.jl @@ -791,8 +791,7 @@ function formulate_flow!( )::Nothing (; connectivity, basin) = p (; graph_flow, flow) = connectivity - (; node_id, allocated, demand, active, return_factor, min_level, allocation_optimized) = - user + (; node_id, allocated, demand, active, return_factor, min_level) = user flow = get_tmp(flow, storage) diff --git a/core/test/allocation.jl b/core/test/allocation.jl index 02c7f8726..2f1b7468e 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -1,5 +1,6 @@ import Ribasim import JuMP +using Test using SQLite using PreallocationTools: get_tmp using DataFrames: DataFrame @@ -30,7 +31,7 @@ end @testset "Allocation objective types" begin toml_path = - normpath(@__DIR__, "../../generated_testmodels/simple_subnetwork/ribasim.toml") + normpath(@__DIR__, "../../generated_testmodels/minimal_subnetwork/ribasim.toml") @test ispath(toml_path) config = Ribasim.Config(toml_path; allocation_objective_type = "quadratic_absolute") From d51fee7e5f0a6d38a6b18477ca0d98936ccccd36 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 12:33:31 +0100 Subject: [PATCH 14/21] Add docstrings --- core/src/allocation.jl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index afff70efb..9001f243a 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -399,6 +399,12 @@ end # return nothing # end +""" +Certain allocation distribution types use absolute values in the objective function. +Since most optimization packages do not support the absolute value function directly, +New variables are introduced that act as the absolute value of an expression by +posing the appropriate constraints. +""" function add_variables_absolute_value!( problem::JuMP.Model, allocgraph_edge_ids_user_demand::Dict{Int, Int}, @@ -541,6 +547,12 @@ function add_constraints_user_returnflow!( return nothing end +""" +Minimizing |expr| can be achieved by introducing a new variable expr_abs +and posing the following constraints: +expr_abs >= expr +expr_abs >= -expr +""" function add_constraints_absolute_value!( problem::JuMP.Model, allocgraph_edge_ids_user_demand::Dict{Int, Int}, @@ -709,9 +721,10 @@ function AllocationModel( end """ - +Set the objective for the given priority. +For an objective with absolute values this also involves adjusting constraints. """ -function set_objective!( +function set_objective_priority!( allocation_model::AllocationModel, user::User, t::Float64, @@ -890,7 +903,7 @@ function allocate!(p::Parameters, allocation_model::AllocationModel, t::Float64) # of an existing objective function because this is not supported for # quadratic terms: # https://jump.dev/JuMP.jl/stable/manual/objective/#Modify-an-objective-coefficient - set_objective!(allocation_model, user, t, priority_idx) + set_objective_priority!(allocation_model, user, t, priority_idx) # Solve the allocation problem for this priority JuMP.optimize!(problem) From 9467f2e08e77f1ac90365e0180cd506d1998342f Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 13:24:31 +0100 Subject: [PATCH 15/21] Add tests --- core/test/allocation.jl | 79 +++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 47 deletions(-) diff --git a/core/test/allocation.jl b/core/test/allocation.jl index a2ae99434..04d3251c2 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -5,6 +5,7 @@ using Test using SQLite using PreallocationTools: get_tmp using DataFrames: DataFrame +using SciMLBase: successful_retcode @testset "Allocation solve" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/subnetwork/ribasim.toml") @@ -22,7 +23,7 @@ using DataFrames: DataFrame Ribasim.allocate!(p, allocation_model, 0.0) F = JuMP.value.(allocation_model.problem[:F]) - @test F ≈ [0.0, 4.0, 0.0, 0.0, 0.0, 4.5] + @test F ≈ [0.0, 4.0, 0.0, 0.0, 0.0, 4.0] allocated = p.user.allocated @test allocated[1] ≈ [0.0, 4.0] @@ -36,56 +37,40 @@ end @test ispath(toml_path) config = Ribasim.Config(toml_path; allocation_objective_type = "quadratic_absolute") - Ribasim.run(config) + model = Ribasim.run(config) + @test successful_retcode(model) + problem = model.integrator.p.connectivity.allocation_models[1].problem + objective = JuMP.objective_function(problem) + @test objective isa JuMP.QuadExpr # Quadratic expression + F = problem[:F] + @test JuMP.UnorderedPair{JuMP.VariableRef}(F[2], F[2]) in keys(objective.terms) # F[2]^2 term + @test JuMP.UnorderedPair{JuMP.VariableRef}(F[3], F[3]) in keys(objective.terms) # F[3]^2 term config = Ribasim.Config(toml_path; allocation_objective_type = "quadratic_relative") - Ribasim.run(config) + model = Ribasim.run(config) + @test successful_retcode(model) + problem = model.integrator.p.connectivity.allocation_models[1].problem + objective = JuMP.objective_function(problem) + @test objective isa JuMP.QuadExpr # Quadratic expression + @test objective.aff.constant == 2.0 config = Ribasim.Config(toml_path; allocation_objective_type = "linear_absolute") - Ribasim.run(config) + model = Ribasim.run(config) + @test successful_retcode(model) + problem = model.integrator.p.connectivity.allocation_models[1].problem + objective = JuMP.objective_function(problem) + @test objective isa JuMP.AffExpr # Affine expression + @test :F_abs in keys(problem.obj_dict) + F_abs = problem[:F_abs] + @test objective == F_abs[3] + F_abs[2] config = Ribasim.Config(toml_path; allocation_objective_type = "linear_relative") - Ribasim.run(config) - - # model = Ribasim.run(toml_path) - # record = model.integrator.p.user.record - # where_5 = (record.user_node_id .== 5) - # where_6 = .!where_5 - - # @test all(record.demand[where_5] .== 1.0e-3) - # @test all( - # isapprox( - # record.allocated[where_5], - # collect(range(1.0e-3, 0.0, sum(where_5))); - # rtol = 0.01, - # ), - # ) - # @test all( - # isapprox( - # record.abstracted[where_5][2:end], - # collect(range(1.0e-3, 0.0, sum(where_5)))[2:end]; - # rtol = 0.01, - # ), - # ) - # @test all( - # isapprox( - # record.demand[where_6], - # collect(range(1.0e-3, 2.0e-3, sum(where_5))); - # rtol = 0.01, - # ), - # ) - # @test all( - # isapprox( - # record.allocated[where_6], - # collect(range(1.0e-3, 2.0e-3, sum(where_5))); - # rtol = 0.01, - # ), - # ) - # @test all( - # isapprox( - # record.abstracted[where_6][2:end], - # collect(range(1.0e-3, 2.0e-3, sum(where_5)))[2:end]; - # rtol = 0.01, - # ), - # ) + model = Ribasim.run(config) + @test successful_retcode(model) + problem = model.integrator.p.connectivity.allocation_models[1].problem + objective = JuMP.objective_function(problem) + @test objective isa JuMP.AffExpr # Affine expression + @test :F_abs in keys(problem.obj_dict) + F_abs = problem[:F_abs] + @test objective == F_abs[3] + F_abs[2] end From 186dcc71568bab706035284f23a7fc74d9da6808 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 13:27:07 +0100 Subject: [PATCH 16/21] Some more tests --- core/test/allocation.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/test/allocation.jl b/core/test/allocation.jl index 04d3251c2..3651eb706 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation.jl @@ -53,6 +53,9 @@ end objective = JuMP.objective_function(problem) @test objective isa JuMP.QuadExpr # Quadratic expression @test objective.aff.constant == 2.0 + F = problem[:F] + @test JuMP.UnorderedPair{JuMP.VariableRef}(F[2], F[2]) in keys(objective.terms) # F[2]^2 term + @test JuMP.UnorderedPair{JuMP.VariableRef}(F[3], F[3]) in keys(objective.terms) # F[3]^2 term config = Ribasim.Config(toml_path; allocation_objective_type = "linear_absolute") model = Ribasim.run(config) From 132660e4a078834c7133bd56bcfd5e546da7e5fc Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 14:15:44 +0100 Subject: [PATCH 17/21] Docs update --- core/src/allocation.jl | 2 +- docs/core/allocation.qmd | 32 ++++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index 9001f243a..f70a09b4b 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -8,7 +8,7 @@ function get_node_id_mapping( subnetwork_node_ids::Vector{Int}, source_edge_ids::Vector{Int}, ) - (; lookup, connectivity, user) = p + (; lookup, connectivity) = p (; graph_flow, edge_ids_flow_inv) = connectivity # Mapping node_id => (allocgraph_node_id, type) where such a correspondence exists; diff --git a/docs/core/allocation.qmd b/docs/core/allocation.qmd index 35f09279a..8e0424c0e 100644 --- a/docs/core/allocation.qmd +++ b/docs/core/allocation.qmd @@ -94,7 +94,7 @@ There are 2 types of variables whose value has to be determined to solve the all $$ A^\text{basin}_{\hat{i}} \ge 0, \quad \hat{B}_S, $$ -where $\hat{B} = m_S(B_S) \subset \hat{V}_S$ is the set of basin node ids in the allocation graph. +where $\hat{B}_S = m_S(B_S) \subset \hat{V}_S$ is the set of basin node ids in the allocation graph. :::{.callout-note} Currently the basin allocations are not taken into account in the implementation. @@ -102,17 +102,27 @@ Currently the basin allocations are not taken into account in the implementation ### The optimization objective -The goal of allocation is to maximize the flow to the users. Therefore, we use the following optimization objective: +The goal of allocation is to get the flow to the users as close as possible to their demand. To achieve this, the following objectives are supported: + +- `quadratic_absolute`: +$$ + \min \sum_{(\hat{i},\hat{j})\in \hat{E}_S\;:\; \hat{i}\in \hat{U}_S} \left( F_{\hat{i}\hat{j}} - d_j^p(t)\right)^2 +$$ +- `quadratic_relative`: +$$ + \min \sum_{(\hat{i},\hat{j})\in \hat{E}_S\;:\; \hat{i}\in \hat{U}_S} \left( 1 - \frac{F_{\hat{i}\hat{j}}}{d_j^p(t)}\right)^2 +$$ +- `linear_absolute`: +$$ + \min \sum_{(\hat{i},\hat{j})\in \hat{E}_S\;:\; \hat{i}\in \hat{U}_S} \left| F_{\hat{i}\hat{j}} - d_j^p(t)\right| +$$ +- `linear_relative`: +$$ + \min \sum_{(\hat{i},\hat{j})\in \hat{E}_S\;:\; \hat{i}\in \hat{U}_S} \left|1 - \frac{F_{\hat{i}\hat{j}}}{d_j^p(t)}\right| $$ - \max \sum_{(\hat{i},\hat{j})\in \hat{E}_S\;:\; \hat{i}\in \hat{U}_S} F_{\hat{i}\hat{j}} -$$ {#eq-objective} -This is a linear combination of the allocations to the users, where all allocations currently get a weight of $1$. :::{.callout-note} -This optimization objective will probably be modified in the future, to take things into account like: - -- Fair distribution in the case of water scarcity; -- An allocation graph-wide preference ordering over sources. +In the future new optimization objectives will be introduced, for demands of basins and priorities over sources. These will be used in combination with the above, in the form of goal programming. ::: ### The optimization variable constraints @@ -175,7 +185,9 @@ $$ 5. Repeat steps 2-4 for the remaining priorities up to $p_{\max}$. :::{.callout-note} -In the future there will be another solve before the priority 1 solve, taking the demand/supply from basins into account. +In the future there will be 2 more optimization solves: +- One before optimizing for users, taking the demand/supply from basins into account; +- One after optimizing for users, taking preferences over sources into account. ::: ## Example From 62480c23c15dfd874e1d2b1fafdacfc4b49de111 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 14:22:06 +0100 Subject: [PATCH 18/21] division by 0 clarification --- core/src/allocation.jl | 1 + docs/core/allocation.qmd | 2 ++ 2 files changed, 3 insertions(+) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index f70a09b4b..940200a47 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -765,6 +765,7 @@ function set_objective_priority!( # Objective function ∑ |F - d| JuMP.set_normalized_rhs(problem[:abs_positive][allocgraph_edge_id], -d) JuMP.set_normalized_rhs(problem[:abs_negative][allocgraph_edge_id], d) + elseif objective_type == :linear_relative # Objective function ∑ |1 - F/d| JuMP.set_normalized_coefficient( diff --git a/docs/core/allocation.qmd b/docs/core/allocation.qmd index 8e0424c0e..ece267a8a 100644 --- a/docs/core/allocation.qmd +++ b/docs/core/allocation.qmd @@ -121,6 +121,8 @@ $$ \min \sum_{(\hat{i},\hat{j})\in \hat{E}_S\;:\; \hat{i}\in \hat{U}_S} \left|1 - \frac{F_{\hat{i}\hat{j}}}{d_j^p(t)}\right| $$ +To avoid division by $0$ errors, if a `*_relative` objective is used and a demand is $0$, the coefficient of the flow $F_{\hat{i}\hat{j}}$ is set to $0$. + :::{.callout-note} In the future new optimization objectives will be introduced, for demands of basins and priorities over sources. These will be used in combination with the above, in the form of goal programming. ::: From 71dc2c099bd6cd9e920301f1ca94e59d85755caa Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Tue, 14 Nov 2023 14:27:26 +0100 Subject: [PATCH 19/21] Absolute value clarification --- docs/core/allocation.qmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/core/allocation.qmd b/docs/core/allocation.qmd index ece267a8a..6e230f08c 100644 --- a/docs/core/allocation.qmd +++ b/docs/core/allocation.qmd @@ -123,6 +123,8 @@ $$ To avoid division by $0$ errors, if a `*_relative` objective is used and a demand is $0$, the coefficient of the flow $F_{\hat{i}\hat{j}}$ is set to $0$. +The absolute value applied here is not supported in a linear programming context directly; this requires introduction of new variables and constraints. For more details see [here](https://optimization.cbe.cornell.edu/index.php?title=Optimization_with_absolute_values). + :::{.callout-note} In the future new optimization objectives will be introduced, for demands of basins and priorities over sources. These will be used in combination with the above, in the form of goal programming. ::: From 11baf16cf0de3f5dc513fd959d33f9e34150f335 Mon Sep 17 00:00:00 2001 From: Bart de Koning Date: Fri, 17 Nov 2023 09:41:44 +0100 Subject: [PATCH 20/21] Comments adressed --- core/src/allocation.jl | 21 ++------------------- docs/core/allocation.qmd | 6 ++++++ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/core/src/allocation.jl b/core/src/allocation.jl index 940200a47..90bf40210 100644 --- a/core/src/allocation.jl +++ b/core/src/allocation.jl @@ -386,19 +386,6 @@ function add_variables_flow!( return nothing end -# """ -# Add the basin allocation variables A_basin to the allocation problem. -# The variable indices are the allocation graph basin node IDs. -# Non-negativivity constraints are also immediately added to the basin allocation variables. -# """ -# function add_variables_allocation_basin!( -# problem::JuMP.Model, -# allocgraph_node_ids_basin::Vector{Int}, -# )::Nothing -# JuMP.@variable(problem, A_basin[i = allocgraph_node_ids_basin] >= 0.0) -# return nothing -# end - """ Certain allocation distribution types use absolute values in the objective function. Since most optimization packages do not support the absolute value function directly, @@ -626,11 +613,10 @@ function allocation_problem( # Add variables to problem add_variables_flow!(problem, allocgraph_edges) - # add_variables_allocation_basin!(problem, allocgraph_node_ids_basin) add_variables_absolute_value!(problem, allocgraph_edge_ids_user_demand, config) + # TODO: Add variables for allocation to basins # Add constraints to problem - # add_constraints_basin_allocation!(problem, allocgraph_node_ids_basin) add_constraints_capacity!(problem, capacity, allocgraph_edges) add_constraints_source!( problem, @@ -648,9 +634,6 @@ function allocation_problem( add_constraints_absolute_value!(problem, allocgraph_edge_ids_user_demand, config) # TODO: The fractional flow constraints - # # Add objective to problem - # add_objective_function!(problem, allocgraph_edge_ids_user_demand, config) - return problem end @@ -903,7 +886,7 @@ function allocate!(p::Parameters, allocation_model::AllocationModel, t::Float64) # A new objective function is set instead of modifying the coefficients # of an existing objective function because this is not supported for # quadratic terms: - # https://jump.dev/JuMP.jl/stable/manual/objective/#Modify-an-objective-coefficient + # https://jump.dev/JuMP.jl/v1.16/manual/objective/#Modify-an-objective-coefficient set_objective_priority!(allocation_model, user, t, priority_idx) # Solve the allocation problem for this priority diff --git a/docs/core/allocation.qmd b/docs/core/allocation.qmd index 6e230f08c..462e31e58 100644 --- a/docs/core/allocation.qmd +++ b/docs/core/allocation.qmd @@ -123,6 +123,12 @@ $$ To avoid division by $0$ errors, if a `*_relative` objective is used and a demand is $0$, the coefficient of the flow $F_{\hat{i}\hat{j}}$ is set to $0$. +For `*_absolute` objectives the optimizer cares about the actual amount of water allocated to a user, for `*_relative` objectives it cares about the fraction of the demand allocated to the user. For `quadratic_*` objectives the optimizer cares about avoiding large shortages, for `linear_*` objectives it treats all deviations equally. + +:::{.callout-note} +These options for objectives for allocation to users have not been tested thoroughly, and might change in the future. +::: + The absolute value applied here is not supported in a linear programming context directly; this requires introduction of new variables and constraints. For more details see [here](https://optimization.cbe.cornell.edu/index.php?title=Optimization_with_absolute_values). :::{.callout-note} From 4071b07046657f50ab0818bd6938c0ed22dab7eb Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Mon, 20 Nov 2023 14:14:39 +0100 Subject: [PATCH 21/21] Run Julia tests in parallel (#803) This runs our tests on 8 workers. I haven't played around with the number yet, but as-is this is over 2x faster locally. It uses https://github.com/JuliaTesting/ReTestItems.jl/. This runs each testitem in a separate module. Before we were using SafeTestSets to run each test file in a separate module. Therefore some changes were needed, to make each item self-contained. Possibly we can make things a bit nicer with the `testsetup` macro as well. This also collects the output, so overall seems a lot more like our pytest setup. It also makes it easier to run a part of the tests with a regex or tag. I removed the TeamCity and TimerOutput tests, since they were just a PoC and not actually used. I we bring the XML report back we should probably use the [ReTestItems report keyword](https://github.com/JuliaTesting/ReTestItems.jl/blob/7bdf3bb79c756555bb9a55e2902a21b413fae8d3/src/ReTestItems.jl#L157). --------- Co-authored-by: Maarten Pronk <8655030+evetion@users.noreply.github.com> --- Manifest.toml | 2 +- core/Project.toml | 8 +- .../{allocation.jl => allocation_test.jl} | 17 ++- core/test/aqua_test.jl | 4 + core/test/{bmi.jl => bmi_test.jl} | 24 ++-- core/test/{cli.jl => cli_test.jl} | 26 ++-- core/test/{config.jl => config_test.jl} | 23 ++-- core/test/{control.jl => control_test.jl} | 22 ++-- core/test/docs.jl | 9 -- core/test/docs_test.jl | 7 + core/test/{equations.jl => equations_test.jl} | 67 +++------- core/test/{io.jl => io_test.jl} | 51 ++++---- .../{libribasim.jl => libribasim_test.jl} | 10 +- .../{run_models.jl => run_models_test.jl} | 69 ++++++---- core/test/runtests.jl | 22 +--- core/test/{time.jl => time_test.jl} | 10 +- core/test/{utils.jl => utils_test.jl} | 35 ++--- .../{validation.jl => validation_test.jl} | 122 ++++++++++-------- utils/testdata.jl | 17 --- 19 files changed, 257 insertions(+), 288 deletions(-) rename core/test/{allocation.jl => allocation_test.jl} (93%) create mode 100644 core/test/aqua_test.jl rename core/test/{bmi.jl => bmi_test.jl} (75%) rename core/test/{cli.jl => cli_test.jl} (50%) rename core/test/{config.jl => config_test.jl} (84%) rename core/test/{control.jl => control_test.jl} (94%) delete mode 100644 core/test/docs.jl create mode 100644 core/test/docs_test.jl rename core/test/{equations.jl => equations_test.jl} (83%) rename core/test/{io.jl => io_test.jl} (80%) rename core/test/{libribasim.jl => libribasim_test.jl} (85%) rename core/test/{run_models.jl => run_models_test.jl} (88%) rename core/test/{time.jl => time_test.jl} (85%) rename core/test/{utils.jl => utils_test.jl} (92%) rename core/test/{validation.jl => validation_test.jl} (79%) delete mode 100644 utils/testdata.jl diff --git a/Manifest.toml b/Manifest.toml index 054d6da8c..13e077042 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.0-rc1" manifest_format = "2.0" -project_hash = "ebee755404797b896be47dd2693eb495c11ce0d0" +project_hash = "611703660e983c70d4587b9ab9281ac9924eee02" [[deps.ADTypes]] git-tree-sha1 = "5d2e21d7b0d8c22f67483ef95ebdc39c0e6b6003" diff --git a/core/Project.toml b/core/Project.toml index bab715037..06dffdf50 100644 --- a/core/Project.toml +++ b/core/Project.toml @@ -67,8 +67,8 @@ Logging = "<0.0.1,1" LoggingExtras = "1" OrdinaryDiffEq = "6.7" PreallocationTools = "0.4" +ReTestItems = "1.20" SQLite = "1.5.1" -SafeTestsets = "0.1" SciMLBase = "1.60, 2" SparseArrays = "<0.0.1,1" StructArrays = "0.6.13" @@ -76,7 +76,6 @@ TOML = "<0.0.1,1" Tables = "1" TerminalLoggers = "0.1.7" Test = "<0.0.1,1" -TestReports = "0.7" TimeZones = "=1.13.0" TimerOutputs = "0.5" TranscodingStreams = "0.9,0.10" @@ -89,11 +88,10 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" -SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -TestReports = "dcd651b4-b50a-5b6b-8f22-87e9f253a252" [targets] -test = ["Aqua", "CSV", "DataFrames", "Documenter", "IOCapture", "Logging", "SafeTestsets", "TerminalLoggers", "Test", "TestReports", "TOML"] +test = ["Aqua", "CSV", "DataFrames", "Documenter", "IOCapture", "Logging", "ReTestItems", "TerminalLoggers", "Test", "TOML"] diff --git a/core/test/allocation.jl b/core/test/allocation_test.jl similarity index 93% rename from core/test/allocation.jl rename to core/test/allocation_test.jl index 3651eb706..82e9fa755 100644 --- a/core/test/allocation.jl +++ b/core/test/allocation_test.jl @@ -1,13 +1,8 @@ -using Test -import Ribasim -import JuMP -using Test -using SQLite -using PreallocationTools: get_tmp -using DataFrames: DataFrame -using SciMLBase: successful_retcode +@testitem "Allocation solve" begin + using PreallocationTools: get_tmp + import SQLite + import JuMP -@testset "Allocation solve" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/subnetwork/ribasim.toml") @test ispath(toml_path) cfg = Ribasim.Config(toml_path) @@ -32,6 +27,10 @@ using SciMLBase: successful_retcode end @testset "Allocation objective types" begin + using DataFrames: DataFrame + import JuMP + using DataFrames: DataFrame + toml_path = normpath(@__DIR__, "../../generated_testmodels/minimal_subnetwork/ribasim.toml") @test ispath(toml_path) diff --git a/core/test/aqua_test.jl b/core/test/aqua_test.jl new file mode 100644 index 000000000..13a0c0f51 --- /dev/null +++ b/core/test/aqua_test.jl @@ -0,0 +1,4 @@ +@testitem "Aqua" begin + import Aqua + Aqua.test_all(Ribasim; ambiguities = false, persistent_tasks = false) +end diff --git a/core/test/bmi.jl b/core/test/bmi_test.jl similarity index 75% rename from core/test/bmi.jl rename to core/test/bmi_test.jl index 780d6a635..8c6c95dc7 100644 --- a/core/test/bmi.jl +++ b/core/test/bmi_test.jl @@ -1,10 +1,7 @@ -using Test -using Ribasim -import BasicModelInterface as BMI +@testitem "adaptive timestepping" begin + import BasicModelInterface as BMI -toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") - -@testset "adaptive timestepping" begin + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") model = BMI.initialize(Ribasim.Model, toml_path) @test BMI.get_time_units(model) == "s" dt0 = 0.0001269439f0 @@ -21,7 +18,10 @@ toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") @test BMI.get_current_time(model) == 86400.0 end -@testset "fixed timestepping" begin +@testitem "fixed timestepping" begin + import BasicModelInterface as BMI + + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") dt = 10.0 config = Ribasim.Config( toml_path; @@ -43,7 +43,10 @@ end @test BMI.get_current_time(model) == 2dt + 60 end -@testset "get_value_ptr" begin +@testitem "get_value_ptr" begin + import BasicModelInterface as BMI + + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") model = BMI.initialize(Ribasim.Model, toml_path) storage0 = BMI.get_value_ptr(model, "volume") @test storage0 == ones(4) @@ -54,7 +57,10 @@ end @test storage0 === storage != ones(4) end -@testset "get_value_ptr_all_values" begin +@testitem "get_value_ptr_all_values" begin + import BasicModelInterface as BMI + + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") model = BMI.initialize(Ribasim.Model, toml_path) for name in ["volume", "level", "infiltration", "drainage"] diff --git a/core/test/cli.jl b/core/test/cli_test.jl similarity index 50% rename from core/test/cli.jl rename to core/test/cli_test.jl index 64bd53af5..78bb0a727 100644 --- a/core/test/cli.jl +++ b/core/test/cli_test.jl @@ -1,19 +1,25 @@ -using Test -using Ribasim -using IOCapture: capture -using Logging: global_logger, ConsoleLogger +@testitem "version" begin + using IOCapture: capture + using Logging: global_logger, ConsoleLogger -include("../../build/ribasim_cli/src/ribasim_cli.jl") + include("../../build/ribasim_cli/src/ribasim_cli.jl") -@testset "version" begin empty!(ARGS) push!(ARGS, "--version") (; value, output) = capture(ribasim_cli.julia_main) @test value == 0 @test output == string(pkgversion(Ribasim)) + + # the global logger is modified by ribasim_cli; set it back to the default + global_logger(ConsoleLogger()) end -@testset "toml_path" begin +@testitem "toml_path" begin + using IOCapture: capture + using Logging: global_logger, ConsoleLogger + + include("../../build/ribasim_cli/src/ribasim_cli.jl") + model_path = normpath(@__DIR__, "../../generated_testmodels/basic/") toml_path = normpath(model_path, "ribasim.toml") @test ispath(toml_path) @@ -26,7 +32,7 @@ end @show error @show backtrace end -end -# the global logger is modified by ribasim_cli; set it back to the default -global_logger(ConsoleLogger()) + # the global logger is modified by ribasim_cli; set it back to the default + global_logger(ConsoleLogger()) +end diff --git a/core/test/config.jl b/core/test/config_test.jl similarity index 84% rename from core/test/config.jl rename to core/test/config_test.jl index 03cc9bc37..d3449b4cd 100644 --- a/core/test/config.jl +++ b/core/test/config_test.jl @@ -1,12 +1,9 @@ -using Test -using Ribasim -using Dates -using Configurations: UndefKeywordError -using OrdinaryDiffEq: alg_autodiff, AutoFiniteDiff, AutoForwardDiff -using CodecLz4: LZ4FrameCompressor -using CodecZstd: ZstdCompressor +@testitem "config" begin + using CodecLz4: LZ4FrameCompressor + using CodecZstd: ZstdCompressor + using Configurations: UndefKeywordError + using Dates -@testset "config" begin @test_throws UndefKeywordError Ribasim.Config() @test_throws UndefKeywordError Ribasim.Config( startime = now(), @@ -40,11 +37,15 @@ using CodecZstd: ZstdCompressor end @testset "docs" begin - Ribasim.Config(normpath(@__DIR__, "docs.toml")) + config = Ribasim.Config(normpath(@__DIR__, "docs.toml")) + @test config isa Ribasim.Config + @test config.solver.adaptive end end -@testset "Solver" begin +@testitem "Solver" begin + using OrdinaryDiffEq: alg_autodiff, AutoFiniteDiff, AutoForwardDiff + solver = Ribasim.Solver() @test solver.algorithm == "QNDF" Ribasim.Solver(; @@ -74,7 +75,7 @@ end Ribasim.algorithm(Ribasim.Solver(; algorithm = "Euler", autodiff = true)) end -@testset "snake_case" begin +@testitem "snake_case" begin @test Ribasim.snake_case("CamelCase") == "camel_case" @test Ribasim.snake_case("ABCdef") == "a_b_cdef" @test Ribasim.snake_case("snake_case") == "snake_case" diff --git a/core/test/control.jl b/core/test/control_test.jl similarity index 94% rename from core/test/control.jl rename to core/test/control_test.jl index 7418cd142..31dcb9fb8 100644 --- a/core/test/control.jl +++ b/core/test/control_test.jl @@ -1,10 +1,6 @@ -using Test -import Ribasim -using Dates: Date -using Test -using PreallocationTools: get_tmp +@testitem "Pump discrete control" begin + using PreallocationTools: get_tmp -@testset "Pump discrete control" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/pump_discrete_control/ribasim.toml") @test ispath(toml_path) @@ -50,7 +46,7 @@ using PreallocationTools: get_tmp @test all(iszero, flow) end -@testset "Flow condition control" begin +@testitem "Flow condition control" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/flow_condition/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) @@ -71,7 +67,7 @@ end @test isapprox(flow_t_control_ahead, greater_than, rtol = 0.005) end -@testset "Transient level boundary condition control" begin +@testitem "Transient level boundary condition control" begin toml_path = normpath( @__DIR__, "../../generated_testmodels/level_boundary_condition/ribasim.toml", @@ -95,7 +91,7 @@ end @test isapprox(level_t_control_ahead, greater_than, rtol = 0.005) end -@testset "PID control" begin +@testitem "PID control" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/pid_control/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) @@ -134,7 +130,9 @@ end ) end -@testset "TabulatedRatingCurve control" begin +@testitem "TabulatedRatingCurve control" begin + using Dates: Date + toml_path = normpath( @__DIR__, "../../generated_testmodels/tabulated_rating_curve_control/ribasim.toml", @@ -153,7 +151,7 @@ end @test only(p.tabulated_rating_curve.tables).t[2] == 1.2 end -@testset "Setpoint with bounds control" begin +@testitem "Setpoint with bounds control" begin toml_path = normpath( @__DIR__, "../../generated_testmodels/level_setpoint_with_minmax/ribasim.toml", @@ -183,7 +181,7 @@ end @test level[t_2_none_index] ≈ setpoint end -@testset "Set PID target with DiscreteControl" begin +@testitem "Set PID target with DiscreteControl" begin toml_path = normpath( @__DIR__, "../../generated_testmodels/discrete_control_of_pid_control/ribasim.toml", diff --git a/core/test/docs.jl b/core/test/docs.jl deleted file mode 100644 index 5110358f6..000000000 --- a/core/test/docs.jl +++ /dev/null @@ -1,9 +0,0 @@ -using Test -using Documenter -using Ribasim - -DocMeta.setdocmeta!(Ribasim, :DocTestSetup, :(using Ribasim); recursive = true) - -@testset "Doctests" begin - doctest(Ribasim; manual = false) -end diff --git a/core/test/docs_test.jl b/core/test/docs_test.jl new file mode 100644 index 000000000..1f2e78f8c --- /dev/null +++ b/core/test/docs_test.jl @@ -0,0 +1,7 @@ +@testitem "Doctests" begin + using Documenter + + DocMeta.setdocmeta!(Ribasim, :DocTestSetup, :(using Ribasim); recursive = true) + + doctest(Ribasim; manual = false) +end diff --git a/core/test/equations.jl b/core/test/equations_test.jl similarity index 83% rename from core/test/equations.jl rename to core/test/equations_test.jl index 7a6d25b43..dfd700609 100644 --- a/core/test/equations.jl +++ b/core/test/equations_test.jl @@ -1,49 +1,3 @@ -using Test -using Dates -using Ribasim -using Arrow -import BasicModelInterface as BMI -using SciMLBase: successful_retcode -using TimerOutputs -using PreallocationTools: get_tmp - -include("../../utils/testdata.jl") - -datadir = normpath(@__DIR__, "../../generated_testmodels") - -TimerOutputs.enable_debug_timings(Ribasim) # causes recompilation (!) - -@timeit_debug to "qh_relation" @testset "qh_relation" begin - # Basin without forcing - # TODO test QH relation - sleep(0.1) -end - -# show(Ribasim.to) # commented out to avoid spamming the test output -is_running_under_teamcity() && teamcity_message("qh_relation", TimerOutputs.todict(to)) -reset_timer!(Ribasim.to) - -@timeit_debug to "forcing_eqs" @testset "forcing_eqs" begin - # TODO test forcing - sleep(0.05) -end - -# show(Ribasim.to) # commented out to avoid spamming the test output -is_running_under_teamcity() && teamcity_message("forcing_eqs", TimerOutputs.todict(to)) -TimerOutputs.disable_debug_timings(Ribasim) # causes recompilation (!) - -# @testset "bifurcation" begin - -# end - -# @testset "conservation of flow" begin - -# end - -# @testset "salinity" begin - -# end - # # Node equation tests # # The tests below are for the equations of flow associated with particular node types. @@ -61,7 +15,9 @@ TimerOutputs.disable_debug_timings(Ribasim) # causes recompilation (!) # Equation: storage' = -(2*level(storage)-C)/resistance, storage(t0) = storage0 # Solution: storage(t) = limit_storage + (storage0 - limit_storage)*exp(-t/(basin_area*resistance)) # Here limit_storage is the storage at which the level of the basin is equal to the level of the level boundary -@testset "LinearResistance" begin +@testitem "LinearResistance" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/linear_resistance/ribasim.toml") @test ispath(toml_path) @@ -86,7 +42,9 @@ end # Equation: w' = -α/basin_area * w^2, w = (level(storage) - level_min)/basin_area # Solution: w = 1/(α(t-t0)/basin_area + 1/w(t0)), # storage = storage_min + 1/(α(t-t0)/basin_area^2 + 1/(storage(t0)-storage_min)) -@testset "TabulatedRatingCurve" begin +@testitem "TabulatedRatingCurve" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/rating_curve/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) @@ -118,7 +76,9 @@ end # Solution: (implicit, given by Wolfram Alpha). # Note: The Wolfram Alpha solution contains a factor of the hypergeometric function 2F1, but these values are # so close to 1 that they are omitted. -@testset "ManningResistance" begin +@testitem "ManningResistance" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/manning_resistance/ribasim.toml") @test ispath(toml_path) @@ -153,7 +113,9 @@ end # The second order linear inhomogeneous ODE for this model is derived by # differentiating the equation for the storage of the controlled basin # once to time to get rid of the integral term. -@testset "PID control" begin +@testitem "PID control" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/pid_control_equation/ribasim.toml") @test ispath(toml_path) @@ -197,7 +159,10 @@ end # storage1 = storage1(t0) + (t-t0)*(frac*q_boundary - q_pump) # storage2 = storage2(t0) + (t-t0)*q_pump # Note: uses Euler algorithm -@testset "MiscellaneousNodes" begin +@testitem "MiscellaneousNodes" begin + using PreallocationTools: get_tmp + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/misc_nodes/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) diff --git a/core/test/io.jl b/core/test/io_test.jl similarity index 80% rename from core/test/io.jl rename to core/test/io_test.jl index f058f9067..7a00176ec 100644 --- a/core/test/io.jl +++ b/core/test/io_test.jl @@ -1,16 +1,5 @@ -import Arrow -import Legolas -import SQLite -import Tables -using Dates -using Ribasim -using StructArrays: StructVector -using Test -using TestReports - -recordproperty("name", "Input/Output") # TODO To check in TeamCity - -@testset "relativepath" begin +@testitem "relativepath" begin + using Dates # relative to tomldir config = Ribasim.Config(; @@ -39,7 +28,9 @@ recordproperty("name", "Input/Output") # TODO To check in TeamCity @test Ribasim.input_path(config, "/path/to/file") == abspath("/path/to/file") end -@testset "time" begin +@testitem "time" begin + using Dates + t0 = DateTime(2020) @test Ribasim.datetime_since(0.0, t0) === t0 @test Ribasim.datetime_since(1.0, t0) === t0 + Second(1) @@ -49,26 +40,32 @@ end @test Ribasim.seconds_since(DateTime("2020-01-01T00:00:03.142"), t0) ≈ 3.142 end -@testset "findlastgroup" begin +@testitem "findlastgroup" begin @test Ribasim.findlastgroup(2, [5, 4, 2, 2, 5, 2, 2, 2, 1]) === 6:8 @test Ribasim.findlastgroup(2, [2]) === 1:1 @test Ribasim.findlastgroup(3, [5, 4, 2, 2, 5, 2, 2, 2, 1]) === 1:0 end -"Convert an in-memory table to a memory mapped Arrow table" -function to_arrow_table( - path, - table::StructVector{T}, -)::StructVector{T} where {T <: Legolas.AbstractRecord} - open(path; write = true) do io - Arrow.write(io, table) +@testitem "table sort" begin + import Arrow + import Legolas + using StructArrays: StructVector + import SQLite + import Tables + + "Convert an in-memory table to a memory mapped Arrow table" + function to_arrow_table( + path, + table::StructVector{T}, + )::StructVector{T} where {T <: Legolas.AbstractRecord} + open(path; write = true) do io + Arrow.write(io, table) + end + table = Arrow.Table(path) + nt = Tables.columntable(table) + return StructVector{T}(nt) end - table = Arrow.Table(path) - nt = Tables.columntable(table) - return StructVector{T}(nt) -end -@testset "table sort" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/basic_transient/ribasim.toml") config = Ribasim.Config(toml_path) diff --git a/core/test/libribasim.jl b/core/test/libribasim_test.jl similarity index 85% rename from core/test/libribasim.jl rename to core/test/libribasim_test.jl index 9dec51a90..3bcac1fc0 100644 --- a/core/test/libribasim.jl +++ b/core/test/libribasim_test.jl @@ -1,12 +1,10 @@ -using Test -using Ribasim -import BasicModelInterface as BMI +@testitem "libribasim" begin + import BasicModelInterface as BMI -include("../../build/libribasim/src/libribasim.jl") + include("../../build/libribasim/src/libribasim.jl") -toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") -@testset "libribasim" begin # data from which we create pointers for libribasim time = [-1.0] var_name = "volume" diff --git a/core/test/run_models.jl b/core/test/run_models_test.jl similarity index 88% rename from core/test/run_models.jl rename to core/test/run_models_test.jl index 6bdb6eaf0..a0ea08c44 100644 --- a/core/test/run_models.jl +++ b/core/test/run_models_test.jl @@ -1,14 +1,6 @@ -using Dates -using Logging: Debug, with_logger -using Test -using Ribasim -import BasicModelInterface as BMI -using SciMLBase: successful_retcode -import Tables -using PreallocationTools: get_tmp -using DataFrames: DataFrame - -@testset "trivial model" begin +@testitem "trivial model" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/trivial/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) @@ -16,7 +8,9 @@ using DataFrames: DataFrame @test successful_retcode(model) end -@testset "bucket model" begin +@testitem "bucket model" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/bucket/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) @@ -24,7 +18,12 @@ end @test successful_retcode(model) end -@testset "basic model" begin +@testitem "basic model" begin + using Logging: Debug, with_logger + using SciMLBase: successful_retcode + import Tables + using Dates + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") @test ispath(toml_path) @@ -66,7 +65,9 @@ end end end -@testset "basic arrow model" begin +@testitem "basic arrow model" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic_arrow/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) @@ -74,7 +75,9 @@ end @test successful_retcode(model) end -@testset "basic transient model" begin +@testitem "basic transient model" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic_transient/ribasim.toml") @test ispath(toml_path) @@ -86,7 +89,9 @@ end Sys.isapple() end -@testset "sparse and AD/FDM jac solver options" begin +@testitem "sparse and AD/FDM jac solver options" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic_transient/ribasim.toml") @@ -109,7 +114,9 @@ end @test dense_fdm.integrator.sol.u[end] ≈ sparse_ad.integrator.sol.u[end] atol = 1e-3 end -@testset "TabulatedRatingCurve model" begin +@testitem "TabulatedRatingCurve model" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/tabulated_rating_curve/ribasim.toml") @test ispath(toml_path) @@ -121,12 +128,14 @@ end @test model.integrator.p.tabulated_rating_curve.tables[end].t[end] == 1.2 end -"Shorthand for Ribasim.get_area_and_level" -function lookup(profile, S) - Ribasim.get_area_and_level(profile.S, profile.A, profile.h, S)[1:2] -end +@testitem "Profile" begin + import Tables + + "Shorthand for Ribasim.get_area_and_level" + function lookup(profile, S) + Ribasim.get_area_and_level(profile.S, profile.A, profile.h, S)[1:2] + end -@testset "Profile" begin n_interpolations = 100 storage = range(0.0, 1000.0, n_interpolations) @@ -168,11 +177,15 @@ end @test A ≈ 10 * h end -@testset "Outlet constraints" begin +@testitem "Outlet constraints" begin + using DataFrames: DataFrame + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/outlet/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) + @test successful_retcode(model) p = model.integrator.p (; level_boundary, outlet) = p (; level) = level_boundary @@ -197,10 +210,13 @@ end all(isapprox.(level_basin[timesteps .>= t_maximum_level], level.u[3], atol = 5e-2)) end -@testset "User" begin +@testitem "User" begin + using SciMLBase: successful_retcode + toml_path = normpath(@__DIR__, "../../generated_testmodels/user/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) + @test successful_retcode(model) day = 86400.0 @test only(model.integrator.sol(0day)) == 1000.0 @@ -210,7 +226,10 @@ end @test only(model.integrator.sol(180day)) ≈ 509 atol = 1 end -@testset "ManningResistance" begin +@testitem "ManningResistance" begin + using PreallocationTools: get_tmp + using SciMLBase: successful_retcode + """ Apply the "standard step method" finite difference method to find a backwater curve. diff --git a/core/test/runtests.jl b/core/test/runtests.jl index ae80b3db2..442308612 100644 --- a/core/test/runtests.jl +++ b/core/test/runtests.jl @@ -1,21 +1,3 @@ -import Aqua -import Ribasim -using Test: @testset -using SafeTestsets: @safetestset +using ReTestItems, Ribasim -@testset "Ribasim" begin - @safetestset "Input/Output" include("io.jl") - @safetestset "Configuration" include("config.jl") - @safetestset "Validation" include("validation.jl") - @safetestset "Equations" include("equations.jl") - @safetestset "Run Test Models" include("run_models.jl") - @safetestset "Basic Model Interface" include("bmi.jl") - @safetestset "Utility functions" include("utils.jl") - @safetestset "Control" include("control.jl") - @safetestset "Allocation" include("allocation.jl") - @safetestset "Time" include("time.jl") - @safetestset "Docs" include("docs.jl") - @safetestset "Command Line Interface" include("cli.jl") - @safetestset "libribasim" include("libribasim.jl") - Aqua.test_all(Ribasim; ambiguities = false, persistent_tasks = false) -end +runtests(Ribasim; nworkers = min(4, Sys.CPU_THREADS ÷ 2), nworker_threads = 2) diff --git a/core/test/time.jl b/core/test/time_test.jl similarity index 85% rename from core/test/time.jl rename to core/test/time_test.jl index 69cd408ef..02f73af5f 100644 --- a/core/test/time.jl +++ b/core/test/time_test.jl @@ -1,13 +1,13 @@ -using Ribasim -using Dates -using DataFrames: DataFrame -using Test +@testitem "Time dependent flow boundary" begin + using Dates + using DataFrames: DataFrame + using SciMLBase: successful_retcode -@testset "Time dependent flow boundary" begin toml_path = normpath(@__DIR__, "../../generated_testmodels/flow_boundary_time/ribasim.toml") @test ispath(toml_path) model = Ribasim.run(toml_path) + @test successful_retcode(model) flow = DataFrame(Ribasim.flow_table(model)) # only from March to September the FlowBoundary varies diff --git a/core/test/utils.jl b/core/test/utils_test.jl similarity index 92% rename from core/test/utils.jl rename to core/test/utils_test.jl index 1237d4f92..9e007b089 100644 --- a/core/test/utils.jl +++ b/core/test/utils_test.jl @@ -1,25 +1,22 @@ -using Ribasim -using Dictionaries: Indices -using Test -using DataInterpolations: LinearInterpolation -using StructArrays: StructVector -using SQLite -using Logging - -@testset "id_index" begin +@testitem "id_index" begin + using Dictionaries: Indices + ids = Indices([2, 4, 6]) @test Ribasim.id_index(ids, 4) === (true, 2) @test Ribasim.id_index(ids, 5) === (false, 0) end -@testset "profile_storage" begin +@testitem "profile_storage" begin @test Ribasim.profile_storage([0.0, 1.0], [0.0, 1000.0]) == [0.0, 500.0] @test Ribasim.profile_storage([6.0, 7.0], [0.0, 1000.0]) == [0.0, 500.0] @test Ribasim.profile_storage([6.0, 7.0, 9.0], [0.0, 1000.0, 1000.0]) == [0.0, 500.0, 2500.0] end -@testset "bottom" begin +@testitem "bottom" begin + using Dictionaries: Indices + using StructArrays: StructVector + # create two basins with different bottoms/levels area = [[0.01, 1.0], [0.01, 1.0]] level = [[0.0, 1.0], [4.0, 5.0]] @@ -54,7 +51,11 @@ end ) end -@testset "Convert levels to storages" begin +@testitem "Convert levels to storages" begin + using Dictionaries: Indices + using StructArrays: StructVector + using Logging + level = [ 0.0, 0.42601923740838954, @@ -114,7 +115,7 @@ end @test storages ≈ storages_ end -@testset "Expand logic_mapping" begin +@testitem "Expand logic_mapping" begin logic_mapping = Dict{Tuple{Int, String}, String}() logic_mapping[(1, "*T*")] = "foo" logic_mapping[(2, "FF")] = "bar" @@ -159,7 +160,9 @@ end ) end -@testset "Jacobian sparsity" begin +@testitem "Jacobian sparsity" begin + import SQLite + toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml") cfg = Ribasim.Config(toml_path) @@ -191,7 +194,7 @@ end @test jac_prototype.nzval == ones(5) end -@testset "FlatVector" begin +@testitem "FlatVector" begin vv = [[2.2, 3.2], [4.3, 5.3], [6.4, 7.4]] fv = Ribasim.FlatVector(vv) @test length(fv) == 6 @@ -207,7 +210,7 @@ end @test length(fv) == 0 end -@testset "reduction_factor" begin +@testitem "reduction_factor" begin @test Ribasim.reduction_factor(-2.0, 2.0) === 0.0 @test Ribasim.reduction_factor(0.0f0, 2.0) === 0.0f0 @test Ribasim.reduction_factor(0.0, 2.0) === 0.0 diff --git a/core/test/validation.jl b/core/test/validation_test.jl similarity index 79% rename from core/test/validation.jl rename to core/test/validation_test.jl index a193cb816..c9097a194 100644 --- a/core/test/validation.jl +++ b/core/test/validation_test.jl @@ -1,13 +1,7 @@ -using Test -using Ribasim -using Graphs: DiGraph, add_edge! -using Dictionaries: Indices -using DataInterpolations: LinearInterpolation -import SQLite -using Logging -using Test - -@testset "Basin profile validation" begin +@testitem "Basin profile validation" begin + using Dictionaries: Indices + using DataInterpolations: LinearInterpolation + node_id = Indices([1]) level = [[0.0, 0.0, 1.0]] area = [[0.0, 100.0, 90]] @@ -27,7 +21,10 @@ using Test @test itp isa LinearInterpolation end -@testset "Q(h) validation" begin +@testitem "Q(h) validation" begin + import SQLite + using Logging + toml_path = normpath(@__DIR__, "../../generated_testmodels/invalid_qh/ribasim.toml") @test ispath(toml_path) @@ -51,7 +48,10 @@ end "A Q(h) relationship for TabulatedRatingCurve \"\" #2 from the time table has repeated levels, this can not be interpolated." end -@testset "Neighbor count validation" begin +@testitem "Neighbor count validation" begin + using Graphs: DiGraph, add_edge! + using Logging + graph_flow = DiGraph(6) add_edge!(graph_flow, 2, 1) add_edge!(graph_flow, 3, 1) @@ -114,7 +114,11 @@ end ) end -@testset "PidControl connectivity validation" begin +@testitem "PidControl connectivity validation" begin + using Graphs: DiGraph, add_edge! + using Dictionaries: Indices + using Logging + pid_control_node_id = [1, 6] pid_control_listen_node_id = [3, 5] pump_node_id = [2, 4] @@ -150,50 +154,53 @@ end "Listen node #5 of PidControl node #6 is not upstream of controlled pump #2" end -# This test model is not written on Ubuntu CI, see #479 -if !Sys.islinux() - @testset "FractionalFlow validation" begin - toml_path = normpath( - @__DIR__, - "../../generated_testmodels/invalid_fractional_flow/ribasim.toml", +@testitem "FractionalFlow validation" begin + import SQLite + using Logging + + toml_path = normpath( + @__DIR__, + "../../generated_testmodels/invalid_fractional_flow/ribasim.toml", + ) + @test ispath(toml_path) + + config = Ribasim.Config(toml_path) + db_path = Ribasim.input_path(config, config.database) + db = SQLite.DB(db_path) + p = Ribasim.Parameters(db, config) + (; connectivity, fractional_flow) = p + + logger = TestLogger() + with_logger(logger) do + @test !Ribasim.valid_fractional_flow( + connectivity.graph_flow, + fractional_flow.node_id, + fractional_flow.control_mapping, ) - @test ispath(toml_path) - - config = Ribasim.Config(toml_path) - db_path = Ribasim.input_path(config, config.database) - db = SQLite.DB(db_path) - p = Ribasim.Parameters(db, config) - (; connectivity, fractional_flow) = p - - logger = TestLogger() - with_logger(logger) do - @test !Ribasim.valid_fractional_flow( - connectivity.graph_flow, - fractional_flow.node_id, - fractional_flow.control_mapping, - ) - end - - @test length(logger.logs) == 3 - @test logger.logs[1].level == Error - @test logger.logs[1].message == - "Node #7 combines fractional flow outneighbors with other outneigbor types." - @test logger.logs[2].level == Error - @test logger.logs[2].message == - "Fractional flow nodes must have non-negative fractions." - @test logger.logs[2].kwargs[:node_id] == 3 - @test logger.logs[2].kwargs[:fraction] ≈ -0.1 - @test logger.logs[2].kwargs[:control_state] == "" - @test logger.logs[3].level == Error - @test logger.logs[3].message == - "The sum of fractional flow fractions leaving a node must be ≈1." - @test logger.logs[3].kwargs[:node_id] == 7 - @test logger.logs[3].kwargs[:fraction_sum] ≈ 0.4 - @test logger.logs[3].kwargs[:control_state] == "" end + + @test length(logger.logs) == 3 + @test logger.logs[1].level == Error + @test logger.logs[1].message == + "Node #7 combines fractional flow outneighbors with other outneigbor types." + @test logger.logs[2].level == Error + @test logger.logs[2].message == + "Fractional flow nodes must have non-negative fractions." + @test logger.logs[2].kwargs[:node_id] == 3 + @test logger.logs[2].kwargs[:fraction] ≈ -0.1 + @test logger.logs[2].kwargs[:control_state] == "" + @test logger.logs[3].level == Error + @test logger.logs[3].message == + "The sum of fractional flow fractions leaving a node must be ≈1." + @test logger.logs[3].kwargs[:node_id] == 7 + @test logger.logs[3].kwargs[:fraction_sum] ≈ 0.4 + @test logger.logs[3].kwargs[:control_state] == "" end -@testset "DiscreteControl logic validation" begin +@testitem "DiscreteControl logic validation" begin + import SQLite + using Logging + toml_path = normpath( @__DIR__, "../../generated_testmodels/invalid_discrete_control/ribasim.toml", @@ -228,7 +235,9 @@ end "Negative look ahead supplied for listen variable 'flow_rate' from listen node #4." end -@testset "Pump/outlet flow rate sign validation" begin +@testitem "Pump/outlet flow rate sign validation" begin + using Logging + logger = TestLogger() with_logger(logger) do @@ -270,7 +279,10 @@ end "Pump flow rates must be non-negative, found -1.0 for control state 'foo' of #1." end -@testset "Edge type validation" begin +@testitem "Edge type validation" begin + import SQLite + using Logging + toml_path = normpath(@__DIR__, "../../generated_testmodels/invalid_edge_types/ribasim.toml") @test ispath(toml_path) diff --git a/utils/testdata.jl b/utils/testdata.jl deleted file mode 100644 index 7970c76d1..000000000 --- a/utils/testdata.jl +++ /dev/null @@ -1,17 +0,0 @@ -const teamcity_presence_env_var = "TEAMCITY_VERSION" - -is_running_under_teamcity() = haskey(ENV, teamcity_presence_env_var) - -function teamcity_message(name, value) - println("##teamcity['$name' '$value']") -end - -function teamcity_message(name, d::Dict) - println( - "##teamcity[$name " * - string(collect(("'$(key)'='$value' " for (key, value) in pairs(d)))...) * - "]", - ) -end - -nothing