From a0a744aadc55c1d95353895b8ad21881ad9f636f Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 1 Dec 2024 19:25:20 +1100 Subject: [PATCH] Mass WIP commit to reflect changes in Graph representation --- src/Network.jl | 75 ++++++++++++++------- src/Nodes/DamNode.jl | 78 ++++++++++++++------- src/Streamfall.jl | 157 ++++++++++++++++++++++++------------------- 3 files changed, 193 insertions(+), 117 deletions(-) diff --git a/src/Network.jl b/src/Network.jl index 5bb94e7..c5c10a8 100644 --- a/src/Network.jl +++ b/src/Network.jl @@ -1,18 +1,33 @@ +using OrderedCollections + using Cairo, Compose using Graphs, MetaGraphs, GraphPlot using ModelParameters -import YAML: write_file +import YAML: load_file, write_file struct StreamfallNetwork mg::MetaDiGraph end +function load_network(name::String, fn::String) + network = load_file( + fn; + dicttype=OrderedDict + ) + + return create_network(name, network) +end + Base.getindex(sn::StreamfallNetwork, n::String) = get_node(sn, n) Base.getindex(sn::StreamfallNetwork, nid::Int) = get_node(sn, nid) +function node_names(sn::StreamfallNetwork) + verts = vertices(sn.mg) + return [sn[v].name for v in verts] +end function set_prop!(sn::StreamfallNetwork, nid::Int64, prop::Symbol, var::Any)::Nothing MetaGraphs.set_prop!(sn.mg, nid, prop, var) @@ -111,15 +126,15 @@ end """ - create_node(mg::MetaDiGraph, node_name::String, details::Dict, nid::Int) + create_node(mg::MetaDiGraph, node_name::String, details::OrderedDict, nid::Int) Create a node specified with given name (if it does not exist). -Returns -- `this_id`, ID of node (if pre-existing) and +Returns +- `this_id`, ID of node (if pre-existing) and - `nid`, incremented node id for entire network (equal to `this_id` if exists) """ -function create_node(mg::MetaDiGraph, node_name::String, details::Dict, nid::Int) +function create_node(mg::MetaDiGraph, node_name::String, details::OrderedDict, nid::Int) details = copy(details) match = collect(MetaGraphs.filter_vertices(mg, :name, node_name)) @@ -149,46 +164,47 @@ function create_node(mg::MetaDiGraph, node_name::String, details::Dict, nid::Int set_props!(mg, nid, Dict(:name=>node_name, :node=>n, :nfunc=>func)) - + this_id = nid - nid = nid + 1 else this_id = match[1] end - return this_id, nid + return this_id end """ - create_network(name::String, network::Dict)::StreamfallNetwork + create_network(name::String, network::OrderedDict)::StreamfallNetwork Create a StreamNetwork from a YAML-derived specification. # Example ```julia-repl -julia> network_spec = YAML.load_file("example_network.yml") +julia> using OrderedCollections +julia> network_spec = YAML.load_file("example_network.yml"; dicttype=OrderedDict{Any,Any}) julia> sn = create_network("Example Network", network_spec) ``` """ -function create_network(name::String, network::Dict)::StreamfallNetwork +function create_network(name::String, network::OrderedDict)::StreamfallNetwork num_nodes = length(network) mg = MetaDiGraph(num_nodes) MetaGraphs.set_prop!(mg, :description, name) - - nid = 1 - for (node, details) in network + + for (nid, (node, details)) in enumerate(network) n_name = string(node) - this_id, nid = create_node(mg, n_name, details, nid) + this_id = create_node(mg, n_name, details, nid) if haskey(details, "inlets") inlets = details["inlets"] - if !isnothing(inlets) for inlet in inlets - in_id, nid = create_node(mg, string(inlet), network[inlet], nid) - add_edge!(mg, in_id => this_id) + + inlet_id = findall(keys(network) .== inlet)[1] + + in_id = create_node(mg, string(inlet), network[inlet], inlet_id) + add_edge!(mg, inlet_id => nid) end end end @@ -200,8 +216,9 @@ function create_network(name::String, network::Dict)::StreamfallNetwork @assert length(outlets) <= 1 || throw(ArgumentError(msg)) for outlet in outlets - out_id, nid = create_node(mg, string(outlet), network[outlet], nid) - add_edge!(mg, this_id => out_id) + outlet_id = findall(keys(network) .== outlet)[1] + out_id = create_node(mg, string(outlet), network[outlet], outlet_id) + add_edge!(mg, nid => outlet_id) end end end @@ -303,10 +320,18 @@ function Base.show(io::IO, sn::StreamfallNetwork) vs = vertices(sn.mg) - for nid in vs - println(io, "Node $(nid) : \n") + show_verts = vs + if length(vs) > 4 + show_verts = [1, 2, nothing, length(vs)-1, length(vs)] + end + + for nid in show_verts + if isnothing(nid) + println(io, "⋮ \n") + continue + end + println(io, "Node $(nid): \n") show(io, sn[nid]) - print("\n") end end @@ -317,7 +342,7 @@ end Simple plot of stream network. """ function plot_network(sn::StreamfallNetwork; as_html=false) - node_names = ["$(n.name)" for n in sn] + node_labels = ["$(sn[i].name)\n"*string(nameof(typeof(sn[i]))) for i in vertices(sn.mg)] if as_html plot_func = gplothtml @@ -325,7 +350,7 @@ function plot_network(sn::StreamfallNetwork; as_html=false) plot_func = gplot end - plot_func(sn.mg, nodelabel=node_names) + plot_func(sn.mg, nodelabel=node_labels) end diff --git a/src/Nodes/DamNode.jl b/src/Nodes/DamNode.jl index b7deaf7..f851380 100644 --- a/src/Nodes/DamNode.jl +++ b/src/Nodes/DamNode.jl @@ -56,33 +56,31 @@ Base.@kwdef mutable struct DamNode{P, A<:AbstractFloat} <: NetworkNode level::Array{A} = [] discharge::Array{A} = [] outflow::Array{A} = [] - end function DamNode( name::String, - area::Float64, - max_storage::Float64, - storage_coef::Float64, - initial_storage::Float64, + area::F, + max_storage::F, + storage_coef::F, + initial_storage::F, calc_dam_level::Function, calc_dam_area::Function, calc_dam_discharge::Function, - calc_dam_outflow::Function) + calc_dam_outflow::Function) where {F<:Float64} return DamNode(name, area, max_storage, storage_coef, calc_dam_level, calc_dam_area, calc_dam_discharge, calc_dam_outflow, - [initial_storage], [], [], [], [], [], [], []) + F[initial_storage], F[], F[], F[], F[], F[], F[], F[]) end - """ DamNode(name::String, spec::Dict) Create DamNode from a given specification. """ -function DamNode(name::String, spec::Dict) - n = DamNode{Param}(; name=name, area=spec["area"], +function DamNode(name::String, spec::Union{Dict,OrderedDict}) + n = DamNode{Param,Float64}(; name=name, area=spec["area"], max_storage=spec["max_storage"]) node_params = spec["parameters"] @@ -127,8 +125,22 @@ function storage(node::DamNode) return last(node.storage) end +function prep_state!(node::DamNode, timesteps::Int64) + resize!(node.storage, timesteps+1) + node.storage[2:end] .= 0.0 + + node.effective_rainfall = zeros(timesteps) + node.et = zeros(timesteps) + node.inflow = zeros(timesteps) + node.dam_area = zeros(timesteps) + + node.level = zeros(timesteps) + node.discharge = zeros(timesteps) + node.outflow = zeros(timesteps) +end + -function update_state(node::DamNode, storage, rainfall, et, area, discharge, outflow) +function update_state!(node::DamNode, storage, rainfall, et, area, discharge, outflow) push!(node.storage, storage) push!(node.effective_rainfall, rainfall) push!(node.et, et) @@ -139,6 +151,18 @@ function update_state(node::DamNode, storage, rainfall, et, area, discharge, out return nothing end +function update_state!(node::DamNode, ts::Int64, storage, rainfall, et, area, discharge, outflow) + node.storage[ts+1] = storage + + node.effective_rainfall[ts] = rainfall + node.et[ts] = et + node.level[ts] = node.calc_dam_level(storage) + node.dam_area[ts] = area + node.discharge[ts] = discharge + node.outflow[ts] = outflow + + return nothing +end """ @@ -172,10 +196,14 @@ end function run_node!(node::DamNode, climate::Climate; inflow=nothing, extraction=nothing, exchange=nothing) timesteps = sim_length(climate) + prep_state!(node, timesteps) + for ts in 1:timesteps run_node!(node, climate, ts; inflow=inflow, extraction=extraction, exchange=exchange) end + + return nothing end @@ -194,23 +222,25 @@ Run a specific node for a specified time step. - `exchange::DataFrame` : Time series of groundwater flux """ function run_node!(node::DamNode, climate::Climate, timestep::Int; - inflow=nothing, extraction=nothing, exchange=nothing) + inflow=nothing, extraction=nothing, exchange=nothing)::Nothing ts = timestep - if checkbounds(Bool, node.outflow, ts) - if node.outflow[ts] != undef - # already ran for this time step so no need to run - return node.outflow[ts], node.level[ts] - end - end + # if checkbounds(Bool, node.outflow, ts) + # if node.outflow[ts] != undef + # # already ran for this time step so no need to run + # return node.outflow[ts], node.level[ts] + # end + # end node_name = node.name rain, et = climate_values(node, climate, ts) wo = timestep_value(ts, node_name, "releases", extraction) ex = timestep_value(ts, node_name, "exchange", exchange) in_flow = timestep_value(ts, node_name, "inflow", inflow) - vol = node.storage[ts] + current_vol = node.storage[ts] + + run_node!(node, ts, rain, et, current_vol, in_flow, wo, ex) - return run_node!(node, rain, et, vol, in_flow, wo, ex) + return nothing end @@ -229,13 +259,13 @@ Calculate outflow for the dam node for a single time step. - outflow from dam """ function run_node!(node::DamNode, + ts::Int64, rain::Float64, et::Float64, - vol::Float64, + volume::Float64, inflow::Float64, extractions::Float64, gw_flux::Float64) - volume = vol dam_area = node.calc_dam_area(volume) discharge = node.calc_dam_discharge(volume, node.max_storage) @@ -243,9 +273,9 @@ function run_node!(node::DamNode, dam_area, extractions, discharge, node.max_storage) outflow = node.calc_dam_outflow(discharge, extractions) - update_state(node, updated_store, rain, et, dam_area, discharge, outflow) + update_state!(node, ts, updated_store, rain, et, dam_area, discharge, outflow) - return outflow, level(node) + return nothing end diff --git a/src/Streamfall.jl b/src/Streamfall.jl index 96f3a2a..9a7119c 100644 --- a/src/Streamfall.jl +++ b/src/Streamfall.jl @@ -40,9 +40,10 @@ function timestep_value(ts::Int, gauge_id::String, col_partial::String, dataset: names(dataset) ) + amount = 0.0 if !isempty(target_col) - amount = if checkbounds(Bool, dataset.Date, timestep) - dataset[timestep, target_col][1] + amount = if checkbounds(Bool, dataset.Date, ts) + dataset[ts, target_col][1] else 0.0 end @@ -98,60 +99,6 @@ function align_time_frame(timeseries::T...) where {T<:DataFrame} return modded end - -""" - run_node!(sn::StreamfallNetwork, node_id::Int, climate::Climate, timestep::Int; - extraction::Union{DataFrame, Nothing}=nothing, - exchange::Union{DataFrame, Nothing}=nothing) - -Run a model attached to a node for a given time step. -Recurses upstream as needed. - -# Arguments -- `sn::StreamfallNetwork` -- `node_id::Int` -- `climate::Climate` -- `timestep::Int` -- `inflow::DataFrame` : Additional inflow to consider (in ML/timestep) -- `extraction::DataFrame` : Volume of water to be extracted (in ML/timestep) -- `exchange::DataFrame` : Volume of flux (in ML/timestep), where negative values are losses to the groundwater system -""" -function run_node!(sn::StreamfallNetwork, node_id::Int, climate::Climate, timestep::Int; - inflow::Union{DataFrame, Nothing}=nothing, - extraction::Union{DataFrame, Nothing}=nothing, - exchange::Union{DataFrame, Nothing}=nothing) - ts = timestep - - sim_inflow = 0.0 - ins = inneighbors(sn.mg, node_id) - - if !isempty(ins) - for i in ins - # Get inflow from previous node - res = run_node!(sn, i, climate, timestep; - inflow=inflow, - extraction=extraction, - exchange=exchange) - if res isa Number - sim_inflow += res - elseif length(res) > 1 - # get outflow from (outflow, level) - sim_inflow += res[1] - end - end - end - - node = sn[node_id] - ts_inflow = timestep_value(ts, node.name, "inflow", inflow) - ts_inflow += sim_inflow - - run_func! = get_prop(sn, node_id, :nfunc) - - # Run for a time step, dependent on previous state - run_func!(node, climate, ts; inflow=ts_inflow, extraction=extraction, exchange=exchange) -end - - """ run_basin!(sn::StreamfallNetwork, climate::Climate; inflow=nothing, extraction=nothing, exchange=nothing) @@ -160,8 +107,8 @@ Run scenario for an entire catchment/basin. function run_basin!(sn::StreamfallNetwork, climate::Climate; inflow=nothing, extraction=nothing, exchange=nothing) _, outlets = find_inlets_and_outlets(sn) - @inbounds for outlet in outlets - run_node!(sn, outlet, climate; + @inbounds for outlet_id in outlets + run_node!(sn, outlet_id, climate; inflow=inflow, extraction=extraction, exchange=exchange) end end @@ -171,7 +118,7 @@ run_catchment! = run_basin! """ run_node!(sn::StreamfallNetwork, node_id::Int, climate::Climate; - extraction=nothing, exchange=nothing)::Nothing + inflow=nothing, extraction=nothing, exchange=nothing)::Nothing Generic run method that runs a model attached to a given node for all time steps. Recurses upstream as needed. @@ -183,15 +130,86 @@ Recurses upstream as needed. - `extraction::DataFrame` : water orders for each time step (defaults to nothing) - `exchange::DataFrame` : exchange with groundwater system at each time step (defaults to nothing) """ -function run_node!(sn::StreamfallNetwork, node_id::Int, climate::Climate; - inflow=nothing, extraction=nothing, exchange=nothing) +function run_node!( + sn::StreamfallNetwork, node_id::Int, climate::Climate; + inflow=nothing, extraction=nothing, exchange=nothing +)::Nothing timesteps = sim_length(climate) - @inbounds for ts in 1:timesteps - run_node!(sn, node_id, climate, ts; - inflow=inflow, extraction=extraction, exchange=exchange) + + # Run all upstream nodes + sim_inflow = zeros(timesteps) + ins = inneighbors(sn.mg, node_id) + for i in ins + # Get inflow from previous node + run_node!( + sn, i, climate; + inflow=inflow, extraction=extraction, exchange=exchange + ) + + # Add outflow from upstream to inflow + sim_inflow .+= sn[i].outflow end + + # Run this node + run_node!( + sn[node_id], climate; + inflow=sim_inflow, extraction=extraction, exchange=exchange + ) + + return nothing end +# """ +# run_node!(sn::StreamfallNetwork, node_id::Int, climate::Climate, timestep::Int; +# extraction::Union{DataFrame, Nothing}=nothing, +# exchange::Union{DataFrame, Nothing}=nothing) + +# Run a model attached to a node for a given time step. +# Recurses upstream as needed. + +# # Arguments +# - `sn::StreamfallNetwork` +# - `node_id::Int` +# - `climate::Climate` +# - `timestep::Int` +# - `inflow::DataFrame` : Additional inflow to consider (in ML/timestep) +# - `extraction::DataFrame` : Volume of water to be extracted (in ML/timestep) +# - `exchange::DataFrame` : Volume of flux (in ML/timestep), where negative values are losses to the groundwater system +# """ +# function run_node!(sn::StreamfallNetwork, node_id::Int, climate::Climate, timestep::Int; +# inflow::Union{DataFrame, Nothing}=nothing, +# extraction::Union{DataFrame, Nothing}=nothing, +# exchange::Union{DataFrame, Nothing}=nothing) +# ts = timestep + +# sim_inflow = 0.0 +# ins = inneighbors(sn.mg, node_id) + +# if !isempty(ins) +# for i in ins +# prep_state!(sn[node_id], length(climate)) + +# # Get inflow from previous node +# res = run_node!(sn, i, climate, timestep; +# inflow=inflow, +# extraction=extraction, +# exchange=exchange) + +# # Add outflow +# sim_inflow += res +# end +# end + +# node = sn[node_id] +# ts_inflow = timestep_value(ts, node.name, "inflow", inflow) +# ts_inflow += sim_inflow + +# run_func! = get_prop(sn, node_id, :nfunc) + +# # Run for a time step, dependent on previous state +# run_func!(node, climate, ts; inflow=ts_inflow, extraction=extraction, exchange=exchange) +# end + """ run_node!(node::NetworkNode, climate::Climate; @@ -206,7 +224,10 @@ Run a specific node, and only that node, for all time steps. - `extraction::Union{DataFrame, Vector, Number}` : Extractions from this subcatchment - `exchange::Union{DataFrame, Vector, Number}` : Groundwater flux """ -function run_node!(node::NetworkNode, climate::Climate; inflow=nothing, extraction=nothing, exchange=nothing)::Nothing +function run_node!( + node::NetworkNode, climate::Climate; + inflow=nothing, extraction=nothing, exchange=nothing +)::Nothing timesteps = sim_length(climate) prep_state!(node, timesteps) @@ -243,14 +264,14 @@ export EnsembleNode, BaseEnsemble export run_step!, run_timestep! # Network -export find_inlets_and_outlets, inlets, outlets, create_network, create_node -export climate_values, get_node, get_node_id, get_prop, set_prop! +export find_inlets_and_outlets, inlets, outlets, load_network, create_network, create_node +export climate_values, node_names, get_node, get_node_id, get_prop, set_prop! export param_info, update_params!, sim_length, reset! export run_catchment!, run_basin!, run_node!, run_node_with_temp! export calibrate! # Data -export extract_flow, extract_climate +export extract_flow, extract_climate, align_time_frame # plotting methods export quickplot, plot_network, save_figure