Refactor Python code

.github/pull_request_template.md

@@ -29,7 +29,7 @@ Updating other julia files may be required.
 - update `docs/core/equations.qmd`
 - update `docs/core/usage.qmd`
 - update `docs/python/examples.ipynb`  # or start a new example model
-- update `docs/schema*.json` by running `julia --project=docs docs/gen_schema.jl` and `datamodel-codegen --use-title-as-name --input docs/schema/root.schema.json --output python/ribasim/ribasim/models.py`
+- update `docs/schema*.json` by running `julia --project=docs docs/gen_schema.jl` and `datamodel-codegen --use-title-as-name --use-double-quotes --input docs/schema/root.schema.json --output python/ribasim/ribasim/models.py` and `datamodel-codegen --use-title-as-name --use-double-quotes --input docs/schema/Config.schema.json --output python/ribasim/ribasim/config.py`
 - update the instructions in `docs/contribute/*.qmd` if something changes there, e.g. something changes in how a new node type must be defined.
 
 ## QGIS

core/src/config.jl

@@ -65,9 +65,13 @@ for (T, kinds) in pairs(nodekinds)
 end
 const nodetypes = collect(keys(nodekinds))
 
+# A [] in a TOML is parsed as Vector{Union{}}
+Base.convert(::Type{Union{Float64, Vector{Float64}}}, x::Vector{Union{}}) =
+    convert(Vector{Float64}, x)
+
 @option struct Solver <: TableOption
     algorithm::String = "QNDF"
-    saveat::Union{Float64, Vector{Float64}, Vector{Union{}}} = Float64[]
+    saveat::Union{Float64, Vector{Float64}} = Float64[]
     adaptive::Bool = true
     dt::Float64 = 0.0
     abstol::Float64 = 1e-6
@@ -109,15 +113,15 @@ end
     timing::Bool = false
 end
 
-@option @addnodetypes struct Config
+@option @addnodetypes struct Config <: TableOption
     starttime::DateTime
     endtime::DateTime
 
     # [s] Δt for periodic update frequency, including user horizons
     update_timestep::Float64 = 60 * 60 * 24.0
 
     # optional, when Config is created from a TOML file, this is its directory
-    relative_dir::String = pwd()
+    relative_dir::String = "."  # ignored(!)
     input_dir::String = "."
     output_dir::String = "."
 

docs/contribute/addnode.qmd

@@ -247,10 +247,11 @@ If you haven't done so before, you first need to instantiate your docs environme
 Run `julia --project=docs`, followed by running `instantiate` in the Pkg mode (press `]`).
 :::
 
-To generate the Python module `models.py` from the JSON Schemas, run:
+To generate the Python module `models.py` and `config.py` from the JSON Schemas, run:
 
 ```
-datamodel-codegen --use-title-as-name --input docs/schema/root.schema.json --output python/ribasim/ribasim/models.py
+datamodel-codegen --use-title-as-name --use-double-quotes --input docs/schema/root.schema.json --output python/ribasim/ribasim/models.py
+`datamodel-codegen --use-title-as-name --use-double-quotes --input docs/schema/Config.schema.json --output python/ribasim/ribasim/config.py`
 ```
 
 Run [Black](python.qmd#sec-black) to format the generated code.

docs/gen_schema.jl

@@ -11,10 +11,14 @@ using JSON3
 using Legolas
 using InteractiveUtils
 using Dates
+using Configurations
+using Logging
 
 # set empty to have local file references for development
 const prefix = "https://deltares.github.io/Ribasim/schema/"
 
+jsondefault(x) = identity(x)
+jsondefault(x::LogLevel) = "info"
 jsontype(x) = jsontype(typeof(x))
 jsonformat(x) = jsonformat(typeof(x))
 jsontype(::Type{<:AbstractString}) = "string"
@@ -23,28 +27,44 @@ jsontype(::Type{<:AbstractFloat}) = "number"
 jsonformat(::Type{<:Float64}) = "double"
 jsonformat(::Type{<:Float32}) = "float"
 jsontype(::Type{<:Number}) = "number"
-jsontype(::Type{<:AbstractVector}) = "list"
+jsontype(::Type{<:AbstractVector}) = "array"
 jsontype(::Type{<:Bool}) = "boolean"
+jsontype(::Type{LogLevel}) = "string"
+jsontype(::Type{<:Enum}) = "string"
 jsontype(::Type{<:Missing}) = "null"
 jsontype(::Type{<:DateTime}) = "string"
 jsonformat(::Type{<:DateTime}) = "date-time"
 jsontype(::Type{<:Nothing}) = "null"
 jsontype(::Type{<:Any}) = "object"
 jsonformat(::Type{<:Any}) = "default"
-jsontype(T::Union) = unique(filter(!isequal("null"), jsontype.(Base.uniontypes(T))))
+function jsontype(T::Union)
+    t = Base.uniontypes(T)
+    td = Dict(zip(t, jsontype.(t)))
+    filter!(x -> !isequal(x.second, "null"), td)
+    length(td) == 1 && return first(values(td))
+    types = Dict[]
+    for (t, jt) in td
+        nt = Dict{String, Any}("type" => jt)
+        if t <: AbstractVector
+            nt["items"] = Dict("type" => jsontype(eltype(t)))
+        end
+        push!(types, nt)
+    end
+    return Dict("anyOf" => types)
+end
 
 function strip_prefix(T::DataType)
-    (p, v) = rsplit(string(T), 'V'; limit = 2)
+    n = string(T)
+    (p, _) = occursin('V', n) ? rsplit(n, 'V'; limit = 2) : (n, "")
     return string(last(rsplit(p, '.'; limit = 2)))
 end
 
-function gen_root_schema(TT::Vector, prefix = prefix)
-    name = "root"
+function gen_root_schema(TT::Vector, prefix = prefix, name = "root")
     schema = Dict(
         "\$schema" => "https://json-schema.org/draft/2020-12/schema",
         "properties" => Dict{String, Dict}(),
         "\$id" => "$(prefix)$name.schema.json",
-        "title" => "root",
+        "title" => name,
         "description" => "All Ribasim Node types",
         "type" => "object",
     )
@@ -60,7 +80,7 @@ end
 
 os_line_separator() = Sys.iswindows() ? "\r\n" : "\n"
 
-function gen_schema(T::DataType, prefix = prefix)
+function gen_schema(T::DataType, prefix = prefix; pandera = true)
     name = strip_prefix(T)
     schema = Dict(
         "\$schema" => "https://json-schema.org/draft/2020-12/schema",
@@ -71,24 +91,48 @@ function gen_schema(T::DataType, prefix = prefix)
         "properties" => Dict{String, Dict}(),
         "required" => String[],
     )
-    for (fieldname, fieldtype) in zip(fieldnames(T), fieldtypes(T))
-        fieldname = string(fieldname)
-        schema["properties"][fieldname] = Dict(
-            "description" => "$fieldname",
-            "type" => jsontype(fieldtype),
-            "format" => jsonformat(fieldtype),
-        )
-        if !((fieldtype isa Union) && (fieldtype.a === Missing))
+    for (fieldnames, fieldtype) in zip(fieldnames(T), fieldtypes(T))
+        fieldname = string(fieldnames)
+        required = true
+        ref = false
+        if fieldtype <: Ribasim.config.TableOption
+            schema["properties"][fieldname] =
+                Dict("\$ref" => "$(prefix)$(strip_prefix(fieldtype)).schema.json")
+            ref = true
+        else
+            type = jsontype(fieldtype)
+            schema["properties"][fieldname] = Dict{String, Any}(
+                "description" => "$fieldname",
+                "format" => jsonformat(fieldtype),
+            )
+            if type isa AbstractString
+                schema["properties"][fieldname]["type"] = type
+            else
+                merge!(schema["properties"][fieldname], type)
+            end
+        end
+        if T <: Ribasim.config.TableOption
+            d = field_default(T, fieldnames)
+            if !(d isa Configurations.ExproniconLite.NoDefault)
+                if !ref
+                    schema["properties"][fieldname]["default"] = jsondefault(d)
+                end
+                required = false
+            end
+        end
+        if !((fieldtype isa Union) && (fieldtype.a === Missing)) && required
             push!(schema["required"], fieldname)
         end
     end
-    # Temporary hack so pandera will keep the Pydantic record types
-    schema["properties"]["remarks"] = Dict(
-        "description" => "a hack for pandera",
-        "type" => "string",
-        "format" => "default",
-        "default" => "",
-    )
+    if pandera
+        # Temporary hack so pandera will keep the Pydantic record types
+        schema["properties"]["remarks"] = Dict(
+            "description" => "a hack for pandera",
+            "type" => "string",
+            "format" => "default",
+            "default" => "",
+        )
+    end
     open(normpath(@__DIR__, "schema", "$(name).schema.json"), "w") do io
         JSON3.pretty(io, schema)
         println(io)
@@ -106,4 +150,7 @@ end
 for T in subtypes(Legolas.AbstractRecord)
     gen_schema(T)
 end
+for T in subtypes(Ribasim.config.TableOption)
+    gen_schema(T; pandera = false)
+end
 gen_root_schema(subtypes(Legolas.AbstractRecord))

docs/schema/Config.schema.json

@@ -0,0 +1,98 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+        "output": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/Output.schema.json"
+        },
+        "starttime": {
+            "format": "date-time",
+            "description": "starttime",
+            "type": "string"
+        },
+        "update_timestep": {
+            "format": "double",
+            "default": 86400,
+            "description": "update_timestep",
+            "type": "number"
+        },
+        "input_dir": {
+            "format": "default",
+            "default": ".",
+            "description": "input_dir",
+            "type": "string"
+        },
+        "output_dir": {
+            "format": "default",
+            "default": ".",
+            "description": "output_dir",
+            "type": "string"
+        },
+        "level_boundary": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/level_boundary.schema.json"
+        },
+        "pump": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/pump.schema.json"
+        },
+        "discrete_control": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/discrete_control.schema.json"
+        },
+        "solver": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/Solver.schema.json"
+        },
+        "flow_boundary": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/flow_boundary.schema.json"
+        },
+        "pid_control": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/pid_control.schema.json"
+        },
+        "fractional_flow": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/fractional_flow.schema.json"
+        },
+        "relative_dir": {
+            "format": "default",
+            "default": ".",
+            "description": "relative_dir",
+            "type": "string"
+        },
+        "endtime": {
+            "format": "date-time",
+            "description": "endtime",
+            "type": "string"
+        },
+        "manning_resistance": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/manning_resistance.schema.json"
+        },
+        "tabulated_rating_curve": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/tabulated_rating_curve.schema.json"
+        },
+        "logging": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/Logging.schema.json"
+        },
+        "outlet": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/outlet.schema.json"
+        },
+        "geopackage": {
+            "format": "default",
+            "description": "geopackage",
+            "type": "string"
+        },
+        "terminal": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/terminal.schema.json"
+        },
+        "basin": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/basin.schema.json"
+        },
+        "linear_resistance": {
+            "$ref": "https://deltares.github.io/Ribasim/schema/linear_resistance.schema.json"
+        }
+    },
+    "required": [
+        "starttime",
+        "endtime",
+        "geopackage"
+    ],
+    "$id": "https://deltares.github.io/Ribasim/schema/Config.schema.json",
+    "title": "Config",
+    "description": "A Config object based on Ribasim.config.Config",
+    "type": "object"
+}

docs/schema/DiscreteControlCondition.schema.json

@@ -30,9 +30,7 @@
         "look_ahead": {
             "format": "default",
             "description": "look_ahead",
-            "type": [
-                "number"
-            ]
+            "type": "number"
         }
     },
     "required": [

docs/schema/FlowBoundaryStatic.schema.json

@@ -10,9 +10,7 @@
         "active": {
             "format": "default",
             "description": "active",
-            "type": [
-                "boolean"
-            ]
+            "type": "boolean"
         },
         "flow_rate": {
             "format": "double",

docs/schema/FractionalFlowStatic.schema.json

@@ -20,9 +20,7 @@
         "control_state": {
             "format": "default",
             "description": "control_state",
-            "type": [
-                "string"
-            ]
+            "type": "string"
         }
     },
     "required": [

docs/schema/LevelBoundaryStatic.schema.json

@@ -10,9 +10,7 @@
         "active": {
             "format": "default",
             "description": "active",
-            "type": [
-                "boolean"
-            ]
+            "type": "boolean"
         },
         "node_id": {
             "format": "default",

docs/schema/LinearResistanceStatic.schema.json

@@ -10,9 +10,7 @@
         "active": {
             "format": "default",
             "description": "active",
-            "type": [
-                "boolean"
-            ]
+            "type": "boolean"
         },
         "node_id": {
             "format": "default",
@@ -27,9 +25,7 @@
         "control_state": {
             "format": "default",
             "description": "control_state",
-            "type": [
-                "string"
-            ]
+            "type": "string"
         }
     },
     "required": [

docs/schema/Logging.schema.json

@@ -0,0 +1,23 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+        "timing": {
+            "format": "default",
+            "default": false,
+            "description": "timing",
+            "type": "boolean"
+        },
+        "verbosity": {
+            "format": "default",
+            "default": "info",
+            "description": "verbosity",
+            "type": "string"
+        }
+    },
+    "required": [
+    ],
+    "$id": "https://deltares.github.io/Ribasim/schema/Logging.schema.json",
+    "title": "Logging",
+    "description": "A Logging object based on Ribasim.config.Logging",
+    "type": "object"
+}