From 669d26afd3402d4b0f2dd098f24cbc995eda5433 Mon Sep 17 00:00:00 2001 From: Sander van Rijn Date: Wed, 1 Nov 2023 16:35:38 +0100 Subject: [PATCH 1/6] [WIP] Add initial ESDL/XML reader: gather assets from file --- Project.toml | 1 + src/io.jl | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 7e2b825f..2e70000e 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.4.0" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +EzXML = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" diff --git a/src/io.jl b/src/io.jl index a5ed963b..4c18f5d2 100644 --- a/src/io.jl +++ b/src/io.jl @@ -1,5 +1,7 @@ export create_parameters_and_sets_from_file, - create_graph, save_solution_to_file, compute_rp_partitions + create_graph, save_solution_to_file, compute_rp_partitions, read_esdl + +using EzXML """ parameters, sets = create_parameters_and_sets_from_file(input_folder) @@ -372,3 +374,58 @@ function compute_rp_partitions(df, elements, time_steps_per_rp) return rp_partitions end + +""" + read_esdl(file_path; instance_name) + +Read an ESDL file to construct the flow graph and link the related +assets and flows from the specification + +An ESDL file can contain multiple instances. If multiple instances +are present in the file, a specific instance has to be selected. +""" +function read_esdl(file_path; instance_name = nothing) + doc = readxml(file_path) + doc_root = root(doc) + if countelements(doc_root) == 0 + return nothing # no instances + elseif countelements(doc_root) > 1 + instance = nothing + for node in eachelement(doc_root) + if node["name"] == instance_name + instance = node + break + end + end + if isnothing(instance) + throw(KeyError(instance_name, "instance was not found in given ESDL file")) + end + else + instance = firstelement(doc_root) + end + + area = firstelement(instance) # an instance always has 1 top-level area + assets = gather_assets(area) # gather assets + println(assets) + println(length(assets)) +end + +""" + gather_assets(area) + +Recursively gather all assets from an area defined in ESDL +""" +function gather_assets(area) + if area.name == "asset" + return [area] + elseif area.name == "area" + assets = [] + for node in eachelement(area) + asset = gather_assets(node) + if !isnothing(asset) + assets = vcat(assets, asset) + end + end + return assets + end +end From 8465b58fc14951aea350ca36f1a7ed7a35ccb27e Mon Sep 17 00:00:00 2001 From: Sander van Rijn Date: Thu, 2 Nov 2023 10:10:18 +0100 Subject: [PATCH 2/6] replace manual searching with xpath search expression --- src/io.jl | 51 +++++++++++++-------------------------------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/src/io.jl b/src/io.jl index 4c18f5d2..22df4433 100644 --- a/src/io.jl +++ b/src/io.jl @@ -387,45 +387,20 @@ are present in the file, a specific instance has to be selected. function read_esdl(file_path; instance_name = nothing) doc = readxml(file_path) doc_root = root(doc) - if countelements(doc_root) == 0 - return nothing # no instances - elseif countelements(doc_root) > 1 - instance = nothing - for node in eachelement(doc_root) - if node["name"] == instance_name - instance = node - break - end - end - if isnothing(instance) - throw(KeyError(instance_name, "instance was not found in given ESDL file")) + if countelements(doc_root) == 0 || firstelement(doc_root).name != "instance" + throw(ErrorException("no instance was found in given ESDL file")) + elseif isnothing(instance_name) + if countelements(doc_root) > 1 + throw( + ErrorException( + "multiple instances found in file, but no instance_name was given", + ), + ) + else + instance_name = firstelement(docroot)["name"] end - else - instance = firstelement(doc_root) end - area = firstelement(instance) # an instance always has 1 top-level area - assets = gather_assets(area) # gather assets - println(assets) - println(length(assets)) -end - -""" - gather_assets(area) - -Recursively gather all assets from an area defined in ESDL -""" -function gather_assets(area) - if area.name == "asset" - return [area] - elseif area.name == "area" - assets = [] - for node in eachelement(area) - asset = gather_assets(node) - if !isnothing(asset) - assets = vcat(assets, asset) - end - end - return assets - end + # Use XPath expression to find all assets under the instance with name `instance_name` + return findall(doc, "//instance[@name='$instance_name']//asset") end From 1ebf8cf52e010f18fac71a42c22b30c8263f2d96 Mon Sep 17 00:00:00 2001 From: Sander van Rijn Date: Thu, 2 Nov 2023 10:11:49 +0100 Subject: [PATCH 3/6] add ESDL parsing instance detection error handling tests --- test/esdl/no_instance.esdl | 7 +++ test/esdl/two_instance.esdl | 85 +++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test-io.jl | 14 ++++++ 4 files changed, 107 insertions(+) create mode 100644 test/esdl/no_instance.esdl create mode 100644 test/esdl/two_instance.esdl diff --git a/test/esdl/no_instance.esdl b/test/esdl/no_instance.esdl new file mode 100644 index 00000000..1c65ca19 --- /dev/null +++ b/test/esdl/no_instance.esdl @@ -0,0 +1,7 @@ + + + + + + + diff --git a/test/esdl/two_instance.esdl b/test/esdl/two_instance.esdl new file mode 100644 index 00000000..7ec8df76 --- /dev/null +++ b/test/esdl/two_instance.esdl @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/runtests.jl b/test/runtests.jl index aafb892c..87cdb1e1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,7 @@ using Test # Folders names const INPUT_FOLDER = joinpath(@__DIR__, "inputs") const OUTPUT_FOLDER = joinpath(@__DIR__, "outputs") +const ESDL_FOLDER = joinpath(@__DIR__, "esdl") # Add run all test files in test folder include("test-io.jl") diff --git a/test/test-io.jl b/test/test-io.jl index 12a73d3b..79c609a0 100644 --- a/test/test-io.jl +++ b/test/test-io.jl @@ -57,3 +57,17 @@ end @test_throws AssertionError TEM._parse_rp_partition(Val(:math), "3x4", 1:14) end end + +@testset "ESDL parsing" begin + @testset "instance detection error handling" begin + @testset "no instance" begin + @test_throws ErrorException read_esdl(joinpath(ESDL_FOLDER, "no_instance.esdl")) + end + + @testset "two instances" begin + @test_throws ErrorException read_esdl( + joinpath(ESDL_FOLDER, "two_instance.esdl"), + ) + end + end +end From 70c9d5d108406d813eefbc83675de82ac980440c Mon Sep 17 00:00:00 2001 From: Sander van Rijn Date: Thu, 2 Nov 2023 13:27:05 +0100 Subject: [PATCH 4/6] update esdl test files with comments --- test/esdl/no_instance.esdl | 1 + .../{two_instance.esdl => two_instances.esdl} | 63 ++++++++++--------- 2 files changed, 36 insertions(+), 28 deletions(-) rename test/esdl/{two_instance.esdl => two_instances.esdl} (61%) diff --git a/test/esdl/no_instance.esdl b/test/esdl/no_instance.esdl index 1c65ca19..fa32cec3 100644 --- a/test/esdl/no_instance.esdl +++ b/test/esdl/no_instance.esdl @@ -1,4 +1,5 @@ + diff --git a/test/esdl/two_instance.esdl b/test/esdl/two_instances.esdl similarity index 61% rename from test/esdl/two_instance.esdl rename to test/esdl/two_instances.esdl index 7ec8df76..765d0ae5 100644 --- a/test/esdl/two_instance.esdl +++ b/test/esdl/two_instances.esdl @@ -1,4 +1,7 @@ + + + @@ -45,39 +48,43 @@ - - - - - - - - - - + + + + + + + + + + + + + + - - - - + + + - - - - + + + - - - - + + + + - - - + + + + - - - + + + + From 652a567fec1d71e09d66b54a651cc9dc55468db2 Mon Sep 17 00:00:00 2001 From: Sander van Rijn Date: Thu, 2 Nov 2023 13:29:50 +0100 Subject: [PATCH 5/6] Add tests for asset detection --- src/io.jl | 4 +-- test/esdl/one_instance.esdl | 49 +++++++++++++++++++++++++++++++++++++ test/test-io.jl | 22 ++++++++++++++++- 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 test/esdl/one_instance.esdl diff --git a/src/io.jl b/src/io.jl index 22df4433..9b69f25e 100644 --- a/src/io.jl +++ b/src/io.jl @@ -397,10 +397,10 @@ function read_esdl(file_path; instance_name = nothing) ), ) else - instance_name = firstelement(docroot)["name"] + instance_name = firstelement(doc_root)["name"] end end # Use XPath expression to find all assets under the instance with name `instance_name` - return findall(doc, "//instance[@name='$instance_name']//asset") + return findall("//instance[@name='$instance_name']//asset", doc) end diff --git a/test/esdl/one_instance.esdl b/test/esdl/one_instance.esdl new file mode 100644 index 00000000..11175efe --- /dev/null +++ b/test/esdl/one_instance.esdl @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/test-io.jl b/test/test-io.jl index 79c609a0..38164b10 100644 --- a/test/test-io.jl +++ b/test/test-io.jl @@ -66,8 +66,28 @@ end @testset "two instances" begin @test_throws ErrorException read_esdl( - joinpath(ESDL_FOLDER, "two_instance.esdl"), + joinpath(ESDL_FOLDER, "two_instances.esdl"), ) end end + + @testset "instance asset parsing" begin + @testset "unnamed single instance" begin + assets = read_esdl(joinpath(ESDL_FOLDER, "one_instance.esdl")) + @test length(assets) == 8 + end + + @testset "named instance out of multiple" begin + flat_assets = read_esdl( + joinpath(ESDL_FOLDER, "two_instances.esdl"); + instance_name = "Flat", + ) + @test length(flat_assets) == 4 + nested_assets = read_esdl( + joinpath(ESDL_FOLDER, "two_instances.esdl"); + instance_name = "Nested", + ) + @test length(nested_assets) == 10 + end + end end From a2f921587301cbc3efaba5a65e4668adb69804fa Mon Sep 17 00:00:00 2001 From: Sander van Rijn Date: Wed, 15 Nov 2023 10:01:34 +0100 Subject: [PATCH 6/6] read_esdl now outputs graph; assets read separate NOTE: asset-to-graph conversion is currently very lenient with incomplete connections. Test cases have also not yet been updated to be complete & valid --- src/io.jl | 39 ++++++++++++++++++++++++++++++++++++++- test/test-io.jl | 12 +++++++----- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/io.jl b/src/io.jl index 9b69f25e..00e3a3f9 100644 --- a/src/io.jl +++ b/src/io.jl @@ -1,5 +1,5 @@ export create_parameters_and_sets_from_file, - create_graph, save_solution_to_file, compute_rp_partitions, read_esdl + create_graph, save_solution_to_file, compute_rp_partitions, read_esdl, read_esdl_assets using EzXML @@ -385,6 +385,43 @@ An ESDL file can contain multiple instances. If multiple instances are present in the file, a specific instance has to be selected. """ function read_esdl(file_path; instance_name = nothing) + esdl_assets = read_esdl_assets(file_path; instance_name = instance_name) + + # Gather all in-ports + id_to_index = Dict{String,Int}() + for (to_id, asset) in enumerate(esdl_assets) + for port in eachelement(asset) + if port.name != "port" || port["xsi:type"] != "esdl:InPort" + continue + end + id_to_index[port["id"]] = to_id + end + end + + # Create graph based on out-ports and previously gathered in-ports + graph = Graphs.DiGraph(length(esdl_assets)) + for (from_id, asset) in enumerate(esdl_assets) + for port in eachelement(asset) + if port.name != "port" || port["xsi:type"] != "esdl:OutPort" + continue + end + flows = eachsplit(port["connectedTo"], " ") + for flow in flows + if !haskey(id_to_index, flow) + # throw(ErrorException("No InPort with id '$flow' was found")) + println("No InPort with id '$flow' was found, ignoring...") + continue + end + to_id = id_to_index[flow] + Graphs.add_edge!(graph, from_id, to_id) + end + end + end + + return graph +end + +function read_esdl_assets(file_path; instance_name = nothing) doc = readxml(file_path) doc_root = root(doc) if countelements(doc_root) == 0 || firstelement(doc_root).name != "instance" diff --git a/test/test-io.jl b/test/test-io.jl index 38164b10..293a54d7 100644 --- a/test/test-io.jl +++ b/test/test-io.jl @@ -61,11 +61,13 @@ end @testset "ESDL parsing" begin @testset "instance detection error handling" begin @testset "no instance" begin - @test_throws ErrorException read_esdl(joinpath(ESDL_FOLDER, "no_instance.esdl")) + @test_throws ErrorException read_esdl_assets( + joinpath(ESDL_FOLDER, "no_instance.esdl"), + ) end @testset "two instances" begin - @test_throws ErrorException read_esdl( + @test_throws ErrorException read_esdl_assets( joinpath(ESDL_FOLDER, "two_instances.esdl"), ) end @@ -73,17 +75,17 @@ end @testset "instance asset parsing" begin @testset "unnamed single instance" begin - assets = read_esdl(joinpath(ESDL_FOLDER, "one_instance.esdl")) + assets = read_esdl_assets(joinpath(ESDL_FOLDER, "one_instance.esdl")) @test length(assets) == 8 end @testset "named instance out of multiple" begin - flat_assets = read_esdl( + flat_assets = read_esdl_assets( joinpath(ESDL_FOLDER, "two_instances.esdl"); instance_name = "Flat", ) @test length(flat_assets) == 4 - nested_assets = read_esdl( + nested_assets = read_esdl_assets( joinpath(ESDL_FOLDER, "two_instances.esdl"); instance_name = "Nested", )