From 11b0c41a6665511e376147afd591fe5ee3c29066 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 2 Jan 2024 11:10:06 -0700 Subject: [PATCH 1/2] Support serialization of supplemental attributes --- src/InfrastructureSystems.jl | 6 +++ src/component.jl | 10 ++--- src/component_uuids.jl | 33 +++++++++++++++ src/geographic_supplemental_attribute.jl | 9 ++-- src/serialization.jl | 11 +++-- src/supplemental_attributes.jl | 37 +++++++++++++++- src/supplemental_attributes_container.jl | 54 ++++++++++++++++++++++++ src/system_data.jl | 27 +++++++++++- src/utils/test.jl | 45 +++++++++++++------- src/utils/utils.jl | 1 + test/common.jl | 12 +++++- test/test_serialization.jl | 35 ++++++++++++--- test/test_supplemental_attributes.jl | 2 +- test/test_time_series.jl | 7 +++ 14 files changed, 247 insertions(+), 42 deletions(-) create mode 100644 src/component_uuids.jl create mode 100644 src/supplemental_attributes_container.jl diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index 495b83784..689607cd1 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -65,11 +65,15 @@ Optional interface functions: - get_time_series_container() - get_component_uuids() + Required if the struct does not include the field component_uuids. - get_uuid() Subtypes may contain time series. Which requires - get_time_series_container() + +All subtypes must include an instance of ComponentUUIDs in order to track +components attached to each attribute. """ abstract type InfrastructureSystemsSupplementalAttribute <: InfrastructureSystemsType end @@ -113,8 +117,10 @@ include("static_time_series.jl") include("time_series_container.jl") include("time_series_parser.jl") include("containers.jl") +include("component_uuids.jl") include("supplemental_attribute.jl") include("supplemental_attributes.jl") +include("supplemental_attributes_container.jl") include("components.jl") include("geographic_supplemental_attribute.jl") include("generated/includes.jl") diff --git a/src/component.jl b/src/component.jl index 619586f7e..fdfb0f87a 100644 --- a/src/component.jl +++ b/src/component.jl @@ -305,9 +305,9 @@ function attach_supplemental_attribute!( attribute_container = get_supplemental_attributes_container(component) if !haskey(attribute_container, T) - attribute_container[T] = Set{T}() + attribute_container[T] = Dict{Base.UUID, T}() end - push!(attribute_container[T], attribute) + attribute_container[T][get_uuid(attribute)] = attribute @debug "SupplementalAttribute type $T with UUID $(get_uuid(attribute)) stored in component $(summary(component))" _group = LOG_GROUP_SYSTEM return @@ -323,8 +323,8 @@ end function clear_supplemental_attributes!(component::InfrastructureSystemsComponent) container = get_supplemental_attributes_container(component) - for attribute_set in values(container) - for attribute in attribute_set + for attributes in values(container) + for attribute in collect(values(attributes)) detach_component!(attribute, component) detach_supplemental_attribute!(component, attribute) end @@ -346,7 +346,7 @@ function detach_supplemental_attribute!( ), ) end - delete!(container[T], attribute) + delete!(container[T], get_uuid(attribute)) if isempty(container[T]) pop!(container, T) end diff --git a/src/component_uuids.jl b/src/component_uuids.jl new file mode 100644 index 000000000..fbcfdb54c --- /dev/null +++ b/src/component_uuids.jl @@ -0,0 +1,33 @@ +# This is an abstraction of a Set in order to enable de-serialization of supplemental +# attributes. + +struct ComponentUUIDs <: InfrastructureSystemsType + uuids::Set{Base.UUID} + + function ComponentUUIDs(uuids = Set{Base.UUID}()) + new(uuids) + end +end + +Base.copy(x::ComponentUUIDs) = copy(x.uuids) +Base.delete!(x::ComponentUUIDs, uuid) = delete!(x.uuids, uuid) +Base.empty!(x::ComponentUUIDs) = empty!(x.uuids) +Base.filter!(f, x::ComponentUUIDs) = filter!(f, x.uuids) +Base.in(x, y::ComponentUUIDs) = in(x, y.uuids) +Base.isempty(x::ComponentUUIDs) = isempty(x.uuids) +Base.iterate(x::ComponentUUIDs, args...) = iterate(x.uuids, args...) +Base.length(x::ComponentUUIDs) = length(x.uuids) +Base.pop!(x::ComponentUUIDs) = pop!(x.uuids) +Base.pop!(x::ComponentUUIDs, y) = pop!(x.uuids, y) +Base.pop!(x::ComponentUUIDs, y, default) = pop!(x.uuids, y, default) +Base.push!(x::ComponentUUIDs, y) = push!(x.uuids, y) +Base.setdiff!(x::ComponentUUIDs, y::ComponentUUIDs) = setdiff!(x.uuids, y.uuids) +Base.sizehint!(x::ComponentUUIDs, newsz) = sizehint!(x.uuids, newsz) + +function deserialize(::Type{ComponentUUIDs}, data::Dict) + uuids = Set{Base.UUID}() + for uuid in data["uuids"] + push!(uuids, deserialize(Base.UUID, uuid)) + end + return ComponentUUIDs(uuids) +end diff --git a/src/geographic_supplemental_attribute.jl b/src/geographic_supplemental_attribute.jl index 92bd45f55..822e00268 100644 --- a/src/geographic_supplemental_attribute.jl +++ b/src/geographic_supplemental_attribute.jl @@ -3,15 +3,16 @@ Attribute to store Geographic Information about the system components """ struct GeographicInfo <: InfrastructureSystemsSupplementalAttribute geo_json::Dict{String, Any} - component_uuids::Set{UUIDs.UUID} + component_uuids::ComponentUUIDs internal::InfrastructureSystemsInternal end function GeographicInfo(; - geo_json::Dict{String, Any} = Dict{String, Any}(), - component_uuids::Set{UUIDs.UUID} = Set{UUIDs.UUID}(), + geo_json::Dict{String, <:Any} = Dict{String, Any}(), + component_uuids::ComponentUUIDs = ComponentUUIDs(), + internal = InfrastructureSystemsInternal(), ) - return GeographicInfo(geo_json, component_uuids, InfrastructureSystemsInternal()) + return GeographicInfo(geo_json, component_uuids, internal) end get_geo_json(geo::GeographicInfo) = geo.geo_json diff --git a/src/serialization.jl b/src/serialization.jl index e2fb05792..d58ba6a28 100644 --- a/src/serialization.jl +++ b/src/serialization.jl @@ -16,7 +16,7 @@ function to_json( pretty = false, ) where {T <: InfrastructureSystemsType} if !force && isfile(filename) - error("$file already exists. Set force=true to overwrite.") + error("$filename already exists. Set force=true to overwrite.") end result = open(filename, "w") do io return to_json(io, obj; pretty = pretty) @@ -161,8 +161,7 @@ end function deserialize_struct(::Type{TimeSeriesKey}, data::Dict) vals = Dict{Symbol, Any}() - for (field_name, field_type) in - zip(fieldnames(TimeSeriesKey), fieldtypes(TimeSeriesKey)) + for field_name in fieldnames(TimeSeriesKey) val = data[string(field_name)] if field_name == :time_series_type val = getfield(InfrastructureSystems, Symbol(strip_module_name(val))) @@ -176,7 +175,10 @@ function deserialize_to_dict(::Type{T}, data::Dict) where {T} # Note: mostly duplicated in src/deterministic_metadata.jl vals = Dict{Symbol, Any}() for (field_name, field_type) in zip(fieldnames(T), fieldtypes(T)) - val = data[string(field_name)] + name_str = string(field_name) + # Some types may not serialize optional fields. + !haskey(data, name_str) && continue + val = data[name_str] if val isa Dict && haskey(val, METADATA_KEY) metadata = get_serialization_metadata(val) if haskey(metadata, FUNCTION_KEY) @@ -255,6 +257,7 @@ deserialize(::Type{Dates.DateTime}, val::AbstractString) = Dates.DateTime(val) serialize(uuid::Base.UUID) = Dict("value" => string(uuid)) serialize(uuids::Vector{Base.UUID}) = serialize.(uuids) +serialize(uuids::Set{Base.UUID}) = serialize.(uuids) deserialize(::Type{Base.UUID}, data::Dict) = Base.UUID(data["value"]) serialize(value::Complex) = Dict("real" => real(value), "imag" => imag(value)) diff --git a/src/supplemental_attributes.jl b/src/supplemental_attributes.jl index fe6457d23..6b3e8f051 100644 --- a/src/supplemental_attributes.jl +++ b/src/supplemental_attributes.jl @@ -1,5 +1,3 @@ -const SupplementalAttributesContainer = - Dict{DataType, Set{<:InfrastructureSystemsSupplementalAttribute}} const SupplementalAttributesByType = Dict{DataType, Dict{UUIDs.UUID, <:InfrastructureSystemsSupplementalAttribute}} @@ -227,3 +225,38 @@ function get_supplemental_attributes( @assert_op eltype(iter) == T return iter end + +function serialize(attributes::SupplementalAttributes) + data = Vector{Dict{String, Any}}() + for attribute_container in values(attributes.data) + for attribute in values(attribute_container) + push!(data, serialize(attribute)) + end + end + + return data +end + +function deserialize( + ::Type{SupplementalAttributes}, + data::Vector, + time_series_storage::TimeSeriesStorage, +) + attributes = SupplementalAttributesByType() + for attr_dict in data + type = get_type_from_serialization_metadata(get_serialization_metadata(attr_dict)) + if !haskey(attributes, type) + attributes[type] = + Dict{UUIDs.UUID, InfrastructureSystemsSupplementalAttribute}() + end + attr = deserialize(type, attr_dict) + uuid = get_uuid(attr) + if haskey(attributes[type], uuid) + error("Bug: duplicate UUID in attributes container: type=$type uuid=$uuid") + end + attributes[type][uuid] = attr + @debug "Deserialized $(summary(attr))" _group = LOG_GROUP_SERIALIZATION + end + + return SupplementalAttributes(attributes, time_series_storage) +end diff --git a/src/supplemental_attributes_container.jl b/src/supplemental_attributes_container.jl new file mode 100644 index 000000000..55463bff5 --- /dev/null +++ b/src/supplemental_attributes_container.jl @@ -0,0 +1,54 @@ +""" +All components must include a field of this type in order to store supplemental attributes. +""" +struct SupplementalAttributesContainer + data::Dict{DataType, Dict{Base.UUID, <:InfrastructureSystemsSupplementalAttribute}} +end + +function SupplementalAttributesContainer(; + data = Dict{ + DataType, + Dict{Base.UUID, <:InfrastructureSystemsSupplementalAttribute}, + }(), +) + return SupplementalAttributesContainer(data) +end + +Base.getindex(x::SupplementalAttributesContainer, key) = getindex(x.data, key) +Base.haskey(x::SupplementalAttributesContainer, key) = haskey(x.data, key) +Base.isempty(x::SupplementalAttributesContainer) = isempty(x.data) +Base.iterate(x::SupplementalAttributesContainer, args...) = iterate(x.data, args...) +Base.length(x::SupplementalAttributesContainer) = length(x.data) +Base.values(x::SupplementalAttributesContainer) = values(x.data) +Base.delete!(x::SupplementalAttributesContainer, key) = delete!(x.data, key) +Base.empty!(x::SupplementalAttributesContainer) = empty!(x.data) +Base.setindex!(x::SupplementalAttributesContainer, val, key) = setindex!(x.data, val, key) +Base.pop!(x::SupplementalAttributesContainer, key) = pop!(x.data, key) + +function serialize(container::SupplementalAttributesContainer) + return [serialize(uuid) for attrs in values(container) for uuid in keys(attrs)] +end + +function deserialize( + ::Type{SupplementalAttributesContainer}, + uuids::Vector, + system_attributes::Dict{Base.UUID, <:InfrastructureSystemsSupplementalAttribute}, +) + container = SupplementalAttributesContainer() + for uuid_dict in uuids + uuid = deserialize(Base.UUID, uuid_dict) + attribute = system_attributes[uuid] + type = typeof(attribute) + if !haskey(container, type) + container[type] = Dict{Base.UUID, InfrastructureSystemsSupplementalAttribute}() + end + if haskey(container[type], uuid) + error( + "Bug: component supplemental attribute container already has uuid = $uuid", + ) + end + container[type][uuid] = attribute + end + + return container +end diff --git a/src/system_data.jl b/src/system_data.jl index e69bfca3b..ba7ec7d94 100644 --- a/src/system_data.jl +++ b/src/system_data.jl @@ -81,6 +81,7 @@ function SystemData( time_series_params, validation_descriptors, time_series_storage, + attributes, internal, ) components = Components(time_series_storage, validation_descriptors) @@ -88,7 +89,7 @@ function SystemData( return SystemData( components, masked_components, - SupplementalAttributes(time_series_storage), + attributes, time_series_params, time_series_storage, validation_descriptors, @@ -649,7 +650,8 @@ end function serialize(data::SystemData) @debug "serialize SystemData" _group = LOG_GROUP_SERIALIZATION json_data = Dict() - for field in (:components, :masked_components, :time_series_params, :internal) + for field in + (:components, :masked_components, :attributes, :time_series_params, :internal) json_data[string(field)] = serialize(getfield(data, field)) end @@ -726,14 +728,35 @@ function deserialize( ) end + attributes = deserialize(SupplementalAttributes, raw["attributes"], time_series_storage) internal = deserialize(InfrastructureSystemsInternal, raw["internal"]) @debug "deserialize" _group = LOG_GROUP_SERIALIZATION validation_descriptors time_series_storage internal sys = SystemData( time_series_params, validation_descriptors, time_series_storage, + attributes, internal, ) + attributes_by_uuid = Dict{Base.UUID, InfrastructureSystemsSupplementalAttribute}() + for attr_dict in values(attributes.data) + for attr in values(attr_dict) + uuid = get_uuid(attr) + if haskey(attributes_by_uuid, uuid) + error("Bug: Found duplicate supplemental attribute UUID: $uuid") + end + attributes_by_uuid[uuid] = attr + end + end + for component in raw["components"] + if haskey(component, "attributes_container") + component["attributes_container"] = deserialize( + SupplementalAttributesContainer, + component["attributes_container"], + attributes_by_uuid, + ) + end + end # Note: components need to be deserialized by the parent so that they can go through # the proper checks. return sys diff --git a/src/utils/test.jl b/src/utils/test.jl index d5570e336..5333a8ac8 100644 --- a/src/utils/test.jl +++ b/src/utils/test.jl @@ -1,4 +1,3 @@ - mutable struct TestComponent <: InfrastructureSystemsComponent name::String val::Int @@ -7,14 +6,6 @@ mutable struct TestComponent <: InfrastructureSystemsComponent internal::InfrastructureSystemsInternal end -mutable struct AdditionalTestComponent <: InfrastructureSystemsComponent - name::String - val::Int - time_series_container::TimeSeriesContainer - attributes_container::SupplementalAttributesContainer - internal::InfrastructureSystemsInternal -end - function TestComponent(name, val) return TestComponent( name, @@ -25,6 +16,14 @@ function TestComponent(name, val) ) end +mutable struct AdditionalTestComponent <: InfrastructureSystemsComponent + name::String + val::Int + time_series_container::TimeSeriesContainer + attributes_container::SupplementalAttributesContainer + internal::InfrastructureSystemsInternal +end + function AdditionalTestComponent(name, val) return AdditionalTestComponent( name, @@ -35,6 +34,20 @@ function AdditionalTestComponent(name, val) ) end +mutable struct SimpleTestComponent <: InfrastructureSystemsComponent + name::String + val::Int + internal::InfrastructureSystemsInternal +end + +function SimpleTestComponent(name, val) + return SimpleTestComponent(name, val, InfrastructureSystemsInternal()) +end + +function SimpleTestComponent(; name, val, internal = InfrastructureSystemsInternal()) + return SimpleTestComponent(name, val, internal) +end + get_internal(component::TestComponent) = component.internal get_internal(component::AdditionalTestComponent) = component.internal get_val(component::TestComponent) = component.val @@ -57,7 +70,7 @@ function deserialize(::Type{TestComponent}, data::Dict) data["name"], data["val"], deserialize(TimeSeriesContainer, data["time_series_container"]), - SupplementalAttributesContainer(), + data["attributes_container"], deserialize(InfrastructureSystemsInternal, data["internal"]), ) end @@ -100,20 +113,22 @@ end struct TestSupplemental <: InfrastructureSystemsSupplementalAttribute value::Float64 - component_uuids::Set{UUIDs.UUID} + component_uuids::ComponentUUIDs internal::InfrastructureSystemsInternal time_series_container::TimeSeriesContainer end function TestSupplemental(; - value::Float64 = 0.0, - component_uuids::Set{UUIDs.UUID} = Set{UUIDs.UUID}(), + value::Float64, + component_uuids::ComponentUUIDs = ComponentUUIDs(), + time_series_container = TimeSeriesContainer(), + internal::InfrastructureSystemsInternal = InfrastructureSystemsInternal(), ) return TestSupplemental( value, component_uuids, - InfrastructureSystemsInternal(), - TimeSeriesContainer(), + internal, + time_series_container, ) end diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 9f59619f0..9a836d039 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -194,6 +194,7 @@ function compare_values(x::Dict, y::Dict; compare_uuids = false) return match end +compare_values(x::Float64, y::Int; compare_uuids = false) = x == Float64(y) compare_values(::Type{T}, ::Type{T}; compare_uuids = false) where {T} = true compare_values(::Type{T}, ::Type{U}; compare_uuids = false) where {T, U} = false diff --git a/test/common.jl b/test/common.jl index cebe7e888..c5945edbd 100644 --- a/test/common.jl +++ b/test/common.jl @@ -1,5 +1,8 @@ - -function create_system_data(; with_time_series = false, time_series_in_memory = false) +function create_system_data(; + with_time_series = false, + time_series_in_memory = false, + with_supplemental_attributes = false, +) data = IS.SystemData(; time_series_in_memory = time_series_in_memory) name = "Component1" @@ -17,6 +20,11 @@ function create_system_data(; with_time_series = false, time_series_in_memory = IS.@assert_op length(time_series) > 0 end + if with_supplemental_attributes + geo_info = IS.GeographicInfo() + IS.add_supplemental_attribute!(data, component, geo_info) + end + return data end diff --git a/test/test_serialization.jl b/test/test_serialization.jl index d91a65e87..b9dd4231e 100644 --- a/test/test_serialization.jl +++ b/test/test_serialization.jl @@ -1,4 +1,3 @@ - function validate_serialization(sys::IS.SystemData; time_series_read_only = false) #path, io = mktemp() # For some reason files aren't getting deleted when written to /tmp. Using current dir. @@ -92,31 +91,53 @@ end @test result end -@testset "Test verion info" begin +@testset "Test JSON serialization with supplemental attributes" begin + sys = IS.SystemData() + initial_time = Dates.DateTime("2020-09-01") + resolution = Dates.Hour(1) + ta = TimeSeries.TimeArray(range(initial_time; length = 24, step = resolution), ones(24)) + ts = IS.SingleTimeSeries(; data = ta, name = "test") + geo = IS.GeographicInfo(; geo_json = Dict("x" => 1.0, "y" => 2.0)) + + for i in 1:2 + name = "component_$(i)" + component = IS.TestComponent(name, 5) + IS.add_component!(sys, component) + attr = IS.TestSupplemental(; value = Float64(i)) + IS.add_supplemental_attribute!(sys, component, attr) + IS.add_time_series!(sys, attr, ts) + IS.add_supplemental_attribute!(sys, component, geo) + end + + _, result = validate_serialization(sys) + @test result +end + +@testset "Test version info" begin data = IS.serialize_julia_info() @test haskey(data, "julia_version") @test haskey(data, "package_info") end @testset "Test JSON string" begin - component = IS.TestComponent("Component1", 1) + component = IS.SimpleTestComponent("Component1", 1) text = IS.to_json(component) - IS.deserialize(IS.TestComponent, JSON3.read(text, Dict)) == component + IS.deserialize(IS.SimpleTestComponent, JSON3.read(text, Dict)) == component end @testset "Test pretty-print JSON IO" begin - component = IS.TestComponent("Component1", 2) + component = IS.SimpleTestComponent("Component1", 2) io = IOBuffer() IS.to_json(io, component; pretty = false) text = String(take!(io)) @test !occursin(" ", text) - IS.deserialize(IS.TestComponent, JSON3.read(text, Dict)) == component + IS.deserialize(IS.SimpleTestComponent, JSON3.read(text, Dict)) == component io = IOBuffer() IS.to_json(io, component; pretty = true) text = String(take!(io)) @test occursin(" ", text) - IS.deserialize(IS.TestComponent, JSON3.read(text, Dict)) == component + IS.deserialize(IS.SimpleTestComponent, JSON3.read(text, Dict)) == component end @testset "Test ext serialization" begin diff --git a/test/test_supplemental_attributes.jl b/test/test_supplemental_attributes.jl index 6b8ee4608..70ac10460 100644 --- a/test/test_supplemental_attributes.jl +++ b/test/test_supplemental_attributes.jl @@ -90,7 +90,7 @@ end name = "component_$(i)" component = IS.TestComponent(name, 5) IS.add_component!(data, component) - supp_attribute = IS.TestSupplemental() + supp_attribute = IS.TestSupplemental(; value = Float64(i)) IS.add_supplemental_attribute!(data, component, supp_attribute) IS.add_time_series!(data, supp_attribute, ts) end diff --git a/test/test_time_series.jl b/test/test_time_series.jl index 2290901a6..e89ad9ee3 100644 --- a/test/test_time_series.jl +++ b/test/test_time_series.jl @@ -2246,3 +2246,10 @@ end pop!(ENV, IS.TIME_SERIES_DIRECTORY_ENV_VAR) end end + +@testset "Test time series counts" begin + sys = create_system_data_shared_time_series(; time_series_in_memory = true) + counts = IS.get_time_series_counts(sys) + @test counts.static_time_series_count == 1 + @test counts.components_with_time_series == 2 +end From 8147d65ce7dbd2d9b7ec7a517e4237528661c75b Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Tue, 2 Jan 2024 17:38:11 -0700 Subject: [PATCH 2/2] Simply logic --- src/supplemental_attributes.jl | 9 +-------- src/supplemental_attributes_container.jl | 9 ++------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/supplemental_attributes.jl b/src/supplemental_attributes.jl index 6b3e8f051..9cf032e7a 100644 --- a/src/supplemental_attributes.jl +++ b/src/supplemental_attributes.jl @@ -227,14 +227,7 @@ function get_supplemental_attributes( end function serialize(attributes::SupplementalAttributes) - data = Vector{Dict{String, Any}}() - for attribute_container in values(attributes.data) - for attribute in values(attribute_container) - push!(data, serialize(attribute)) - end - end - - return data + return [serialize(y) for x in values(attributes.data) for y in values(x)] end function deserialize( diff --git a/src/supplemental_attributes_container.jl b/src/supplemental_attributes_container.jl index 55463bff5..48096a8c0 100644 --- a/src/supplemental_attributes_container.jl +++ b/src/supplemental_attributes_container.jl @@ -2,15 +2,10 @@ All components must include a field of this type in order to store supplemental attributes. """ struct SupplementalAttributesContainer - data::Dict{DataType, Dict{Base.UUID, <:InfrastructureSystemsSupplementalAttribute}} + data::SupplementalAttributesByType end -function SupplementalAttributesContainer(; - data = Dict{ - DataType, - Dict{Base.UUID, <:InfrastructureSystemsSupplementalAttribute}, - }(), -) +function SupplementalAttributesContainer(; data = SupplementalAttributesByType()) return SupplementalAttributesContainer(data) end