Merge pull request #320 from daniel-thom/serialize-attributes

Support serialization of supplemental attributes

src/InfrastructureSystems.jl

@@ -65,11 +65,15 @@ Optional interface functions:
 
   - get_time_series_container()
   - get_component_uuids()
+    Required if the struct does not include the field component_uuids.
   - get_uuid()
 
 Subtypes may contain time series. Which requires
 
   - get_time_series_container()
+
+All subtypes must include an instance of ComponentUUIDs in order to track
+components attached to each attribute.
 """
 abstract type InfrastructureSystemsSupplementalAttribute <: InfrastructureSystemsType end
 
@@ -113,8 +117,10 @@ include("static_time_series.jl")
 include("time_series_container.jl")
 include("time_series_parser.jl")
 include("containers.jl")
+include("component_uuids.jl")
 include("supplemental_attribute.jl")
 include("supplemental_attributes.jl")
+include("supplemental_attributes_container.jl")
 include("components.jl")
 include("geographic_supplemental_attribute.jl")
 include("generated/includes.jl")

src/component.jl

@@ -305,9 +305,9 @@ function attach_supplemental_attribute!(
     attribute_container = get_supplemental_attributes_container(component)
 
     if !haskey(attribute_container, T)
-        attribute_container[T] = Set{T}()
+        attribute_container[T] = Dict{Base.UUID, T}()
     end
-    push!(attribute_container[T], attribute)
+    attribute_container[T][get_uuid(attribute)] = attribute
     @debug "SupplementalAttribute type $T with UUID $(get_uuid(attribute)) stored in component $(summary(component))" _group =
         LOG_GROUP_SYSTEM
     return
@@ -323,8 +323,8 @@ end
 
 function clear_supplemental_attributes!(component::InfrastructureSystemsComponent)
     container = get_supplemental_attributes_container(component)
-    for attribute_set in values(container)
-        for attribute in attribute_set
+    for attributes in values(container)
+        for attribute in collect(values(attributes))
             detach_component!(attribute, component)
             detach_supplemental_attribute!(component, attribute)
         end
@@ -346,7 +346,7 @@ function detach_supplemental_attribute!(
             ),
         )
     end
-    delete!(container[T], attribute)
+    delete!(container[T], get_uuid(attribute))
     if isempty(container[T])
         pop!(container, T)
     end

src/component_uuids.jl

@@ -0,0 +1,33 @@
+# This is an abstraction of a Set in order to enable de-serialization of supplemental
+# attributes.
+
+struct ComponentUUIDs <: InfrastructureSystemsType
+    uuids::Set{Base.UUID}
+
+    function ComponentUUIDs(uuids = Set{Base.UUID}())
+        new(uuids)
+    end
+end
+
+Base.copy(x::ComponentUUIDs) = copy(x.uuids)
+Base.delete!(x::ComponentUUIDs, uuid) = delete!(x.uuids, uuid)
+Base.empty!(x::ComponentUUIDs) = empty!(x.uuids)
+Base.filter!(f, x::ComponentUUIDs) = filter!(f, x.uuids)
+Base.in(x, y::ComponentUUIDs) = in(x, y.uuids)
+Base.isempty(x::ComponentUUIDs) = isempty(x.uuids)
+Base.iterate(x::ComponentUUIDs, args...) = iterate(x.uuids, args...)
+Base.length(x::ComponentUUIDs) = length(x.uuids)
+Base.pop!(x::ComponentUUIDs) = pop!(x.uuids)
+Base.pop!(x::ComponentUUIDs, y) = pop!(x.uuids, y)
+Base.pop!(x::ComponentUUIDs, y, default) = pop!(x.uuids, y, default)
+Base.push!(x::ComponentUUIDs, y) = push!(x.uuids, y)
+Base.setdiff!(x::ComponentUUIDs, y::ComponentUUIDs) = setdiff!(x.uuids, y.uuids)
+Base.sizehint!(x::ComponentUUIDs, newsz) = sizehint!(x.uuids, newsz)
+
+function deserialize(::Type{ComponentUUIDs}, data::Dict)
+    uuids = Set{Base.UUID}()
+    for uuid in data["uuids"]
+        push!(uuids, deserialize(Base.UUID, uuid))
+    end
+    return ComponentUUIDs(uuids)
+end

src/geographic_supplemental_attribute.jl

@@ -3,15 +3,16 @@ Attribute to store Geographic Information about the system components
 """
 struct GeographicInfo <: InfrastructureSystemsSupplementalAttribute
     geo_json::Dict{String, Any}
-    component_uuids::Set{UUIDs.UUID}
+    component_uuids::ComponentUUIDs
     internal::InfrastructureSystemsInternal
 end
 
 function GeographicInfo(;
-    geo_json::Dict{String, Any} = Dict{String, Any}(),
-    component_uuids::Set{UUIDs.UUID} = Set{UUIDs.UUID}(),
+    geo_json::Dict{String, <:Any} = Dict{String, Any}(),
+    component_uuids::ComponentUUIDs = ComponentUUIDs(),
+    internal = InfrastructureSystemsInternal(),
 )
-    return GeographicInfo(geo_json, component_uuids, InfrastructureSystemsInternal())
+    return GeographicInfo(geo_json, component_uuids, internal)
 end
 
 get_geo_json(geo::GeographicInfo) = geo.geo_json

src/serialization.jl

@@ -16,7 +16,7 @@ function to_json(
     pretty = false,
 ) where {T <: InfrastructureSystemsType}
     if !force && isfile(filename)
-        error("$file already exists. Set force=true to overwrite.")
+        error("$filename already exists. Set force=true to overwrite.")
     end
     result = open(filename, "w") do io
         return to_json(io, obj; pretty = pretty)
@@ -161,8 +161,7 @@ end
 
 function deserialize_struct(::Type{TimeSeriesKey}, data::Dict)
     vals = Dict{Symbol, Any}()
-    for (field_name, field_type) in
-        zip(fieldnames(TimeSeriesKey), fieldtypes(TimeSeriesKey))
+    for field_name in fieldnames(TimeSeriesKey)
         val = data[string(field_name)]
         if field_name == :time_series_type
             val = getfield(InfrastructureSystems, Symbol(strip_module_name(val)))
@@ -176,7 +175,10 @@ function deserialize_to_dict(::Type{T}, data::Dict) where {T}
     # Note: mostly duplicated in src/deterministic_metadata.jl
     vals = Dict{Symbol, Any}()
     for (field_name, field_type) in zip(fieldnames(T), fieldtypes(T))
-        val = data[string(field_name)]
+        name_str = string(field_name)
+        # Some types may not serialize optional fields.
+        !haskey(data, name_str) && continue
+        val = data[name_str]
         if val isa Dict && haskey(val, METADATA_KEY)
             metadata = get_serialization_metadata(val)
             if haskey(metadata, FUNCTION_KEY)
@@ -255,6 +257,7 @@ deserialize(::Type{Dates.DateTime}, val::AbstractString) = Dates.DateTime(val)
 
 serialize(uuid::Base.UUID) = Dict("value" => string(uuid))
 serialize(uuids::Vector{Base.UUID}) = serialize.(uuids)
+serialize(uuids::Set{Base.UUID}) = serialize.(uuids)
 deserialize(::Type{Base.UUID}, data::Dict) = Base.UUID(data["value"])
 
 serialize(value::Complex) = Dict("real" => real(value), "imag" => imag(value))

src/supplemental_attributes.jl

@@ -1,5 +1,3 @@
-const SupplementalAttributesContainer =
-    Dict{DataType, Set{<:InfrastructureSystemsSupplementalAttribute}}
 const SupplementalAttributesByType =
     Dict{DataType, Dict{UUIDs.UUID, <:InfrastructureSystemsSupplementalAttribute}}
 
@@ -227,3 +225,31 @@ function get_supplemental_attributes(
     @assert_op eltype(iter) == T
     return iter
 end
+
+function serialize(attributes::SupplementalAttributes)
+    return [serialize(y) for x in values(attributes.data) for y in values(x)]
+end
+
+function deserialize(
+    ::Type{SupplementalAttributes},
+    data::Vector,
+    time_series_storage::TimeSeriesStorage,
+)
+    attributes = SupplementalAttributesByType()
+    for attr_dict in data
+        type = get_type_from_serialization_metadata(get_serialization_metadata(attr_dict))
+        if !haskey(attributes, type)
+            attributes[type] =
+                Dict{UUIDs.UUID, InfrastructureSystemsSupplementalAttribute}()
+        end
+        attr = deserialize(type, attr_dict)
+        uuid = get_uuid(attr)
+        if haskey(attributes[type], uuid)
+            error("Bug: duplicate UUID in attributes container: type=$type uuid=$uuid")
+        end
+        attributes[type][uuid] = attr
+        @debug "Deserialized $(summary(attr))" _group = LOG_GROUP_SERIALIZATION
+    end
+
+    return SupplementalAttributes(attributes, time_series_storage)
+end

src/supplemental_attributes_container.jl

@@ -0,0 +1,49 @@
+"""
+All components must include a field of this type in order to store supplemental attributes.
+"""
+struct SupplementalAttributesContainer
+    data::SupplementalAttributesByType
+end
+
+function SupplementalAttributesContainer(; data = SupplementalAttributesByType())
+    return SupplementalAttributesContainer(data)
+end
+
+Base.getindex(x::SupplementalAttributesContainer, key) = getindex(x.data, key)
+Base.haskey(x::SupplementalAttributesContainer, key) = haskey(x.data, key)
+Base.isempty(x::SupplementalAttributesContainer) = isempty(x.data)
+Base.iterate(x::SupplementalAttributesContainer, args...) = iterate(x.data, args...)
+Base.length(x::SupplementalAttributesContainer) = length(x.data)
+Base.values(x::SupplementalAttributesContainer) = values(x.data)
+Base.delete!(x::SupplementalAttributesContainer, key) = delete!(x.data, key)
+Base.empty!(x::SupplementalAttributesContainer) = empty!(x.data)
+Base.setindex!(x::SupplementalAttributesContainer, val, key) = setindex!(x.data, val, key)
+Base.pop!(x::SupplementalAttributesContainer, key) = pop!(x.data, key)
+
+function serialize(container::SupplementalAttributesContainer)
+    return [serialize(uuid) for attrs in values(container) for uuid in keys(attrs)]
+end
+
+function deserialize(
+    ::Type{SupplementalAttributesContainer},
+    uuids::Vector,
+    system_attributes::Dict{Base.UUID, <:InfrastructureSystemsSupplementalAttribute},
+)
+    container = SupplementalAttributesContainer()
+    for uuid_dict in uuids
+        uuid = deserialize(Base.UUID, uuid_dict)
+        attribute = system_attributes[uuid]
+        type = typeof(attribute)
+        if !haskey(container, type)
+            container[type] = Dict{Base.UUID, InfrastructureSystemsSupplementalAttribute}()
+        end
+        if haskey(container[type], uuid)
+            error(
+                "Bug: component supplemental attribute container already has uuid = $uuid",
+            )
+        end
+        container[type][uuid] = attribute
+    end
+
+    return container
+end

src/system_data.jl

@@ -81,14 +81,15 @@ function SystemData(
     time_series_params,
     validation_descriptors,
     time_series_storage,
+    attributes,
     internal,
 )
     components = Components(time_series_storage, validation_descriptors)
     masked_components = Components(time_series_storage, validation_descriptors)
     return SystemData(
         components,
         masked_components,
-        SupplementalAttributes(time_series_storage),
+        attributes,
         time_series_params,
         time_series_storage,
         validation_descriptors,
@@ -649,7 +650,8 @@ end
 function serialize(data::SystemData)
     @debug "serialize SystemData" _group = LOG_GROUP_SERIALIZATION
     json_data = Dict()
-    for field in (:components, :masked_components, :time_series_params, :internal)
+    for field in
+        (:components, :masked_components, :attributes, :time_series_params, :internal)
         json_data[string(field)] = serialize(getfield(data, field))
     end
 
@@ -726,14 +728,35 @@ function deserialize(
         )
     end
 
+    attributes = deserialize(SupplementalAttributes, raw["attributes"], time_series_storage)
     internal = deserialize(InfrastructureSystemsInternal, raw["internal"])
     @debug "deserialize" _group = LOG_GROUP_SERIALIZATION validation_descriptors time_series_storage internal
     sys = SystemData(
         time_series_params,
         validation_descriptors,
         time_series_storage,
+        attributes,
         internal,
     )
+    attributes_by_uuid = Dict{Base.UUID, InfrastructureSystemsSupplementalAttribute}()
+    for attr_dict in values(attributes.data)
+        for attr in values(attr_dict)
+            uuid = get_uuid(attr)
+            if haskey(attributes_by_uuid, uuid)
+                error("Bug: Found duplicate supplemental attribute UUID: $uuid")
+            end
+            attributes_by_uuid[uuid] = attr
+        end
+    end
+    for component in raw["components"]
+        if haskey(component, "attributes_container")
+            component["attributes_container"] = deserialize(
+                SupplementalAttributesContainer,
+                component["attributes_container"],
+                attributes_by_uuid,
+            )
+        end
+    end
     # Note: components need to be deserialized by the parent so that they can go through
     # the proper checks.
     return sys

src/utils/test.jl

@@ -1,4 +1,3 @@
-
 mutable struct TestComponent <: InfrastructureSystemsComponent
     name::String
     val::Int
@@ -7,14 +6,6 @@ mutable struct TestComponent <: InfrastructureSystemsComponent
     internal::InfrastructureSystemsInternal
 end
 
-mutable struct AdditionalTestComponent <: InfrastructureSystemsComponent
-    name::String
-    val::Int
-    time_series_container::TimeSeriesContainer
-    attributes_container::SupplementalAttributesContainer
-    internal::InfrastructureSystemsInternal
-end
-
 function TestComponent(name, val)
     return TestComponent(
         name,
@@ -25,6 +16,14 @@ function TestComponent(name, val)
     )
 end
 
+mutable struct AdditionalTestComponent <: InfrastructureSystemsComponent
+    name::String
+    val::Int
+    time_series_container::TimeSeriesContainer
+    attributes_container::SupplementalAttributesContainer
+    internal::InfrastructureSystemsInternal
+end
+
 function AdditionalTestComponent(name, val)
     return AdditionalTestComponent(
         name,
@@ -35,6 +34,20 @@ function AdditionalTestComponent(name, val)
     )
 end
 
+mutable struct SimpleTestComponent <: InfrastructureSystemsComponent
+    name::String
+    val::Int
+    internal::InfrastructureSystemsInternal
+end
+
+function SimpleTestComponent(name, val)
+    return SimpleTestComponent(name, val, InfrastructureSystemsInternal())
+end
+
+function SimpleTestComponent(; name, val, internal = InfrastructureSystemsInternal())
+    return SimpleTestComponent(name, val, internal)
+end
+
 get_internal(component::TestComponent) = component.internal
 get_internal(component::AdditionalTestComponent) = component.internal
 get_val(component::TestComponent) = component.val
@@ -57,7 +70,7 @@ function deserialize(::Type{TestComponent}, data::Dict)
         data["name"],
         data["val"],
         deserialize(TimeSeriesContainer, data["time_series_container"]),
-        SupplementalAttributesContainer(),
+        data["attributes_container"],
         deserialize(InfrastructureSystemsInternal, data["internal"]),
     )
 end
@@ -100,20 +113,22 @@ end
 
 struct TestSupplemental <: InfrastructureSystemsSupplementalAttribute
     value::Float64
-    component_uuids::Set{UUIDs.UUID}
+    component_uuids::ComponentUUIDs
     internal::InfrastructureSystemsInternal
     time_series_container::TimeSeriesContainer
 end
 
 function TestSupplemental(;
-    value::Float64 = 0.0,
-    component_uuids::Set{UUIDs.UUID} = Set{UUIDs.UUID}(),
+    value::Float64,
+    component_uuids::ComponentUUIDs = ComponentUUIDs(),
+    time_series_container = TimeSeriesContainer(),
+    internal::InfrastructureSystemsInternal = InfrastructureSystemsInternal(),
 )
     return TestSupplemental(
         value,
         component_uuids,
-        InfrastructureSystemsInternal(),
-        TimeSeriesContainer(),
+        internal,
+        time_series_container,
     )
 end