add code to store 3D in HDF

src/core/dataset.jl

@@ -111,25 +111,43 @@ function make_system_state(
     return InMemoryDataset(NaN, timestamp, resolution, 0, 1, columns)
 end
 
-function get_dataset_value(s::InMemoryDataset, date::Dates.DateTime)
+function get_dataset_value(
+    s::T,
+    date::Dates.DateTime,
+) where {T <: Union{InMemoryDataset{1}, InMemoryDataset{2}}}
     s_index = find_timestamp_index(s.timestamps, date)
     if isnothing(s_index)
         error("Request time stamp $date not in the state")
     end
     return s.values[:, s_index]
 end
 
+function get_dataset_value(s::InMemoryDataset{3}, date::Dates.DateTime)
+    s_index = find_timestamp_index(s.timestamps, date)
+    if isnothing(s_index)
+        error("Request time stamp $date not in the state")
+    end
+    return s.values[:, :, s_index]
+end
+
 function get_column_names(k::OptimizationContainerKey, s::InMemoryDataset)
     return get_column_names(k, s.values)
 end
 
-function get_last_recorded_value(s::InMemoryDataset)
+function get_last_recorded_value(s::InMemoryDataset{2})
     if get_last_recorded_row(s) == 0
         error("The Dataset hasn't been written yet")
     end
     return s.values[:, get_last_recorded_row(s)]
 end
 
+function get_last_recorded_value(s::InMemoryDataset{3})
+    if get_last_recorded_row(s) == 0
+        error("The Dataset hasn't been written yet")
+    end
+    return s.values[:, :, get_last_recorded_row(s)]
+end
+
 function get_end_of_step_timestamp(s::InMemoryDataset)
     return s.timestamps[s.end_of_step_index]
 end
@@ -153,17 +171,20 @@ function get_value_timestamp(s::InMemoryDataset, date::Dates.DateTime)
     return s.timestamps[s_index]
 end
 
-function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 2}, index::Int)
+# These set_value! methods expect a single time_step value because they are used to update
+#the state so the incoming vals will have one dimension less than the DataSet. The exception
+# is for vals of Dimension 1 which are still stored in DataSets of dimension 2.
+function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 2}, index::Int)
     s.values[:, index] = vals[:, index]
     return
 end
 
-function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 1}, index::Int)
+function set_value!(s::InMemoryDataset{2}, vals::DenseAxisArray{Float64, 1}, index::Int)
     s.values[:, index] = vals
     return
 end
 
-function set_value!(s::InMemoryDataset, vals::DenseAxisArray{Float64, 3}, index::Int)
+function set_value!(s::InMemoryDataset{3}, vals::DenseAxisArray{Float64, 2}, index::Int)
     s.values[:, :, index] = vals
     return
 end

src/core/definitions.jl

@@ -24,6 +24,7 @@ const JuMPVariableMatrix = DenseAxisArray{
         JuMP.Containers._AxisLookup{Tuple{Int64, Int64}},
     },
 }
+const JuMPFloatMatrix = DenseAxisArray{Float64, 2}
 const JuMPFloatArray = DenseAxisArray{Float64}
 const JuMPVariableArray = DenseAxisArray{JuMP.VariableRef}
 

src/parameters/update_parameters.jl

@@ -560,7 +560,7 @@ end
 
 function _fix_parameter_value!(
     container::OptimizationContainer,
-    parameter_array::JuMPFloatArray,
+    parameter_array::DenseAxisArray{Float64, 2},
     parameter_attributes::VariableValueAttributes,
 )
     affected_variable_keys = parameter_attributes.affected_keys

src/simulation/hdf_simulation_store.jl

@@ -592,11 +592,10 @@ function write_result!(
     ::Dates.DateTime,
     data::DenseAxisArray{Float64, 3, <:NTuple{3, Any}},
 )
-    @show "Here"
-    #=
     output_cache = get_output_cache(store.cache, model_name, key)
     cur_size = get_size(store.cache)
-    add_result!(output_cache, index, to_matrix(data), is_full(store.cache, cur_size))
+
+    add_result!(output_cache, index, data.data, is_full(store.cache, cur_size))
 
     if get_dirty_size(output_cache) >= get_min_flush_size(store.cache)
         discard = !should_keep_in_cache(output_cache)
@@ -614,7 +613,6 @@ function write_result!(
     #end
 
     @debug "write_result" get_size(store.cache) encode_key_as_string(key)
-    =#
     return
 end
 
@@ -627,10 +625,10 @@ function write_result!(
     key::OptimizationContainerKey,
     index::EmulationModelIndexType,
     simulation_time::Dates.DateTime,
-    data::Matrix{Float64},
+    data::Array{Float64},
 )
     dataset = _get_em_dataset(store, key)
-    _write_dataset!(dataset.values, data, index:index)
+    _write_dataset!(dataset.values, data, index)
     set_last_recorded_row!(dataset, index)
     set_update_timestamp!(dataset, simulation_time)
     return
@@ -642,7 +640,21 @@ function write_result!(
     key::OptimizationContainerKey,
     index::EmulationModelIndexType,
     simulation_time::Dates.DateTime,
-    data,
+    data::DenseAxisArray{Float64, 2},
+)
+    data_array = Array{Float64, 3}(undef, size(data)[1], size(data)[2], 1)
+    data_array[:, :, 1] = data
+    write_result!(store, model_name, key, index, simulation_time, data_array)
+    return
+end
+
+function write_result!(
+    store::HdfSimulationStore,
+    model_name::Symbol,
+    key::OptimizationContainerKey,
+    index::EmulationModelIndexType,
+    simulation_time::Dates.DateTime,
+    data::DenseAxisArray{Float64, 1},
 )
     write_result!(store, model_name, key, index, simulation_time, to_matrix(data))
     return
@@ -984,7 +996,7 @@ function _read_length(::Type{OptimizerStats}, store::HdfSimulationStore)
 end
 
 function _write_dataset!(
-    dataset,
+    dataset::HDF5.Dataset,
     array::Matrix{Float64},
     row_range::UnitRange{Int64},
     ::Val{3},
@@ -995,7 +1007,7 @@ function _write_dataset!(
 end
 
 function _write_dataset!(
-    dataset,
+    dataset::HDF5.Dataset,
     array::Matrix{Float64},
     row_range::UnitRange{Int64},
     ::Val{2},
@@ -1005,13 +1017,38 @@ function _write_dataset!(
     return
 end
 
-function _write_dataset!(dataset, array::Matrix{Float64}, row_range::UnitRange{Int64})
-    _write_dataset!(dataset, array, row_range, Val{ndims(dataset)}())
+function _write_dataset!(
+    dataset::HDF5.Dataset,
+    array::Array{Float64, 3},
+    row_range::UnitRange{Int64},
+    ::Val{3},
+)
+    dataset[row_range, :, :] = array
+    @debug "wrote dataset" dataset row_range
+    return
+end
+
+function _write_dataset!(dataset::HDF5.Dataset, array::Array{Float64}, index::Int)
+    _write_dataset!(dataset, array, index:index, Val{ndims(dataset)}())
     return
 end
 
-function _write_dataset!(dataset, array::Array{Float64, 3}, row_range::UnitRange{Int64})
+function _write_dataset!(
+    dataset::HDF5.Dataset,
+    array::Array{Float64, 3},
+    row_range::UnitRange{Int64},
+)
     dataset[:, :, row_range] = array
     @debug "wrote dataset" dataset row_range
     return
 end
+
+function _write_dataset!(
+    dataset::HDF5.Dataset,
+    array::Array{Float64, 4},
+    row_range::UnitRange{Int64},
+)
+    dataset[:, :, :, row_range] = array
+    @debug "wrote dataset" dataset row_range
+    return
+end

src/simulation/simulation_state.jl

@@ -410,7 +410,8 @@ function update_system_state!(
     set_update_timestamp!(system_dataset, ts)
     # Keep coordination between fields. System state is an array of size 1
     system_dataset.timestamps[1] = ts
-    set_dataset_values!(state, key, 1, get_dataset_value(decision_dataset, simulation_time))
+    data_set_value = get_dataset_value(decision_dataset, simulation_time)
+    set_dataset_values!(state, key, 1, data_set_value)
     # This value shouldn't be other than one and after one execution is no-op.
     set_last_recorded_row!(system_dataset, 1)
     return