From 129d11ed82a20f4eceedc8fb3ec7f94b2b94af1e Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Mon, 16 Sep 2019 17:27:25 -0600 Subject: [PATCH 1/2] Forward forecast indexing and splitting methods to TimeSeries. --- src/forecasts.jl | 145 ++++++++++++++++++++++++++++++++- src/generated/Deterministic.jl | 4 + src/generated/Probabilistic.jl | 4 + src/generated/ScenarioBased.jl | 4 + src/utils/generate_structs.jl | 8 ++ test/test_forecasts.jl | 88 ++++++++++++++++++++ 6 files changed, 251 insertions(+), 2 deletions(-) diff --git a/src/forecasts.jl b/src/forecasts.jl index 8c9bb4faa..eae99fc2e 100644 --- a/src/forecasts.jl +++ b/src/forecasts.jl @@ -530,8 +530,10 @@ function convert_type!( throw(DataFormatError("unmatched timeseries UUID: $uuid $forecast")) end timeseries = forecast_uuid_to_timeseries[uuid] - forecast_base_type = getfield(InfrastructureSystems, Symbol(strip_module_names(string(forecast.type)))) - push!(forecasts_, convert_type(forecast_base_type, forecast, component_cache, timeseries)) + forecast_base_type = getfield(InfrastructureSystems, + Symbol(strip_module_names(string(forecast.type)))) + val = convert_type(forecast_base_type, forecast, component_cache, timeseries) + push!(forecasts_, val) end _add_forecasts!(forecasts, forecasts_) @@ -605,3 +607,142 @@ function get_resolution(ts::TimeSeries.TimeArray) return res[1] end +""" +Creates a new forecast from an existing forecast with a split TimeArray. + +# Arguments +- `is_copy::Bool=true`: Reset internal indices because the TimeArray is a fresh copy. +""" +function _split_forecast( + forecast::T, + data::TimeSeries.TimeArray; + is_copy=true, + ) where T <: Forecast + vals = [] + for (fname, ftype) in zip(fieldnames(T), fieldtypes(T)) + if ftype <: TimeSeries.TimeArray + val = data + elseif ftype <: InfrastructureSystemsInternal + # Need to create a new UUID. + continue + else + val = getfield(forecast, fname) + end + + push!(vals, val) + end + + new_forecast = T(vals...) + if is_copy + new_forecast.start_index = 1 + new_forecast.horizon = length(get_data(new_forecast)) + end + return new_forecast +end + +function Base.getindex(forecast::Forecast, args...) + return _split_forecast(forecast, getindex(get_timeseries(forecast), args...)) +end + +Base.first(forecast::Forecast) = head(forecast, 1) + +Base.last(forecast::Forecast) = tail(forecast, 1) + +Base.firstindex(forecast::Forecast) = firstindex(get_timeseries(forecast)) + +Base.lastindex(forecast::Forecast) = lastindex(get_timeseries(forecast)) + +Base.lastindex(forecast::Forecast, d) = lastindex(get_timeseries(forecast), d) + +Base.eachindex(forecast::Forecast) = eachindex(get_timeseries(forecast)) + +Base.iterate(forecast::Forecast, n = 1) = iterate(get_timeseries(forecast), n) + +""" + when(forecast::Forecast, period::Function, t::Integer) + +Refer to TimeSeries.when(). Underlying data is copied. +""" +function when(forecast::Forecast, period::Function, t::Integer) + new = _split_forecast(forecast, TimeSeries.when(get_timeseries(forecast), period, t)) + +end + +""" + from(forecast::Forecast, timestamp) + +Return a forecast truncated starting with timestamp. Underlying data is not copied. +""" +function from(forecast::Forecast, timestamp) + # Don't use TimeSeries.from because it makes a copy. + start_index = get_start_index(forecast) + end_index = start_index + get_horizon(forecast) + for i in get_start_index(forecast) : end_index + if TimeSeries.timestamp(get_data(forecast))[i] >= timestamp + fcast = _split_forecast(forecast, get_data(forecast); is_copy=false) + fcast.start_index = i + fcast.horizon = end_index - i + return fcast + end + end + + # Do whatever TimeSeries does if the timestamp is after the forecast. + return _split_forecast(forecast, TimeSeries.from(get_timeseries(forecast), timestamp)) +end + +""" + to(forecast::Forecast, timestamp) + +Return a forecast truncated after timestamp. Underlying data is not copied. +""" +function to(forecast::Forecast, timestamp) + # Don't use TimeSeries.from because it makes a copy. + start_index = get_start_index(forecast) + end_index = start_index + get_horizon(forecast) + for i in get_start_index(forecast) : end_index + tstamp = TimeSeries.timestamp(get_data(forecast))[i] + if tstamp < timestamp + continue + elseif tstamp == timestamp + end_index = i + else + @assert tstamp > timestamp + end_index = i - 1 + end + + fcast = _split_forecast(forecast, get_data(forecast); is_copy=false) + fcast.horizon = end_index - start_index + 1 + return fcast + end + + # Do whatever TimeSeries does if the timestamp is after the forecast. + return _split_forecast(forecast, TimeSeries.to(get_timeseries(forecast), timestamp)) +end + +""" + head(forecast::Forecast) + head(forecast::Forecast, num) + +Return a forecast with only the first num values. +""" +function head(forecast::Forecast) + return _split_forecast(forecast, TimeSeries.head(get_timeseries(forecast))) +end + +function head(forecast::Forecast, num) + return _split_forecast(forecast, TimeSeries.head(get_timeseries(forecast), num)) +end + +""" + tail(forecast::Forecast) + tail(forecast::Forecast, num) + +Return a forecast with only the ending num values. +""" +function tail(forecast::Forecast) + return _split_forecast(forecast, TimeSeries.tail(get_timeseries(forecast))) +end + +function tail(forecast::Forecast, num) + return _split_forecast(forecast, TimeSeries.tail(get_timeseries(forecast), num)) +end diff --git a/src/generated/Deterministic.jl b/src/generated/Deterministic.jl index f591c6076..c3c7e570e 100644 --- a/src/generated/Deterministic.jl +++ b/src/generated/Deterministic.jl @@ -22,6 +22,10 @@ function Deterministic(; component, label, resolution, initial_time, data, start Deterministic(component, label, resolution, initial_time, data, start_index, horizon, ) end +function Deterministic{T}(component, label, resolution, initial_time, data, start_index, horizon, ) where T <: InfrastructureSystemsType + Deterministic(component, label, resolution, initial_time, data, start_index, horizon, InfrastructureSystemsInternal()) +end + """Get Deterministic component.""" get_component(value::Deterministic) = value.component diff --git a/src/generated/Probabilistic.jl b/src/generated/Probabilistic.jl index 2d1543f04..20aa49b0c 100644 --- a/src/generated/Probabilistic.jl +++ b/src/generated/Probabilistic.jl @@ -23,6 +23,10 @@ function Probabilistic(; component, label, resolution, initial_time, percentiles Probabilistic(component, label, resolution, initial_time, percentiles, data, start_index, horizon, ) end +function Probabilistic{T}(component, label, resolution, initial_time, percentiles, data, start_index, horizon, ) where T <: InfrastructureSystemsType + Probabilistic(component, label, resolution, initial_time, percentiles, data, start_index, horizon, InfrastructureSystemsInternal()) +end + """Get Probabilistic component.""" get_component(value::Probabilistic) = value.component diff --git a/src/generated/ScenarioBased.jl b/src/generated/ScenarioBased.jl index e76353824..3c5523f03 100644 --- a/src/generated/ScenarioBased.jl +++ b/src/generated/ScenarioBased.jl @@ -23,6 +23,10 @@ function ScenarioBased(; component, label, resolution, initial_time, scenario_co ScenarioBased(component, label, resolution, initial_time, scenario_count, data, start_index, horizon, ) end +function ScenarioBased{T}(component, label, resolution, initial_time, scenario_count, data, start_index, horizon, ) where T <: InfrastructureSystemsType + ScenarioBased(component, label, resolution, initial_time, scenario_count, data, start_index, horizon, InfrastructureSystemsInternal()) +end + """Get ScenarioBased component.""" get_component(value::ScenarioBased) = value.component diff --git a/src/utils/generate_structs.jl b/src/utils/generate_structs.jl index 5075911e2..6c49d4f54 100644 --- a/src/utils/generate_structs.jl +++ b/src/utils/generate_structs.jl @@ -34,6 +34,14 @@ function {{struct_name}}(; {{#parameters}}{{^internal}}{{name}}, {{/internal}}{{ {{struct_name}}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}}) end +{{#parametric}} +function {{struct_name}}{T}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}}) where T <: InfrastructureSystemsType + {{#parameters}} + {{/parameters}} + {{struct_name}}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}}InfrastructureSystemsInternal()) +end +{{/parametric}} + {{#has_null_values}} # Constructor for demo purposes; non-functional. diff --git a/test/test_forecasts.jl b/test/test_forecasts.jl index 6a5cd279c..8577b26c4 100644 --- a/test/test_forecasts.jl +++ b/test/test_forecasts.jl @@ -1,4 +1,7 @@ +import Dates +import TimeSeries + function does_forecast_have_component(data::SystemData, component) found = false for forecast in iterate_forecasts(data) @@ -86,3 +89,88 @@ end data = create_system_data(; with_forecasts=true) summary(devnull, data.forecasts) end + +@testset "Test split_forecast" begin + data = create_system_data(; with_forecasts=true) + forecast = get_all_forecasts(data)[1] + + forecasts = get_forecasts(IS.Deterministic, data, IS.get_initial_time(forecast)) + split_forecasts!(data, forecasts, Dates.Hour(6), 12) + initial_times = get_forecast_initial_times(data) + @test length(initial_times) == 3 + + for initial_time in initial_times + for fcast in get_forecasts(Deterministic, data, initial_time) + # The backing TimeArray must be the same. + @test get_data(fcast) === get_data(forecast) + @test length(fcast) == 12 + end + end +end + +@testset "Test forecast forwarding methods" begin + data = create_system_data(; with_forecasts=true) + forecast = get_all_forecasts(data)[1] + + # Iteration + size = 24 + @test length(forecast) == size + i = 0 + for x in forecast + i += 1 + end + @test i == size + + # Indexing + @test length(forecast[1:16]) == 16 + + # when + fcast = IS.when(forecast, TimeSeries.hour, 3) + @test length(fcast) == 1 +end + +@testset "Test forecast head" begin + data = create_system_data(; with_forecasts=true) + forecast = get_all_forecasts(data)[1] + fcast = IS.head(forecast) + # head returns a length of 6 by default, but don't hard-code that. + @test length(fcast) < length(forecast) + + fcast = IS.head(forecast, 10) + @test length(fcast) == 10 +end + +@testset "Test forecast tail" begin + data = create_system_data(; with_forecasts=true) + forecast = get_all_forecasts(data)[1] + fcast = IS.tail(forecast) + # tail returns a length of 6 by default, but don't hard-code that. + @test length(fcast) < length(forecast) + + fcast = IS.head(forecast, 10) + @test length(fcast) == 10 +end + +@testset "Test forecast from" begin + data = create_system_data(; with_forecasts=true) + forecast = get_all_forecasts(data)[1] + start_time = Dates.DateTime(Dates.today()) + Dates.Hour(3) + fcast = IS.from(forecast, start_time) + @test get_data(fcast) === get_data(forecast) + @test get_start_index(fcast) == 4 + @test length(fcast) == 21 + @test TimeSeries.timestamp(IS.get_timeseries(fcast))[1] == start_time +end + +@testset "Test forecast from" begin + data = create_system_data(; with_forecasts=true) + forecast = get_all_forecasts(data)[1] + for end_time in (Dates.DateTime(Dates.today()) + Dates.Hour(15), + Dates.DateTime(Dates.today()) + Dates.Hour(15) + Dates.Minute(5)) + fcast = IS.to(forecast, end_time) + @test get_data(fcast) === get_data(forecast) + @test get_start_index(fcast) + get_horizon(fcast) == 17 + @test length(fcast) == 16 + @test TimeSeries.timestamp(IS.get_timeseries(fcast))[end] <= end_time + end +end From a4543ec6499fe2d3917feac61167a0670ae6ede7 Mon Sep 17 00:00:00 2001 From: Daniel Thom Date: Mon, 16 Sep 2019 18:47:59 -0600 Subject: [PATCH 2/2] Revert extra newline. --- src/generated/Deterministic.jl | 1 - src/generated/Probabilistic.jl | 1 - src/generated/ScenarioBased.jl | 1 - src/utils/generate_structs.jl | 1 - 4 files changed, 4 deletions(-) diff --git a/src/generated/Deterministic.jl b/src/generated/Deterministic.jl index c3c7e570e..340966850 100644 --- a/src/generated/Deterministic.jl +++ b/src/generated/Deterministic.jl @@ -26,7 +26,6 @@ function Deterministic{T}(component, label, resolution, initial_time, data, star Deterministic(component, label, resolution, initial_time, data, start_index, horizon, InfrastructureSystemsInternal()) end - """Get Deterministic component.""" get_component(value::Deterministic) = value.component """Get Deterministic label.""" diff --git a/src/generated/Probabilistic.jl b/src/generated/Probabilistic.jl index 20aa49b0c..e31fc28b9 100644 --- a/src/generated/Probabilistic.jl +++ b/src/generated/Probabilistic.jl @@ -27,7 +27,6 @@ function Probabilistic{T}(component, label, resolution, initial_time, percentile Probabilistic(component, label, resolution, initial_time, percentiles, data, start_index, horizon, InfrastructureSystemsInternal()) end - """Get Probabilistic component.""" get_component(value::Probabilistic) = value.component """Get Probabilistic label.""" diff --git a/src/generated/ScenarioBased.jl b/src/generated/ScenarioBased.jl index 3c5523f03..fab508126 100644 --- a/src/generated/ScenarioBased.jl +++ b/src/generated/ScenarioBased.jl @@ -27,7 +27,6 @@ function ScenarioBased{T}(component, label, resolution, initial_time, scenario_c ScenarioBased(component, label, resolution, initial_time, scenario_count, data, start_index, horizon, InfrastructureSystemsInternal()) end - """Get ScenarioBased component.""" get_component(value::ScenarioBased) = value.component """Get ScenarioBased label.""" diff --git a/src/utils/generate_structs.jl b/src/utils/generate_structs.jl index 6c49d4f54..1161e2844 100644 --- a/src/utils/generate_structs.jl +++ b/src/utils/generate_structs.jl @@ -41,7 +41,6 @@ function {{struct_name}}{T}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{ {{struct_name}}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}}InfrastructureSystemsInternal()) end {{/parametric}} - {{#has_null_values}} # Constructor for demo purposes; non-functional.