From e1fff39e3b3c3622407da877988c0aecab76ae65 Mon Sep 17 00:00:00 2001 From: Stuart Daines Date: Sat, 6 Aug 2022 12:17:27 +0100 Subject: [PATCH] OutputWriter fixes - standardize OutputWriter argument names coords_record, coords_record_units - handle case where OutputWriter has less records than DataFrame has rows --- src/FieldRecord.jl | 27 +++++++++++++++++---------- src/OutputWriters.jl | 35 ++++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/FieldRecord.jl b/src/FieldRecord.jl index d7c3ea1..16d5e77 100644 --- a/src/FieldRecord.jl +++ b/src/FieldRecord.jl @@ -1,11 +1,18 @@ """ - FieldRecord{D <: AbstractData, S <: AbstractSpace, ...} + FieldRecord{D <: AbstractData, S <: AbstractSpace, V, N, M, R} + FieldRecord( + f::PB.Field{D, S, V, N, M}, attributes; + coords_record, + sizehint::Union{Nothing, Int}=nothing + ) -> fr -A series of records each containing a `PALEOboxes.Field`. +A series of `records::R` each containing the `values` from a `PALEOboxes.Field{D, S, N, V, M}`. + +A `coords_record` may be attached to provide a coordinate (eg model time) corresponding to `records`. # Implementation -Fields with array values are stored in `records` as a Vector of arrays. -Fields with single values (`field_single_element` true) are stored as a Vector of `eltype(Field.values)`. +Fields with array `values` are stored in `records` as a Vector of arrays. +Fields with single `values` (`field_single_element` true) are stored as a Vector of `eltype(Field.values)`. """ struct FieldRecord{D <: PB.AbstractData, S <: PB.AbstractSpace, V, N, M, R} records::Vector{R} @@ -44,7 +51,7 @@ field_single_element(::Type{PB.Field{D, S, V, N, M}}) where {D, S, V, N, M} = fi field_single_element(::Type{FR}) where {FR <: FieldRecord} = field_single_element(eltype(FR)) field_single_element(f::T) where {T} = field_single_element(T) -"create empty FieldRecord" + function FieldRecord( f::PB.Field{D, S, V, N, M}, attributes; coords_record, @@ -63,9 +70,9 @@ function FieldRecord( return FieldRecord{D, S, V, N, M, eltype(records)}(records, f.data_dims, f.mesh, attributes, coords_record) end -"create a new FieldRecord, containing supplied `existing_values` data arrays" +"create a new FieldRecord, containing supplied `existing_values::Vector` data arrays" function wrap_fieldrecord( - existing_values, + existing_values::Vector, field_data::Type, data_dims::NTuple{N, PB.NamedDimension}, data_type::Union{DataType, Missing}, @@ -78,10 +85,10 @@ function wrap_fieldrecord( # existing_values, field_data, data_dims, data_type, space, spatial_size(space, mesh), # ) if field_single_element(field_data, N, space, M) - # assume existing_values is a Vector, with element a Field value, to be stored in Field as a length-1 Vector - V = Array{eltype(existing_values), 1} + # assume existing_values is a Vector, with each element to be stored in Field values::V as a length 1 Vector + V = Vector{eltype(existing_values)} else - # assume existing_values is a Vector of Field values + # assume existing_values is a Vector of Field values::V V = eltype(existing_values) end diff --git a/src/OutputWriters.jl b/src/OutputWriters.jl index dbc4755..c034a36 100644 --- a/src/OutputWriters.jl +++ b/src/OutputWriters.jl @@ -49,16 +49,18 @@ Implementations should define methods for: PALEOmodel.AbstractOutputWriter """ - initialize!(output::PALEOmodel.AbstractOutputWriter, model, modeldata, nrecords [;rec_coord=:tmodel]) + initialize!( + output::PALEOmodel.AbstractOutputWriter, model, modeldata, nrecords + [;coords_record=:tmodel] [coords_record_units="yr"] + ) Initialize from a PALEOboxes::Model, reserving memory for an assumed output dataset of `nrecords`. -The default for `rec_coord` is `:tmodel`, for a sequence of records following the time evolution +The default for `coords_record` is `:tmodel`, for a sequence of records following the time evolution of the model. """ function initialize!( - output::PALEOmodel.AbstractOutputWriter, model::PB.Model, modeldata::PB.ModelData, nrecords; - rec_coord::Symbol=:tmodel + output::PALEOmodel.AbstractOutputWriter, model::PB.Model, modeldata::PB.ModelData, nrecords ) end @@ -222,8 +224,8 @@ Base.length(output::OutputMemoryDomain) = output._nrecs "create from a PALEOboxes::Domain" function OutputMemoryDomain( - dom::PB.Domain, modeldata::PB.ModelData, nrecords; - coords_record=:tmodel, coords_units="yr" + dom::PB.Domain, modeldata::PB.ModelData, nrecords::Integer; + coords_record::Symbol=:tmodel, coords_record_units::AbstractString="yr" ) odom = OutputMemoryDomain( @@ -271,7 +273,7 @@ function OutputMemoryDomain( odom.metadata[String(coords_record)] = Dict( :var_name=>String(coords_record), :domain_name=>dom.name, :vfunction=>PB.VF_Undefined, :description=>"output record coordinate", - :field_data=>PB.ScalarData, :space=>PB.ScalarSpace, :data_dims=>(), :units=>coords_units, + :field_data=>PB.ScalarData, :space=>PB.ScalarSpace, :data_dims=>(), :units=>coords_record_units, ) # add variables @@ -304,8 +306,8 @@ end "create from a DataFrames DataFrame containing scalar data" function OutputMemoryDomain( name::AbstractString, data::DataFrames.DataFrame; - metadata::Dict{String, Dict{Symbol, Any}}=Dict("tmodel"=>Dict{Symbol, Any}(:units=>"yr")), - coords_record=:tmodel, + coords_record::Symbol=:tmodel, coords_record_units::AbstractString="yr", + metadata::Dict{String, Dict{Symbol, Any}}=Dict(coords_record=>Dict{Symbol, Any}(:units=>coords_record_units)), ) # create minimal metadata for scalar Variables for vname in DataFrames.names(data) @@ -413,7 +415,9 @@ function PB.get_field(odom::OutputMemoryDomain, varname) varname in DataFrames.names(df) || error("Variable $varname not found in output (no column '$varname' in Dataframe output.domains[\"$(odom.name)\"].data)") - vdata = df[!, Symbol(varname)] + use_all_records = DataFrames.nrow(df) == odom._nrecs + # vdata = df[!, Symbol(varname)] + vdata = use_all_records ? df[!, Symbol(varname)] : df[1:odom._nrecs, Symbol(varname)] attributes = get(odom.metadata, varname, nothing) @@ -438,12 +442,15 @@ function PB.get_field(odom::OutputMemoryDomain, varname) coords_record=[ PB.FixedCoord( String(odom.coords_record), - df[!, odom.coords_record], + # df[!, odom.coords_record], + use_all_records ? df[!, odom.coords_record] : df[1:odom._nrecs, odom.coords_record], odom.metadata[String(odom.coords_record)] ), ] ) + # @Infiltrator.infiltrate + return fr end @@ -656,7 +663,8 @@ end function initialize!( output::OutputMemory, model::PB.Model, modeldata::PB.ModelData, nrecords; - rec_coord::Symbol=:tmodel + rec_coord::Symbol=:tmodel, # deprecated + coords_record::Symbol=rec_coord, coords_record_units::AbstractString="yr", ) # Create Dict of DataFrames with output @@ -664,7 +672,8 @@ function initialize!( for dom in model.domains output.domains[dom.name] = OutputMemoryDomain( dom, modeldata, nrecords, - coords_record=rec_coord, + coords_record=coords_record, + coords_record_units=coords_record_units, ) end