diff --git a/ext/RastersArchGDALExt/gdal_source.jl b/ext/RastersArchGDALExt/gdal_source.jl index 86600774..6483d58f 100644 --- a/ext/RastersArchGDALExt/gdal_source.jl +++ b/ext/RastersArchGDALExt/gdal_source.jl @@ -71,7 +71,7 @@ function Base.write(filename::AbstractString, ::GDALsource, A::AbstractRaster{T} if write mod = RA._mod(eltype, missingval_pair, scale, offset, coerce) open(A1; write=true) do O - R = RA._maybe_modify(AG.RasterDataset(dataset), mod) + R = RA.ModifiedDiskArray(AG.RasterDataset(dataset), mod) R .= parent(O) if hasdim(A, Band()) f(R) @@ -296,7 +296,7 @@ function AG.RasterDataset(f::Function, A::AbstractRaster; rds = AG.RasterDataset(dataset) mv = RA.missingval(rds) => RA.missingval(O) mod = RA._mod(eltype, mv, scale, offset, coerce) - RA._maybe_modify(rds, mod) .= parent(O) + RA.ModifiedDiskArray(rds, mod) .= parent(O) f(rds) end end diff --git a/src/Rasters.jl b/src/Rasters.jl index e660e7c8..44382835 100644 --- a/src/Rasters.jl +++ b/src/Rasters.jl @@ -54,7 +54,7 @@ using ColorTypes: RGB using CommonDataModel: AbstractDataset, AbstractVariable -using DiskArrays: @implement_diskarray +import DiskArrays: @implement_diskarray, isdisk using GeometryOpsCore: Planar, Spherical export Planar, Spherical diff --git a/src/array.jl b/src/array.jl index 79797862..fdf52acd 100644 --- a/src/array.jl +++ b/src/array.jl @@ -62,8 +62,8 @@ filename(A::DiskArrays.AbstractDiskArray) = filename(parent(A)) cleanreturn(A::AbstractRaster) = rebuild(A, cleanreturn(parent(A))) cleanreturn(x) = x -isdisk(A::AbstractRaster) = parent(A) isa DiskArrays.AbstractDiskArray -isdisk(x) = false +DiskArrays.isdisk(A::AbstractRaster) = isdisk(parent(A)) +DiskArrays.isdisk(x) = false ismem(A::AbstractRaster) = !isdisk(A) function Base.:(==)(A::AbstractRaster{T,N}, B::AbstractRaster{T,N}) where {T,N} @@ -298,7 +298,7 @@ function Raster(filename::AbstractString; kw... ) source = sourcetrait(filename, source) - _open(filename; source, mod=NoMod()) do ds + _open(filename; source, mod=nothing) do ds Raster(ds, filename; source, kw...) end::Raster end @@ -334,7 +334,7 @@ function Raster(ds, filename::AbstractString; source = sourcetrait(filename, source) # Open the dataset and variable specified by `name`, at `group` level if provided # At this level we do not apply `mod`. - data_out, dims_out, metadata_out, missingval_out = _open(source, ds; name=name1, group, mod=NoMod()) do var + data_out, dims_out, metadata_out, missingval_out = _open(source, ds; name=name1, group, mod=nothing) do var metadata_out = isnokw(metadata) ? _metadata(var) : metadata missingval_out = _read_missingval_pair(var, metadata_out, missingval) # Generate mod for scaling diff --git a/src/filearray.jl b/src/filearray.jl index a599731d..0ea57470 100644 --- a/src/filearray.jl +++ b/src/filearray.jl @@ -67,12 +67,20 @@ end function DA.readblock!(A::FileArray, dst, r::AbstractUnitRange...) open(A) do O - DA.readblock!(O, dst, r...) + if isdisk(O) + DA.readblock!(O, dst, r...) + else + dest[r...] .= view(parent(O), r...) + end end end function DA.writeblock!(A::FileArray, src, r::AbstractUnitRange...) open(A; write=A.write) do O - DA.writeblock!(O, src, r...) + if isdisk(A) + DA.writeblock!(O, src, r...) + else + parent(O)[r...] .= src + end end end diff --git a/src/modifieddiskarray.jl b/src/modifieddiskarray.jl index cabdf18f..2996ad06 100644 --- a/src/modifieddiskarray.jl +++ b/src/modifieddiskarray.jl @@ -1,5 +1,7 @@ -abstract type AbstractModifications end -struct NoMod{T,Mi} <: AbstractModifications + +abstract type AbstractModifications{T} end + +struct NoMod{T,Mi} <: AbstractModifications{T} missingval::Mi end NoMod{T}(missingval::Mi) where {T,Mi} = NoMod{T,Mi}(missingval) @@ -10,7 +12,8 @@ NoMod{T}(::NoKW) where T = NoMod{T}(nothing) Base.eltype(::NoMod{T}) where T = T source_eltype(::NoMod{T}) where T = T -struct Mod{T1,T2,Mi,S,O,F} <: AbstractModifications +# Modifies array values by scale +struct Mod{T1,T2,Mi,S,O,F} <: AbstractModifications{T1} missingval::Mi scale::S offset::O @@ -48,71 +51,73 @@ _outer_missingval(m::AbstractModifications) = _outer_missingval(m.missingval) _outer_missingval(mv::Pair) = mv[2] _outer_missingval(mv) = mv -struct ModifiedDiskArray{I,T,N,V,M} <: DiskArrays.AbstractDiskArray{T,N} - var::V +# A wrapper for disk arrays that modifieds values lazily: +# scale and offset or replacing missing values +struct ModifiedDiskArray{T,N,D,M} <: DiskArrays.AbstractDiskArray{T,N} + data::D mod::M end -function ModifiedDiskArray(v::V, m::M; invert=false) where {V<:AbstractArray{<:Any,N},M} where N - T = invert ? source_eltype(m) : eltype(m) - return ModifiedDiskArray{invert,T,N,V,M}(v, m) +function ModifiedDiskArray( + data::D, mod::M +) where {D<:AbstractArray{<:Any,N},M<:AbstractModifications{T}} where {T,N} + return ModifiedDiskArray{T,N,D,M}(data, mod) end -Base.parent(A::ModifiedDiskArray) = A.var -Base.size(A::ModifiedDiskArray, args...) = size(parent(A), args...) +_maybe_modify(A::AbstractArray, mod::AbstractModifications) = + ModifiedDiskArray(A, mod) +_maybe_modify(A::AbstractArray, ::Nothing) = A + filename(A::ModifiedDiskArray) = filename(parent(A)) missingval(A::ModifiedDiskArray) = A.missingval +_metadata(A::ModifiedDiskArray, args...) = _metadata(parent(A), args...) + +Base.parent(A::ModifiedDiskArray) = A.data +Base.size(A::ModifiedDiskArray, args...) = size(parent(A), args...) + DiskArrays.haschunks(A::ModifiedDiskArray) = DiskArrays.haschunks(parent(A)) DiskArrays.eachchunk(A::ModifiedDiskArray) = DiskArrays.eachchunk(parent(A)) function DiskArrays.readblock!( - A::ModifiedDiskArray{false,<:Any,0}, out_block, I::AbstractVector... + A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:Mod}, outer_block, I::AbstractVector... ) - out_block[] = _applymod(parent(A)[I...][], A.mod) - return out_block -end -function DiskArrays.readblock!( - A::ModifiedDiskArray{true,T,<:Any,0}, out_block, I::AbstractVector... -) where T - out_block[] = _invertmod(Val{T}(), parent(A)[I...], A.mod) - return out_block + if isdisk(parent(A)) + inner_block = similar(outer_block, eltype(parent(A))) + DiskArrays.readblock!(parent(A), inner_block, I...) + outer_block .= _applymod.(inner_block, (A.mod,)) + else + inner_block = view(parent(A), I...) + outer_block .= _applymod.(inner_block, (A.mod,)) + end + return outer_block end function DiskArrays.readblock!( - A::ModifiedDiskArray{false}, out_block, I::AbstractVector... + A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:NoMod}, out_block, I::AbstractVector... ) - out_block .= _applymod.(parent(A)[I...], (A.mod,)) - return out_block -end -function DiskArrays.readblock!( - A::ModifiedDiskArray{true,T}, out_block, I::AbstractVector... -) where T - out_block .= _invertmod.((Val{T}(),), parent(A)[I...], (A.mod,)) - return out_block + if isdisk(parent(A)) + DiskArrays.readblock!(parent(A), out_block, I...) + else + out_block .= view(parent(A), I...) + end end function DiskArrays.writeblock!( - A::ModifiedDiskArray{false,<:Any,0,<:AbstractArray{T}}, block, I::AbstractVector... -) where T - - parent(A)[I...] = _invertmod(Val{source_eltype(A.mod)}(), block[], A.mod) - return nothing -end -function DiskArrays.writeblock!( - A::ModifiedDiskArray{true,<:Any,0,<:AbstractArray{T}}, _block, I::AbstractVector... -) where T - parent(A)[I...] = _applymod(Val{eltype(A.mod)}(), block[], A.mod) - return nothing -end -function DiskArrays.writeblock!( - A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:AbstractArray{T}}, block, I::AbstractVector... -) where T - parent(A)[I...] = _invertmod.((Val{source_eltype(A.mod)}(),), block, (A.mod,)) - return nothing + A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:Mod}, block, I::AbstractVector... +) + if isdisk(parent(A)) + modblock = _invertmod.((Val{source_eltype(A.mod)}(),), block, (A.mod,)) + return DiskArrays.writeblock!(parent(A), modblock, I...) + else + parent(A)[I...] = _invertmod.((Val{source_eltype(A.mod)}(),), block, (A.mod,)) + end end function DiskArrays.writeblock!( - A::ModifiedDiskArray{true,<:Any,<:Any,<:AbstractArray{T}}, _block, I::AbstractVector... -) where T - parent(A)[I...] = _applymod.((Val{eltype(A.mod)}(),), block, (A.mod,)) - return nothing + A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:NoMod}, block, I::AbstractVector... +) + if isdisk(parent(A)) + DiskArrays.writeblock!(parent(A), block, I...) + else + parent(A)[I...] = block + end end Base.@assume_effects :foldable function _applymod(x, m::Mod) @@ -197,9 +202,6 @@ _mod_inverse_eltype(::AbstractArray{T}, ::NoMod) where T = T _mod_inverse_eltype(::AbstractArray{T}, m::Mod) where T = Base.promote_op(_invertmod, typeof(m.coerce), T, typeof(m)) -_maybe_modify(var, m::Mod; kw...) = ModifiedDiskArray(var, m; kw...) -_maybe_modify(var, ::NoMod; kw...) = var - _write_missingval_pair(A, missingval::Pair; kw...) = missingval function _write_missingval_pair(A, missingval; verbose=true, eltype, metadata=metadata(A), required=false diff --git a/src/openstack.jl b/src/openstack.jl index 0f3bec92..d885bbc5 100644 --- a/src/openstack.jl +++ b/src/openstack.jl @@ -17,7 +17,7 @@ struct OpenStack{X,K,T,DS,M} mods::M end function OpenStack{X,K,T}( - dataset::DS, mods::M=NoMod() + dataset::DS, mods::M ) where {X,K,T,DS,M} OpenStack{X,K,T,DS,M}(dataset, mods) end diff --git a/src/series.jl b/src/series.jl index 337eb871..eda31b9a 100644 --- a/src/series.jl +++ b/src/series.jl @@ -24,7 +24,8 @@ abstract type AbstractRasterSeries{T,N,D,A} <: AbstractDimArray{T,N,D,A} end DD.metadata(A::AbstractRasterSeries) = NoMetadata() DD.name(A::AbstractRasterSeries) = NoName() DD.label(A::AbstractRasterSeries) = "" -isdisk(A::AbstractRasterSeries) = any(isdisk, A) + +DiskArrays.isdisk(A::AbstractRasterSeries) = any(isdisk, A) """ modify(f, series::AbstractRasterSeries) diff --git a/src/sources/commondatamodel.jl b/src/sources/commondatamodel.jl index d247e094..31ae777d 100644 --- a/src/sources/commondatamodel.jl +++ b/src/sources/commondatamodel.jl @@ -499,10 +499,10 @@ function Base.write(filename::AbstractString, source::Source, s::AbstractRasterS ds = sourceconstructor(source)(filename, mode; attrib=_attribdict(metadata(s))) missingval = _stack_nt(s, isnokw(missingval) ? Rasters.missingval(s) : missingval) try - map(keys(s)) do k + mods = map(keys(s)) do k writevar!(ds, source, s[k]; missingval=missingval[k], kw...) end - f(OpenStack{Source,K,T}(ds)) + f(OpenStack{Source,K,T}(ds, mods)) finally close(ds) end @@ -578,7 +578,7 @@ function writevar!(ds::AbstractDataset, source::CDMsource, A::AbstractRaster{T,N f(m) end - return nothing + return mod end const CDMallowedType = Union{Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Float32,Float64,Char,String} diff --git a/src/stack.jl b/src/stack.jl index 37d97b3e..f3267184 100644 --- a/src/stack.jl +++ b/src/stack.jl @@ -36,7 +36,7 @@ filename(stack::AbstractRasterStack{<:Any,<:Any,<:Any,<:NamedTuple}) = filename(stack::AbstractRasterStack{<:Any,<:Any,<:Any,<:Union{FileStack,OpenStack}}) = filename(parent(stack)) -isdisk(st::AbstractRasterStack) = any(isdisk, layers(st)) +DiskArrays.isdisk(st::AbstractRasterStack) = any(isdisk, layers(st)) setcrs(x::AbstractRasterStack, crs) = set(x, setcrs(dims(x), crs)...) setmappedcrs(x::AbstractRasterStack, mappedcrs) = set(x, setmappedcrs(dims(x), mappedcrs)...) @@ -557,10 +557,9 @@ function _layer_stack(filename; FileStack{typeof(source)}(ds, filename; name, group, mods, vars) else map(layers.vars, layermetadata_vec, mod_vec) do var, md, mod - modvar = _maybe_modify(var, mod) + modvar = ModifiedDiskArray(var, mod) checkmem && _checkobjmem(modvar) - x = Array(modvar) - x isa AbstractArray ? x : fill(x) # Catch an NCDatasets bug + Array(modvar) end |> NT end mv_outer = NT(map(_outer_missingval, mod_vec)) diff --git a/test/runtests.jl b/test/runtests.jl index 33637e4f..9e9df342 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,7 +29,7 @@ end @time @safetestset "sources" begin include("sources/sources.jl") end @time @safetestset "commondatamodel" begin include("sources/commondatamodel.jl") end @time @safetestset "ncdatasets" begin include("sources/ncdatasets.jl") end -@time @safetestset "zarrdatasets" begin include("sources/zarr.jl") end +# @time @safetestset "zarrdatasets" begin include("sources/zarr.jl") end if !Sys.iswindows() # GRIBDatasets doesn't work on Windows for now @time @safetestset "gribdatasets" begin include("sources/gribdatasets.jl") end diff --git a/test/series.jl b/test/series.jl index e5ebb5ed..41d6cd97 100644 --- a/test/series.jl +++ b/test/series.jl @@ -127,5 +127,5 @@ end @test Rasters.isdisk(series) @test !Rasters.isdisk(read(series)) @test Rasters.isdisk(Rasters.combine(series)) - end + end end diff --git a/test/sources/grd.jl b/test/sources/grd.jl index dc6a87b7..a2ff0b84 100644 --- a/test/sources/grd.jl +++ b/test/sources/grd.jl @@ -325,12 +325,6 @@ end @test parent(eagerstack[:a]) isa Array end - @testset "replace_missing keyword" begin - st = RasterStack((a=grdpath, b=grdpath); replace_missing=true) - @test eltype(st) == @NamedTuple{a::Union{Missing,Float32},b::Union{Missing,Float32}} - @test missingval(st) === missing - end - @test length(layers(grdstack)) == 2 @test dims(grdstack) isa Tuple{<:X,<:Y,<:Band} diff --git a/test/sources/ncdatasets.jl b/test/sources/ncdatasets.jl index bf933742..925121a2 100644 --- a/test/sources/ncdatasets.jl +++ b/test/sources/ncdatasets.jl @@ -79,10 +79,10 @@ end @test parent(nocf_nomask_array) isa Array{Float32} @test parent(raw_array) isa Array{Float32} open(lazycfarray) do A - @test parent(A) isa Rasters.ModifiedDiskArray{false,Union{Missing,Float32}} + @test parent(A) isa Rasters.ModifiedDiskArray{Union{Missing,Float32}} end open(lazynocfarray) do A - @test parent(A) isa Rasters.ModifiedDiskArray{false,Union{Missing,Float32}} + @test parent(A) isa Rasters.ModifiedDiskArray{Union{Missing,Float32}} end open(lazynocf_nomask_array) do A @test parent(parent(A)) isa NCDatasets.Variable{Float32}