Skip to content

Commit

Permalink
tweak(Store/plugins): reduce fetch! overhead
Browse files Browse the repository at this point in the history
It seems that in huge data collections this can become a not
insignificant time sink.
  • Loading branch information
tecosaur committed Oct 12, 2024
1 parent 9e7b6f5 commit b576b8c
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 10 deletions.
23 changes: 14 additions & 9 deletions Store/src/inventory.jl
Original file line number Diff line number Diff line change
Expand Up @@ -658,18 +658,23 @@ If `storer` is storable (either by default, or explicitly enabled), open it, and
presumably save it in the Store along the way.
"""
function fetch!(@nospecialize(storer::DataStorage))
if shouldstore(storer) || @getparam(storer."save"::Bool, false) === true
for type in (FilePath, IO, IOStream)
if QualifiedType(type) in storer.type
handle = @advise storage(storer, type, write=false)
if handle isa IO
close(handle)
return true
elseif !isnothing(handle)
return true
global STORE_RECORD_ACCESS = false
try
if shouldstore(storer) || @getparam(storer."save"::Bool, false) === true
for type in (FilePath, IO, IOStream)
if QualifiedType(type) in storer.type
handle = @advise storage(storer, type, write=false)
if handle isa IO
close(handle)
return true
elseif !isnothing(handle)
return true
end
end
end
end
finally
STORE_RECORD_ACCESS = true
end
false
end
Expand Down
12 changes: 11 additions & 1 deletion Store/src/plugins.jl
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@ A few (system-wide) settings determine garbage collection behaviour:
# Store plugin
# ------------

"""
STORE_RECORD_ACCESS::Bool
Whether access to stored entries should be updated in the inventory.
This is usually appropriate, but may be worth disabling for collection-wide
checks, as performed by `fetch!` for instance.
"""
STORE_RECORD_ACCESS::Bool = true

"""
store_get_a( <storage(storer::DataStorage, as::Type; write::Bool)> )
Expand Down Expand Up @@ -55,7 +64,8 @@ function store_get_a(f::typeof(storage), @nospecialize(storer::DataStorage), as:
elseif !isnothing(file) && isfile(file)
# If using a cache file, ensure the parent collection is registered
# as a reference.
update_source!(inventory, source, storer.dataset.collection)
STORE_RECORD_ACCESS &&
update_source!(inventory, source, storer.dataset.collection)
if as === IO || as === IOStream
@log_do "store:open" "Opening $as for $(sprint(show, storer.dataset.name)) from the store"
(identity, (open(file, "r"),))
Expand Down

0 comments on commit b576b8c

Please sign in to comment.